class DitherSkyRecipe(EmirRecipe):
"""Recipe to process data taken in dither sky mode.
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
skyframe = Result(prods.MasterSky)
def run(self, rinput):
_logger.debug('instrument %s, mode %s', rinput.obresult.instrument,
rinput.obresult.mode)
_logger.info('starting sky reduction with dither')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_segmentation(rinput,
flow,
method=median,
errors=True)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.info('end sky reduction with dither')
result = self.create_result(skyframe=hdulist)
return result
class TestSkyCorrectRecipe(EmirRecipe):
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = Requirement(prods.MasterIntensityFlat,
'Master Sky calibration')
frame = Result(prods.ProcessedImage)
def run(self, rinput):
self.logger.info('starting simple sky reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput,
flow,
method=median)
hdr = hdulist[0].header
self.set_base_headers(hdr)
# Update SEC to 0
hdr['SEC'] = 0
result = self.create_result(frame=hdulist)
return result
class WavelengthCalibrationRecipe(EmirRecipe):
"""Recipe to calibrate the spectral response.
**Observing modes:**
* Wavelength calibration (4.5)
**Inputs:**
* List of line positions
* Calibrations up to spectral flatfielding
**Outputs:**
* Wavelength calibration structure
**Procedure:**
* TBD
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_spectral_ff = reqs.MasterSpectralFlatFieldRequirement()
cal = Result(prods.WavelengthCalibration)
def run(self, rinput):
return self.create_result(cal=prods.WavelengthCalibration())
class TelescopeFineFocusRecipe(EmirRecipe):
"""
Recipe to compute the telescope focus.
**Observing modes:**
* Telescope fine focus
**Inputs:**
* A list of images
* A list of sky images
* Bias, dark, flat
* A model of the detector
* List of focii
**Outputs:**
* Best focus
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
objects = Parameter([], 'List of x-y pair of object coordinates'),
focus = Product(TelescopeFocus)
def run(self, rinput):
return self.create_result(focus=TelescopeFocus())
class DTUFocusRecipe(EmirRecipe):
"""
Recipe to compute the DTU focus.
**Observing modes:**
* EMIR focus control
**Inputs:**
* A list of images
* A list of sky images
* Bias, dark, flat
* A model of the detector
* List of focii
**Outputs:**
* Best focus
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
objects = Parameter([], 'List of x-y pair of object coordinates'),
msm_pattern = Parameter([], 'List of x-y pair of slit coordinates'),
dtu_focus_range = Parameter(
'dtu_focus_range', [], 'Focus range of the DTU: begin, end and step')
focus = Product(DTUFocus)
def run(self, rinput):
return self.create_result(focus=DTUFocus())
class SimpleSkyRecipe(EmirRecipe):
"""Recipe to process data taken in intensity flat-field mode.
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
skyframe = Result(prods.MasterSky)
def run(self, rinput):
_logger.info('starting sky reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput,
flow,
method=median,
errors=True)
hdr = hdulist[0].header
self.set_base_headers(hdr)
result = self.create_result(skyframe=hdulist)
return result
class TelescopeRoughFocusRecipe(EmirRecipe):
"""Recipe to compute the telescope focus.
**Observing modes:**
* Telescope rough focus
* Emir focus control
**Inputs:**
* A list of images
* A list of sky images
* Bias, dark, flat
* A model of the detector
* List of focii
**Outputs:**
* Best focus
"""
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
objects = Parameter([], 'List of x-y pair of object coordinates'),
focus_range = Parameter([], 'Focus range: begin, end and step')
focus = Result(prods.TelescopeFocus)
def run(self, rinput):
return self.create_result(focus=prods.TelescopeFocus())
class StareImageRecipe2(EmirRecipe):
"""Process images in Stare Image Mode"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
result_image = Result(prods.ProcessedImage)
def run(self, rinput):
self.logger.info('starting stare image reduction (offline)')
frames = rinput.obresult.frames
datamodel = self.datamodel
with extra.manage_frame(frames) as list_of:
c_img = extra.combine_frames(list_of, datamodel, method=combine.mean, errors=False)
flow = self.init_filters(rinput)
# Correct Bias if needed
# Correct Dark if needed
# Correct FF
processed_img = flow(c_img)
hdr = processed_img[0].header
self.set_base_headers(hdr)
self.logger.debug('append BPM')
if rinput.master_bpm is not None:
self.logger.debug('using BPM from inputs')
hdul_bpm = rinput.master_bpm.open()
hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0])
else:
self.logger.debug('using empty BPM')
hdu_bpm = extra.generate_empty_bpm_hdu(processed_img[0])
# Append the BPM to the result
processed_img.append(hdu_bpm)
self.logger.info('end stare image (off) reduction')
result = self.create_result(result_image=processed_img)
return result
def set_base_headers(self, hdr):
"""Set metadata in FITS headers."""
hdr = super(StareImageRecipe2, self).set_base_headers(hdr)
# Set EXP to 0
hdr['EXP'] = 0
return hdr
class SpectralFlatRecipe(EmirRecipe):
"""Recipe to process data taken in intensity flat-field mode."""
master_bpm = reqs.MasterBadPixelMaskRequirement()
obresult = reqs.ObservationResultRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
flatframe = Result(prods.MasterSpectralFlat)
def run(self, rinput):
return self.create_result(flatframe=prods.MasterSpectralFlat())
class StareImageBaseRecipe(EmirRecipe):
"""Process images in Stare Image Mode"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement(optional=True)
frame = Result(prods.ProcessedImage)
def __init__(self, *args, **kwargs):
super(StareImageBaseRecipe, self).__init__(*args, **kwargs)
if False:
self.query_options['master_sky'] = Ignore()
@emirdrp.decorators.loginfo
@timeit
def run(self, rinput):
self.logger.info('starting stare image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(
rinput,
flow,
method=combine.median
)
hdr = hdulist[0].header
self.set_base_headers(hdr)
if rinput.master_bpm:
hdul_bpm = rinput.master_bpm.open()
hdu_bpm = extra.generate_bpm_hdu(hdul_bpm[0])
else:
hdu_bpm = extra.generate_empty_bpm_hdu(hdulist[0])
# Append the BPM to the result
hdulist.append(hdu_bpm)
self.logger.info('end stare image reduction')
result = self.create_result(frame=hdulist)
return result
def set_base_headers(self, hdr):
"""Set metadata in FITS headers."""
hdr = super(StareImageBaseRecipe, self).set_base_headers(hdr)
# Update EXP to 0
hdr['EXP'] = 0
return hdr
class MaskCheckRecipe(EmirRecipe):
"""
Acquire a target.
Recipe for the processing of multi-slit/long-slit check images.
**Observing modes:**
* MSM and LSM check
"""
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
def run(self, rinput):
self.logger.info("Start MaskCheckRecipe")
self.logger.info("End MaskCheckRecipe")
return self.create_result()
class BarDetectionRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.NominalPositions,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
average_box_row_size = Parameter(
7, 'Number of rows to average for fine centering (odd)')
average_box_col_size = Parameter(
21, 'Number of columns to extract for fine centering (odd)')
fit_peak_npoints = Parameter(
3, 'Number of points to use for fitting the peak (odd)')
# Recipe Products
frame = Result(prods.ProcessedImage)
# derivative = Result(prods.ProcessedImage)
slits = Result(tarray.ArrayType)
positions3 = Result(tarray.ArrayType)
positions5 = Result(tarray.ArrayType)
positions7 = Result(tarray.ArrayType)
positions9 = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
TSUTC1 = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
def run(self, rinput):
self.logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(hdulist, 'reduced_image.fits')
try:
rotang = hdr['ROTANG']
tsutc1 = hdr['TSUTC1']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
if len(csupos) != 2 * EMIR_NBARS:
raise RecipeError('Number of CSUPOS != 2 * NBARS')
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('start finding bars')
allpos, slits = find_bars(
hdulist,
rinput.bars_nominal_positions,
csupos,
dtur,
average_box_row_size=rinput.average_box_row_size,
average_box_col_size=rinput.average_box_col_size,
fit_peak_npoints=rinput.fit_peak_npoints,
median_filter_size=rinput.median_filter_size,
logger=self.logger)
self.logger.debug('end finding bars')
if self.intermediate_results:
with open('ds9.reg', 'w') as ds9reg:
slits_to_ds9_reg(ds9reg, slits)
result = self.create_result(
frame=hdulist,
slits=slits,
positions9=allpos[9],
positions7=allpos[7],
positions5=allpos[5],
positions3=allpos[3],
DTU=dtub,
ROTANG=rotang,
TSUTC1=tsutc1,
csupos=csupos,
csusens=csusens,
)
return result
class MaskImagingRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.CoordinateList2DType,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
average_box_row_size = Parameter(
7, 'Number of rows to average for fine centering (odd)')
average_box_col_size = Parameter(
21, 'Number of columns to extract for fine centering (odd)')
fit_peak_npoints = Parameter(
3, 'Number of points to use for fitting the peak (odd)')
# Recipe Products
frame = Result(prods.ProcessedImage)
# derivative = Result(prods.ProcessedImage)
slits = Result(tarray.ArrayType)
positions3 = Result(tarray.ArrayType)
positions5 = Result(tarray.ArrayType)
positions7 = Result(tarray.ArrayType)
positions9 = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
TSUTC1 = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
def run(self, rinput):
self.logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
tsutc1 = hdr['TSUTC1']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise numina.exceptions.RecipeError(error)
self.logger.debug('finding bars')
# Processed array
arr = hdulist[0].data
# Median filter of processed array (two times)
mfilter_size = rinput.median_filter_size
self.logger.debug('median filtering X, %d columns', mfilter_size)
arr_median = median_filter(arr, size=(1, mfilter_size))
self.logger.debug('median filtering X, %d rows', mfilter_size)
arr_median = median_filter(arr_median, size=(mfilter_size, 1))
# Median filter of processed array (two times) in the other direction
# for Y coordinates
self.logger.debug('median filtering Y, %d rows', mfilter_size)
arr_median_alt = median_filter(arr, size=(mfilter_size, 1))
self.logger.debug('median filtering Y, %d columns', mfilter_size)
arr_median_alt = median_filter(arr_median_alt, size=(1, mfilter_size))
xfac = dtur[0] / EMIR_PIXSCALE
yfac = -dtur[1] / EMIR_PIXSCALE
vec = [yfac, xfac]
self.logger.debug('DTU shift is %s', vec)
# and the table of approx positions of the slits
barstab = rinput.bars_nominal_positions
# Currently, we only use fields 0 and 2
# of the nominal positions file
# Number or rows used
# These other parameters cab be tuned also
bstart = 1
bend = 2047
self.logger.debug('ignoring columns outside %d - %d', bstart, bend - 1)
# extract a region to average
wy = (rinput.average_box_row_size // 2)
wx = (rinput.average_box_col_size // 2)
self.logger.debug('extraction window is %d rows, %d cols', 2 * wy + 1,
2 * wx + 1)
# Fit the peak with these points
wfit = 2 * (rinput.fit_peak_npoints // 2) + 1
self.logger.debug('fit with %d points', wfit)
# Minimum threshold
threshold = 5 * EMIR_RON
# Savitsky and Golay (1964) filter to compute the X derivative
# scipy >= xx has a savgol_filter function
# for compatibility we do it manually
allpos = {}
ypos3_kernel = None
slits = numpy.zeros((EMIR_NBARS, 8), dtype='float')
self.logger.info('start finding bars')
for ks in [3, 5, 7, 9]:
self.logger.debug('kernel size is %d', ks)
# S and G kernel for derivative
kw = ks * (ks * ks - 1) / 12.0
coeffs_are = -numpy.arange((1 - ks) // 2, (ks - 1) // 2 + 1) / kw
if ks == 3:
ypos3_kernel = coeffs_are
self.logger.debug('kernel weights are %s', coeffs_are)
self.logger.debug('derive image in X direction')
arr_deriv = convolve1d(arr_median, coeffs_are, axis=-1)
# Axis 0 is
#
self.logger.debug('derive image in Y direction (with kernel=3)')
arr_deriv_alt = convolve1d(arr_median_alt, ypos3_kernel, axis=0)
positions = []
for coords in barstab:
lbarid = int(coords[0])
rbarid = lbarid + EMIR_NBARS
ref_y_coor = coords[1] + vec[1]
poly_coeffs = coords[2:]
prow = coor_to_pix_1d(ref_y_coor) - 1
fits_row = prow + 1 # FITS pixel index
# A function that returns the center of the bar
# given its X position
def center_of_bar(x):
# Pixel values are 0-based
return polyval(x + 1 - vec[0], poly_coeffs) + vec[1] - 1
self.logger.debug('looking for bars with ids %d - %d', lbarid,
rbarid)
self.logger.debug('reference y position is Y %7.2f',
ref_y_coor)
# if ref_y_coor is outlimits, skip this bar
# ref_y_coor is in FITS format
if (ref_y_coor >= 2047) or (ref_y_coor <= 1):
self.logger.debug(
'reference y position is outlimits, skipping')
positions.append([lbarid, fits_row, fits_row, 1, 0, 3])
positions.append([rbarid, fits_row, fits_row, 1, 0, 3])
continue
# Left bar
self.logger.debug('measure left border (%d)', lbarid)
centery, xpos, fwhm, st = char_bar_peak_l(arr_deriv,
prow,
bstart,
bend,
threshold,
center_of_bar,
wx=wx,
wy=wy,
wfit=wfit)
xpos1 = xpos
positions.append(
[lbarid, centery + 1, fits_row, xpos + 1, fwhm, st])
# Right bar
self.logger.debug('measure rigth border (%d)', rbarid)
centery, xpos, fwhm, st = char_bar_peak_r(arr_deriv,
prow,
bstart,
bend,
threshold,
center_of_bar,
wx=wx,
wy=wy,
wfit=wfit)
positions.append(
[rbarid, centery + 1, fits_row, xpos + 1, fwhm, st])
xpos2 = xpos
#
if st == 0:
self.logger.debug('measure top-bottom borders')
try:
y1, y2, statusy = char_bar_height(arr_deriv_alt,
xpos1,
xpos2,
centery,
threshold,
wh=35,
wfit=wfit)
except Exception as error:
self.logger.warning('Error computing height: %s',
error)
statusy = 44
if statusy in [0, 40]:
# Main border is detected
positions[-1][1] = y2 + 1
positions[-2][1] = y2 + 1
else:
# Update status
positions[-1][-1] = 4
positions[-2][-1] = 4
else:
self.logger.debug('slit is not complete')
y1, y2 = 0, 0
# Update positions
self.logger.debug(
'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d',
*positions[-2])
self.logger.debug(
'bar %d centroid-y %9.4f, row %d x-pos %9.4f, FWHM %6.3f, status %d',
*positions[-1])
if ks == 5:
slits[lbarid -
1] = [xpos1, y2, xpos2, y2, xpos2, y1, xpos1, y1]
# FITS coordinates
slits[lbarid - 1] += 1.0
self.logger.debug('inserting bars %d-%d into "slits"',
lbarid, rbarid)
allpos[ks] = numpy.asarray(
positions, dtype='float') # GCS doesn't like lists of lists
self.logger.debug('end finding bars')
result = self.create_result(
frame=hdulist,
slits=slits,
positions9=allpos[9],
positions7=allpos[7],
positions5=allpos[5],
positions3=allpos[3],
DTU=dtub,
ROTANG=rotang,
TSUTC1=tsutc1,
csupos=csupos,
csusens=csusens,
)
return result
class TestSlitMaskDetectionRecipe(EmirRecipe):
# Recipe Requirements
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the Canny algorithm')
canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm')
canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm')
obj_min_size = Parameter(200, 'Minimum size of the slit')
obj_max_size = Parameter(3000, 'Maximum size of the slit')
slit_size_ratio = Parameter(4.0, 'Minimum ratio between height and width for slits')
# Recipe Results
frame = Result(prods.DataFrameType)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
def run(self, rinput):
self.logger.info('starting slit processing')
self.logger.info('basic image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('finding slits')
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
obj_min_size = rinput.obj_min_size
obj_max_size = rinput.obj_max_size
data1 = hdulist[0].data
self.logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize_raw(data2)
# Find edges with Canny
self.logger.debug('Find edges with Canny, sigma %f', canny_sigma)
# These thresholds corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
self.logger.debug('Find edges, Canny high threshold %f', high_threshold)
self.logger.debug('Find edges, Canny low threshold %f', low_threshold)
edges = canny(img_grey, sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Fill edges
self.logger.debug('Fill holes')
fill_slits = ndimage.binary_fill_holes(edges)
self.logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
self.logger.debug('%d objects found', nb_labels)
# Filter on the area of the labeled region
# Perhaps we could ignore this filtering and
# do it later?
self.logger.debug('Filter objects by size')
# Sizes of regions
sizes = numpy.bincount(label_objects.ravel())
self.logger.debug('Min size is %d', obj_min_size)
self.logger.debug('Max size is %d', obj_max_size)
mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size)
# Filter out regions
nids, = numpy.where(mask_sizes)
mm = numpy.in1d(label_objects, nids)
mm.shape = label_objects.shape
fill_slits_clean = numpy.where(mm, 1, 0)
#plt.imshow(fill_slits_clean)
# and relabel
self.logger.debug('Label filtered objects')
relabel_objects, nb_labels = ndimage.label(fill_slits_clean)
self.logger.debug('%d objects found after filtering', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
self.logger.debug('Find regions and centers')
regions = ndimage.find_objects(relabel_objects)
centers = ndimage.center_of_mass(data2, labels=relabel_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=rinput.slit_size_ratio
)
result = self.create_result(frame=hdulist, slitstable=table,
DTU=dtub,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa
)
return result
class TestSlitDetectionRecipe(EmirRecipe):
# Recipe Requirements
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
canny_high_threshold = Parameter(0.04, 'High threshold for the Canny algorithm')
canny_low_threshold = Parameter(0.01, 'High threshold for the Canny algorithm')
# Recipe Results
frame = Result(prods.ProcessedImage)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
def run(self, rinput):
self.logger.info('starting slit processing')
self.logger.info('basic image reduction')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
self.logger.debug('finding slits')
# Filter values below 0.0
self.logger.debug('Filter values below 0')
data1 = hdulist[0].data[:]
data1[data1 < 0.0] = 0.0
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
self.logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize_raw(data2)
# Find edges with Canny
self.logger.debug('Find edges, Canny sigma %f', canny_sigma)
# These thresholds corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
self.logger.debug('Find edges, Canny high threshold %f', high_threshold)
self.logger.debug('Find edges, Canny low threshold %f', low_threshold)
edges = canny(img_grey, sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Fill edges
self.logger.debug('Fill holes')
# I do a dilation and erosion to fill
# possible holes in 'edges'
fill = ndimage.binary_dilation(edges)
fill2 = ndimage.binary_fill_holes(fill)
fill_slits = ndimage.binary_erosion(fill2)
self.logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
self.logger.debug('%d objects found', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
self.logger.debug('Find regions and centers')
regions = ndimage.find_objects(label_objects)
centers = ndimage.center_of_mass(data2, labels=label_objects,
index=ids
)
table = char_slit(data2, regions,
slit_size_ratio=-1.0
)
result = self.create_result(frame=hdulist,
slitstable=table,
DTU=dtub,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa
)
return result
class BarDetectionRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
bars_nominal_positions = Requirement(prods.CoordinateList2DType,
'Nominal positions of the bars')
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
canny_high_threshold = Parameter(0.04,
'High threshold for the canny algorithm')
canny_low_threshold = Parameter(0.01,
'High threshold for the canny algorithm')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
ROTANG = Result(float)
csupos = Result(tarray.ArrayType)
csusens = Result(tarray.ArrayType)
param_median_filter_size = Result(float)
param_canny_high_threshold = Result(float)
param_canny_low_threshold = Result(float)
def run(self, rinput):
logger = logging.getLogger('numina.recipes.emir')
logger.info('starting processing for bars detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
try:
rotang = hdr['ROTANG']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
csupos = datamodel.get_csup_from_header(hdr)
csusens = datamodel.get_cs_from_header(hdr)
except KeyError as error:
logger.error(error)
raise numina.exceptions.RecipeError(error)
logger.debug('finding bars')
arr = hdulist[0].data
# Median filter
logger.debug('median filtering')
mfilter_size = rinput.median_filter_size
arr_median = median_filter(arr, size=mfilter_size)
# Image is mapped between 0 and 1
# for the full range [0: 2**16]
logger.debug('image scaling to 0-1')
arr_grey = normalize_raw(arr_median)
# Find borders
logger.debug('find borders')
canny_sigma = rinput.canny_sigma
# These threshols corespond roughly with
# value x (2**16 - 1)
high_threshold = rinput.canny_high_threshold
low_threshold = rinput.canny_low_threshold
edges = canny(arr_grey,
sigma=canny_sigma,
high_threshold=high_threshold,
low_threshold=low_threshold)
# Number or rows used
# These other parameters cab be tuned also
total = 5
maxdist = 1.0
bstart = 100
bend = 1900
positions = []
nt = total // 2
xfac = dtur[0] / EMIR_PIXSCALE
yfac = -dtur[1] / EMIR_PIXSCALE
vec = [yfac, xfac]
logger.debug('DTU shift is %s', vec)
# Based on the 'edges image'
# and the table of approx positions of the slits
barstab = rinput.bars_nominal_positions
# Currently, we only use fields 0 and 2
# of the nominal positions file
for coords in barstab:
lbarid = int(coords[0])
rbarid = lbarid + 55
ref_y_coor = coords[2] + vec[1]
prow = coor_to_pix_1d(ref_y_coor) - 1
fits_row = prow + 1 # FITS pixel index
logger.debug('looking for bars with ids %d - %d', lbarid, rbarid)
logger.debug('reference y position is Y %7.2f', ref_y_coor)
# Find the position of each bar
bpos = find_position(edges, prow, bstart, bend, total)
nbars_found = len(bpos)
# If no bar is found, append and empty token
if nbars_found == 0:
logger.debug('bars %d, %d not found at row %d', lbarid, rbarid,
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
elif nbars_found == 2:
# Order values by increasing X
centl, centr = sorted(bpos, key=lambda cen: cen[0])
c1 = centl[0]
c2 = centr[0]
logger.debug('bars found at row %d between %7.2f - %7.2f',
fits_row, c1, c2)
# Compute FWHM of the collapsed profile
cslit = arr_grey[prow - nt:prow + nt + 1, :]
pslit = cslit.mean(axis=0)
# Add 1 to return FITS coordinates
epos, epos_f, error = locate_bar_l(pslit, c1)
thisres1 = lbarid, fits_row, epos + 1, epos_f + 1, error
epos, epos_f, error = locate_bar_r(pslit, c2)
thisres2 = rbarid, fits_row, epos + 1, epos_f + 1, error
elif nbars_found == 1:
logger.warning(
'only 1 edge found at row %d, not yet implemented',
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
else:
logger.warning(
'3 or more edges found at row %d, not yet implemented',
fits_row)
thisres1 = (lbarid, fits_row, 0, 0, 1)
thisres2 = (rbarid, fits_row, 0, 0, 1)
positions.append(thisres1)
positions.append(thisres2)
logger.debug('end finding bars')
result = self.create_result(
frame=hdulist,
positions=positions,
DTU=dtub,
ROTANG=rotang,
csupos=csupos,
csusens=csusens,
param_median_filter_size=rinput.median_filter_size,
param_canny_high_threshold=rinput.canny_high_threshold,
param_canny_low_threshold=rinput.canny_low_threshold)
return result
class TestPinholeRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
pinhole_nominal_positions = Requirement(
prods.CoordinateList2DType, 'Nominal positions of the pinholes')
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
_logger.info('starting processing for slit detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.debug('finding pinholes')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
_logger.error(error)
raise numina.exceptions.RecipeError(error)
if rinput.shift_coordinates:
xdtur, ydtur, zdtur = dtur
xfac = xdtur / EMIR_PIXSCALE
yfac = -ydtur / EMIR_PIXSCALE
vec = numpy.array([yfac, xfac])
_logger.info('shift is %s', vec)
ncenters = rinput.pinhole_nominal_positions + vec
else:
_logger.info('using pinhole coordinates as they are')
ncenters = rinput.pinhole_nominal_positions
_logger.info('pinhole characterization')
positions = pinhole_char(hdulist[0].data,
ncenters,
box=rinput.box_half_size,
recenter_pinhole=rinput.recenter,
maxdist=rinput.max_recenter_radius)
_logger.info('alternate pinhole characterization')
positions_alt = pinhole_char2(
hdulist[0].data,
ncenters,
recenter_pinhole=rinput.recenter,
recenter_half_box=rinput.box_half_size,
recenter_maxdist=rinput.max_recenter_radius)
result = self.create_result(
frame=hdulist,
positions=positions,
positions_alt=positions_alt,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class TestPointSourceRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
self.logger.info('starting processing for object detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
self.logger.debug('finding point sources')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
self.logger.error(error)
raise RecipeError(error)
data = hdulist[0].data
# Copy needed in numpy 1.7
# This seems already bitswapped??
# FIXME: check this works offline/online
# ndata = data.byteswap().newbyteorder()
# data = data.byteswap(inplace=True).newbyteorder()
snr_detect = 5.0
fwhm = 4.0
npixels = 15
box_shape = [64, 64]
self.logger.info('point source detection2')
self.logger.info('using internal mask to remove corners')
# Corners
mask = numpy.zeros_like(data, dtype='int32')
mask[2000:, 0:80] = 1
mask[2028:, 2000:] = 1
mask[:50, 1950:] = 1
mask[:100, :50] = 1
# Remove corner regions
self.logger.info('compute background map, %s', box_shape)
bkg = sep.Background(data)
self.logger.info('reference fwhm is %5.1f pixels', fwhm)
self.logger.info('detect threshold, %3.1f over background', snr_detect)
self.logger.info('convolve with gaussian kernel, FWHM %3.1f pixels',
fwhm)
sigma = fwhm * gaussian_fwhm_to_sigma
#
kernel = Gaussian2DKernel(sigma)
kernel.normalize()
thresh = snr_detect * bkg.globalrms
data_s = data - bkg.back()
objects, segmap = sep.extract(data - bkg.back(),
thresh,
minarea=npixels,
filter_kernel=kernel.array,
segmentation_map=True,
mask=mask)
fits.writeto('segmap.fits', segmap)
self.logger.info('detected %d objects', len(objects))
# Hardcoded values
rs2 = 15.0
fit_rad = 10.0
flux_min = 1000.0
flux_max = 30000.0
self.logger.debug('Flux limit is %6.1f %6.1f', flux_min, flux_max)
# FIXME: this should be a view, not a copy
xall = objects['x']
yall = objects['y']
mm = numpy.array([xall, yall]).T
self.logger.info('computing FWHM')
# Find objects with pairs inside fit_rad
kdtree = KDTree(mm)
nearobjs = (kdtree.query_ball_tree(kdtree, r=fit_rad))
positions = []
for idx, obj in enumerate(objects):
x0 = obj['x']
y0 = obj['y']
sl = image_box2d(x0, y0, data.shape, (fit_rad, fit_rad))
# sl_sky = image_box2d(x0, y0, data.shape, (rs2, rs2))
part_s = data_s[sl]
# Logical coordinates
xx0 = x0 - sl[1].start
yy0 = y0 - sl[0].start
_, fwhm_x, fwhm_y = compute_fwhm_2d_simple(part_s, xx0, yy0)
if min(fwhm_x, fwhm_x) < 3:
continue
if flux_min > obj['peak'] or flux_max < obj['peak']:
continue
# nobjs is the number of object inside fit_rad
nobjs = len(nearobjs[idx])
flag = 0 if nobjs == 1 else 1
positions.append([idx, x0, y0, obj['peak'], fwhm_x, fwhm_y, flag])
self.logger.info('saving photometry')
positions = numpy.array(positions)
positions_alt = positions
self.logger.info('end processing for object detection')
result = self.create_result(
frame=hdulist,
positions=positions_alt,
positions_alt=positions_alt,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
class TestMaskRecipe(EmirRecipe):
# Recipe Requirements
#
obresult = reqs.ObservationResultRequirement()
master_bpm = reqs.MasterBadPixelMaskRequirement()
master_bias = reqs.MasterBiasRequirement()
master_dark = reqs.MasterDarkRequirement()
master_flat = reqs.MasterIntensityFlatFieldRequirement()
master_sky = reqs.MasterSkyRequirement()
pinhole_nominal_positions = Requirement(
prods.CoordinateList2DType, 'Nominal positions of the pinholes')
shift_coordinates = Parameter(
True, 'Use header information to'
' shift the pinhole positions from (0,0) '
'to X_DTU, Y_DTU')
box_half_size = Parameter(4, 'Half of the computation box size in pixels')
recenter = Parameter(True, 'Recenter the pinhole coordinates')
max_recenter_radius = Parameter(2.0, 'Maximum distance for recentering')
median_filter_size = Parameter(5, 'Size of the median box')
canny_sigma = Parameter(3.0, 'Sigma for the canny algorithm')
obj_min_size = Parameter(200, 'Minimum size of the slit')
obj_max_size = Parameter(3000, 'Maximum size of the slit')
slit_size_ratio = Parameter(
4.0, 'Minimum ratio between height and width for slits')
# Recipe Results
frame = Result(prods.ProcessedImage)
positions = Result(tarray.ArrayType)
positions_alt = Result(tarray.ArrayType)
slitstable = Result(tarray.ArrayType)
DTU = Result(tarray.ArrayType)
filter = Result(str)
readmode = Result(str)
ROTANG = Result(float)
DETPA = Result(float)
DTUPA = Result(float)
param_recenter = Result(bool)
param_max_recenter_radius = Result(float)
param_box_half_size = Result(float)
def run(self, rinput):
_logger.info('starting processing for slit detection')
flow = self.init_filters(rinput)
hdulist = basic_processing_with_combination(rinput, flow=flow)
hdr = hdulist[0].header
self.set_base_headers(hdr)
_logger.debug('finding pinholes')
try:
filtername = hdr['FILTER']
readmode = hdr['READMODE']
rotang = hdr['ROTANG']
detpa = hdr['DETPA']
dtupa = hdr['DTUPA']
dtub, dtur = datamodel.get_dtur_from_header(hdr)
except KeyError as error:
_logger.error(error)
raise numina.exceptions.RecipeError(error)
if rinput.shift_coordinates:
xdtur, ydtur, zdtur = dtur
xfac = xdtur / EMIR_PIXSCALE
yfac = -ydtur / EMIR_PIXSCALE
vec = numpy.array([yfac, xfac])
_logger.info('shift is %s', vec)
ncenters = rinput.pinhole_nominal_positions + vec
else:
_logger.info('using pinhole coordinates as they are')
ncenters = rinput.pinhole_nominal_positions
_logger.info('pinhole characterization')
positions = pinhole_char(hdulist[0].data,
ncenters,
box=rinput.box_half_size,
recenter_pinhole=rinput.recenter,
maxdist=rinput.max_recenter_radius)
_logger.info('alternate pinhole characterization')
positions_alt = pinhole_char2(
hdulist[0].data,
ncenters,
recenter_pinhole=rinput.recenter,
recenter_half_box=rinput.box_half_size,
recenter_maxdist=rinput.max_recenter_radius)
_logger.debug('finding slits')
# First, prefilter with median
median_filter_size = rinput.median_filter_size
canny_sigma = rinput.canny_sigma
obj_min_size = rinput.obj_min_size
obj_max_size = rinput.obj_max_size
data1 = hdulist[0].data
_logger.debug('Median filter with box %d', median_filter_size)
data2 = median_filter(data1, size=median_filter_size)
# Grey level image
img_grey = normalize(data2)
# Find edges with canny
_logger.debug('Find edges with canny, sigma %d', canny_sigma)
edges = canny(img_grey, sigma=canny_sigma)
# Fill edges
_logger.debug('Fill holes')
fill_slits = ndimage.binary_fill_holes(edges)
_logger.debug('Label objects')
label_objects, nb_labels = ndimage.label(fill_slits)
_logger.debug('%d objects found', nb_labels)
# Filter on the area of the labeled region
# Perhaps we could ignore this filtering and
# do it later?
_logger.debug('Filter objects by size')
# Sizes of regions
sizes = numpy.bincount(label_objects.ravel())
_logger.debug('Min size is %d', obj_min_size)
_logger.debug('Max size is %d', obj_max_size)
mask_sizes = (sizes > obj_min_size) & (sizes < obj_max_size)
# Filter out regions
nids, = numpy.where(mask_sizes)
mm = numpy.in1d(label_objects, nids)
mm.shape = label_objects.shape
fill_slits_clean = numpy.where(mm, 1, 0)
# and relabel
_logger.debug('Label filtered objects')
relabel_objects, nb_labels = ndimage.label(fill_slits_clean)
_logger.debug('%d objects found after filtering', nb_labels)
ids = list(six.moves.range(1, nb_labels + 1))
_logger.debug('Find regions and centers')
regions = ndimage.find_objects(relabel_objects)
centers = ndimage.center_of_mass(data2,
labels=relabel_objects,
index=ids)
table = char_slit(data2,
regions,
slit_size_ratio=rinput.slit_size_ratio)
result = self.create_result(
frame=hdulist,
positions=positions,
positions_alt=positions_alt,
slitstable=table,
filter=filtername,
DTU=dtub,
readmode=readmode,
ROTANG=rotang,
DETPA=detpa,
DTUPA=dtupa,
param_recenter=rinput.recenter,
param_max_recenter_radius=rinput.max_recenter_radius,
param_box_half_size=rinput.box_half_size)
return result
