def get_corrector_s(rinput, meta, ins, datamodel):
sky_info = meta.get('master_sky')
if sky_info is None:
return numina.util.node.IdNode()
else:
with rinput.master_sky.open() as hdul:
_logger.info('loading sky')
_logger.debug('sky info: %s', sky_info)
sky_corrector = proc.SkyCorrector(
hdul[0].data,
datamodel=datamodel,
calibid=datamodel.get_imgid(hdul))
return sky_corrector
def run_single(self, rinput):
# FIXME: remove this, is deprecated
obresult = rinput.obresult
# just in case images are in result, instead of frames
if not obresult.frames:
frames = obresult.results
else:
frames = obresult.frames
img_info = []
data_hdul = []
for f in frames:
img = f.open()
data_hdul.append(img)
info = {}
info['tstamp'] = img[0].header['tstamp']
info['airmass'] = img[0].header['airmass']
img_info.append(info)
channels = FULL
use_errors = True
# Initial checks
baseimg = data_hdul[0]
has_num_ext = 'NUM' in baseimg
has_bpm_ext = 'BPM' in baseimg
baseshape = baseimg[0].shape
subpixshape = baseshape
base_header = baseimg[0].header
compute_sky = 'NUM-SK' not in base_header
compute_sky_advanced = False
self.logger.debug('base image is: %s',
self.datamodel.get_imgid(baseimg))
self.logger.debug('images have NUM extension: %s', has_num_ext)
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
self.logger.debug('compute sky is needed: %s', compute_sky)
if compute_sky:
self.logger.info('compute sky simple')
sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
self.save_intermediate_img(sky_result, 'sky_init.fits')
sky_result.writeto('sky_init.fits', overwrite=True)
sky_data = sky_result[0].data
self.logger.debug('sky image has shape %s', sky_data.shape)
self.logger.info('sky correction in individual images')
corrector = proc.SkyCorrector(
sky_data,
self.datamodel,
calibid=self.datamodel.get_imgid(sky_result))
# If we do not update keyword SKYADD
# there is no sky subtraction
for m in data_hdul:
m[0].header['SKYADD'] = True
# this is a little hackish
# sky corrected
data_hdul_s = [corrector(m) for m in data_hdul]
base_header = data_hdul_s[0][0].header
else:
sky_result = None
data_hdul_s = data_hdul
self.logger.info('Computing offsets from WCS information')
finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
data_hdul_s, baseshape, subpixshape)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
# Resizing target imgs
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if has_num_ext:
self.logger.debug('Using NUM extension')
masks = [
numpy.where(m['NUM'].data, 0, 1).astype('int16')
for m in data_hdul
]
elif has_bpm_ext:
self.logger.debug('Using BPM extension')
#
masks = [
numpy.where(m['BPM'].data, 1, 0).astype('int16')
for m in data_hdul
]
else:
self.logger.warning('BPM missing, use zeros instead')
false_mask = numpy.zeros(baseshape, dtype='int16')
masks = [false_mask for _ in data_arr_sr]
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp,
use_errors)
self.save_intermediate_img(hdulist, 'result_initial1.fits')
compute_cross_offsets = True
if compute_cross_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions_c = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions_c = self.compute_regions_from_objs(hdulist[0].data,
finalshape,
box=20)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
data_arr_sr, regions_c, refine=True, tol=1)
#
# Combined offsets
# Offsets in numpy order, swaping
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = offsets_xy_t[:, ::-1]
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape, offsetsp = narray.combine_shape(
subpixshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
self.logger.info('Shape of resized array (crosscorr) is %s',
finalshape)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate2 img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r,
'interm2_%03d.fits' % idx)
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_initial2.fits')
except Exception as error:
self.logger.warning('Error during cross-correlation, %s',
error)
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
data_arr_0 = [(d[r] + s)
for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
data_arr_r = [d.copy() for d in data_arr_sr]
for inum in range(1, rinput.iterations + 1):
# superflat
sf_data = self.compute_superflat(data_arr_0, objmask, regions,
channels)
fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
# apply superflat
data_arr_rf = data_arr_r
for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
arr_f = arr / sf_data
#arr_f = arr
base[reg] = arr_f
# compute sky advanced
data_arr_sky = []
data_arr_rfs = []
self.logger.info('Step %d, SC: computing advanced sky', inum)
scale = rinput.sky_images_sep_time * 60
tstamps = numpy.array([info['tstamp'] for info in img_info])
for idx, hdu in enumerate(data_hdul):
diff1 = tstamps - tstamps[idx]
idxs1 = (diff1 > 0) & (diff1 < scale)
idxs2 = (diff1 < 0) & (diff1 > -scale)
l1, = numpy.nonzero(idxs1)
l2, = numpy.nonzero(idxs2)
limit1 = l1[-rinput.sky_images:]
limit2 = l2[:rinput.sky_images]
len_l1 = len(limit1)
len_l2 = len(limit2)
self.logger.info('For image %s, using %d-%d images)', idx,
len_l1, len_l2)
if len_l1 + len_l2 == 0:
self.logger.error('No sky image available for frame %d',
idx)
raise ValueError('No sky image')
skydata = []
skymasks = []
skyscales = []
my_region = regions[idx]
my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
for i in numpy.concatenate((limit1, limit2)):
region_s = regions[i]
data_s = data_arr_rf[i][region_s]
mask_s = objmask[region_s]
scale_s = numpy.median(data_s)
skydata.append(data_s)
skymasks.append(mask_s)
skyscales.append(scale_s)
self.logger.debug('computing background with %d frames',
len(skydata))
sky, _, num = nacom.median(skydata, skymasks, scales=skyscales)
# rescale
sky *= my_sky_scale
binmask = num == 0
if numpy.any(binmask):
# We have pixels without
# sky background information
self.logger.warn(
'pixels without sky information when correcting %d',
idx)
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
narray.fixpix2(sky, binmask, out=sky, iterations=1)
name = 'sky_%d_%03d.fits' % (inum, idx)
fits.writeto(name, sky, overwrite=True)
name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
data_arr_sky.append(sky)
arr = numpy.copy(data_arr_rf[idx])
arr[my_region] = data_arr_rf[idx][my_region] - sky
data_arr_rfs.append(arr)
# subtract sky advanced
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_rfs):
self.save_intermediate_array(
arr_r, 'interm_%d_%03d.fits' % (inum, idx))
hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)
# For next step
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_0 = [
(d[r] + s)
for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)
]
data_arr_r = [d.copy() for d in data_arr_rfs]
result = self.create_result(frame=hdulist)
self.logger.info('end of dither recipe')
return result
def run_single(self, rinput):
# Open all images
obresult = rinput.obresult
data_hdul = []
for f in obresult.frames:
img = f.open()
data_hdul.append(img)
use_errors = True
# Initial checks
baseimg = data_hdul[0]
has_num_ext = 'NUM' in baseimg
has_bpm_ext = 'BPM' in baseimg
baseshape = baseimg[0].shape
subpixshape = baseshape
base_header = baseimg[0].header
compute_sky = 'NUM-SK' not in base_header
compute_sky_advanced = False
self.logger.debug('base image is: %s',
self.datamodel.get_imgid(baseimg))
self.logger.debug('images have NUM extension: %s', has_num_ext)
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
self.logger.debug('compute sky is needed: %s', compute_sky)
if compute_sky:
self.logger.info('compute sky simple')
sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
self.save_intermediate_img(sky_result, 'sky_init.fits')
sky_result.writeto('sky_init.fits', overwrite=True)
sky_data = sky_result[0].data
self.logger.debug('sky image has shape %s', sky_data.shape)
self.logger.info('sky correction in individual images')
corrector = proc.SkyCorrector(
sky_data,
self.datamodel,
calibid=self.datamodel.get_imgid(sky_result))
# If we do not update keyword SKYADD
# there is no sky subtraction
for m in data_hdul:
m[0].header['SKYADD'] = True
# this is a little hackish
# sky corrected
data_hdul_s = [corrector(m) for m in data_hdul]
base_header = data_hdul_s[0][0].header
else:
sky_result = None
data_hdul_s = data_hdul
self.logger.info('Computing offsets from WCS information')
finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
data_hdul_s, baseshape, subpixshape)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
# Resizing target imgs
data_arr_sr, regions = resize_arrays([m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)
hdulist = self.combine(data_arr_sr, data_hdul, finalshape, offsetsp,
refpix, use_errors)
self.save_intermediate_img(hdulist, 'result_initial1.fits')
compute_cross_offsets = True
if compute_cross_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions = self.compute_regions_from_objs(hdulist[0].data,
finalshape,
box=20)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
data_arr_sr, regions, refine=True, tol=1)
#
# Combined offsets
# Offsets in numpy order, swaping
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = offsets_xy_t[:, ::-1]
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape, offsetsp = combine_shape(subpixshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
self.logger.info('Shape of resized array (crosscorr) is %s',
finalshape)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
data_arr_sr, regions = resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate2 img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r,
'interm2_%03d.fits' % idx)
hdulist = self.combine(data_arr_sr, data_hdul, finalshape,
offsetsp, refpix, use_errors)
self.save_intermediate_img(hdulist, 'result_initial2.fits')
except Exception as error:
self.logger.warning('Error during cross-correlation, %s',
error)
result = self.create_result(frame=hdulist, sky=sky_result)
self.logger.info('end of dither recipe')
return result
def run(self, rinput):
self.logger.info('starting reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=median)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# rectification and wavelength calibration (if a model has
# been provided)
if rinput.master_rectwv:
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image, rinput.master_rectwv)
# apply rectification and wavelength calibration
stare_image = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# save as JSON file in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(stare_image,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# image_wl_calibrated = True
else:
stare_image = reduced_image
self.logger.info('No wavelength calibration provided')
grism_value = hdr.get('GRISM', 'unknown')
self.logger.debug('GRISM is %s', grism_value)
if grism_value.lower() == 'open':
self.logger.debug('GRISM is %s, so this seems OK', grism_value)
# image_wl_calibrated = False
if rinput.master_sky:
# Sky subtraction after rectification
msky = rinput.master_sky.open()
# Check if images have the same size.
# if so, go ahead
if msky[0].data.shape != stare_image[0].data.shape:
self.logger.warning(
"sky and current image don't have the same shape")
else:
sky_corrector = proc.SkyCorrector(
msky[0].data,
datamodel=self.datamodel,
calibid=self.datamodel.get_imgid(msky))
stare_image = sky_corrector(stare_image)
else:
self.logger.info('No sky image provided')
# save results in results directory
self.logger.info('end reduction of stare spectra')
result = self.create_result(reduced_image=reduced_image,
stare=stare_image)
return result