def test_median(self):
'''Median combine: combination of integer arrays.'''
# Inputs
input1 = numpy.array([[1, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
input2 = numpy.array([[1, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
input3 = numpy.array([[7, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
input4 = numpy.array([[7, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
input5 = numpy.array([[7, 2, 1, 4], [1, 2, 0, 4], [44, 2, 2, 0]])
inputs = [input1, input2, input3, input4, input5]
out = median(inputs)
rres = input3
rvar = [16.954474097331239, 0.0, 1.2558869701726849,
0.0, 0.0, 0.0, 2.8257456828885403, 0.0,
384.61538461538458, 0.0, 1.2558869701726847,
5.0235478806907397]
# Checking
for cal, precal in zip(out[0].flat, rres.flat):
self.assertAlmostEqual(cal, precal)
for cal, precal in zip(out[1].flat, rvar):
self.assertAlmostEqual(cal, precal)
for cal, precal in zip(out[2].flat, itertools.repeat(5)):
self.assertEqual(cal, precal)
def hdu_creation(self, obresult, params=None):
if params is None:
params = {}
basicflow = self.__generate_flow(params, obresult.configuration)
cdata = []
headers = []
hdulist = []
try:
for frame in obresult.images:
hdulist = frame.open()
hdulist = basicflow(hdulist)
cdata.append(hdulist)
self.logger.info('stacking %d images using median', len(cdata))
data = combine.median([d[0].data for d in cdata], dtype='float32')
template_header = cdata[0][0].header
for header in hdulist:
if 'PRIMARY' in header.name:
headers.append(fits.PrimaryHDU(data[0], header=template_header))
else:
headers.append(header)
finally:
for hdulist in cdata:
hdulist.close()
return fits.HDUList(headers), data
def combine_frames(self, frames, extinction, out=None, step=0):
self.logger.debug('Step %d, opening sky-subtracted frames', step)
def fits_open(name):
"""Open FITS with memmap in readonly mode"""
return fits.open(name, mode='readonly', memmap=True)
frameslll = [fits_open(frame.lastname)
for frame in frames if frame.valid_target]
self.logger.debug('Step %d, opening mask frames', step)
mskslll = [fits_open(frame.resized_mask)
for frame in frames if frame.valid_target]
self.logger.debug('Step %d, combining %d frames', step, len(frameslll))
try:
extinc = [pow(10, -0.4 * frame.metadata['airmass'] * extinction)
for frame in frames if frame.valid_target]
data = [i['primary'].data for i in frameslll]
masks = [i['primary'].data for i in mskslll]
headers = [i['primary'].header for i in frameslll]
out = nacom.median(data, masks, scales=extinc, dtype='float32', out=out)
base_header = headers[0]
hdu = fits.PrimaryHDU(out[0], header=base_header)
hdu.header['history'] = "Combined %d images using '%s'" % (len(frameslll), 'median')
hdu.header['history'] = 'Combination time {}'.format(datetime.datetime.utcnow().isoformat())
for img in frameslll:
hdu.header['history'] = "Image {}".format(img[0].header['uuid'])
prevnum = base_header.get('NUM-NCOM', 1)
hdu.header['NUM-NCOM'] = prevnum * len(frameslll)
hdu.header['NUMRNAM'] = 'FullDitheredImagesRecipe'
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = 'FULL_DITHERED_IMAGE'
# Headers of last image
hdu.header['TSUTC2'] = headers[-1]['TSUTC2']
varhdu = fits.ImageHDU(out[1], name='VARIANCE')
num = fits.ImageHDU(out[2].astype('uint8'), name='MAP')
result = fits.HDUList([hdu, varhdu, num])
# saving the three extensions
fits.writeto('result_i%0d.fits' % step, out[0], overwrite=True)
fits.writeto('result_i%0d_var.fits' % step, out[1], overwrite=True)
fits.writeto('result_i%0d_npix.fits' % step, out[2], overwrite=True)
result.writeto('result_i%0d_full.fits' % step, overwrite=True)
return result
finally:
self.logger.debug('Step %d, closing sky-subtracted frames', step)
for f in frameslll:
f.close()
self.logger.debug('Step %d, closing mask frames', step)
for f in mskslll:
f.close()
def compute_sky_advanced(data, omasks):
from numina.array.combine import median
d = data[0]
m = omasks[0]
median_sky = numpy.median(d[m == 0])
result = numpy.zeros(data[0].shape)
result += median_sky
return result
result = median(data, omasks)
return result[0]
def compute_superflat(self, images_info, segmask=None, step=0):
self.logger.info("Step %d, SF: combining the frames without offsets",
step)
base_imgs = [img.resized_base for img in images_info]
with nfcom.manage_fits(base_imgs) as imgs:
data = []
masks = []
for img, img_info in zip(imgs, images_info):
self.logger.debug('Step %d, opening resized frame %s', step,
img_info.resized_base)
data.append(img['primary'].data[img_info.valid_region])
scales = [numpy.median(d) for d in data]
if segmask is not None:
masks = [segmask[frame.valid_region] for frame in images_info]
else:
for frame in images_info:
self.logger.debug('Step %d, opening resized mask %s', step,
frame.resized_mask)
hdulist = fits.open(frame.resized_mask,
memmap=True,
mode='readonly')
#filelist.append(hdulist)
masks.append(hdulist['primary'].data[frame.valid_region])
masks = None
self.logger.debug('Step %d, combining %d frames', step, len(data))
sf_data, _sf_var, sf_num = nacom.median(
data,
masks,
scales=scales,
dtype='float32',
#blank=1.0 / scales[0]
)
# Normalize, flat has mean = 1
sf_data[sf_data == 0] = 1e-5
sf_data /= sf_data.mean()
#sf_data[sf_data <= 0] = 1.0
# Auxiliary data
sfhdu = fits.PrimaryHDU(sf_data)
self.save_intermediate_img(sfhdu, name_skyflat('comb', step))
return sf_data
def compute_superflat(self, images_info, segmask=None, step=0):
self.logger.info("Step %d, SF: combining the frames without offsets", step)
base_imgs = [img.resized_base for img in images_info]
with nfcom.manage_fits(base_imgs) as imgs:
data = []
masks = []
for img, img_info in zip(imgs, images_info):
self.logger.debug('Step %d, opening resized frame %s',
step, img_info.resized_base)
data.append(img['primary'].data[img_info.valid_region])
scales = [numpy.median(d) for d in data]
if segmask is not None:
masks = [segmask[frame.valid_region] for frame in images_info]
else:
for frame in images_info:
self.logger.debug('Step %d, opening resized mask %s',
step, frame.resized_mask)
hdulist = fits.open(
frame.resized_mask, memmap=True, mode='readonly')
#filelist.append(hdulist)
masks.append(hdulist['primary'].data[frame.valid_region])
masks = None
self.logger.debug('Step %d, combining %d frames', step, len(data))
sf_data, _sf_var, sf_num = nacom.median(data, masks, scales=scales,
dtype='float32',
#blank=1.0 / scales[0]
)
# Normalize, flat has mean = 1
sf_data[sf_data == 0] = 1e-5
sf_data /= sf_data.mean()
#sf_data[sf_data <= 0] = 1.0
# Auxiliary data
sfhdu = fits.PrimaryHDU(sf_data)
self.save_intermediate_img(sfhdu, name_skyflat('comb', step))
return sf_data
def test_median2(self):
'''Median combine: combination an even number of integer arrays.'''
# Inputs
input1 = numpy.array([[1, 2, 3, -4]])
input2 = numpy.array([[1, 2, 6, 4]])
input3 = numpy.array([[7, 3, 8, -4]])
input4 = numpy.array([[7, 2, 3, 4]])
inputs = [input1, input2, input3, input4]
out = median(inputs)
rres = numpy.array([[4, 2, 4.5, 0.0]], dtype='float')
rvar = [18.838304552590266, 0.39246467817896391, 9.419152276295133,
33.490319204604916]
# Checking
for cal, precal in zip(out[0].flat, rres.flat):
self.assertAlmostEqual(cal, precal)
for cal, precal in zip(out[1].flat, rvar):
self.assertAlmostEqual(cal, precal)
for cal, precal in zip(out[2].flat, itertools.repeat(4)):
self.assertEqual(cal, precal)
def compute_advanced_sky_for_frame(self,
frame,
skyframes,
step=0,
save=True):
_logger.info('Correcting sky in frame %s', frame.lastname)
_logger.info('with sky computed from frames')
for i in skyframes:
_logger.info('%s', i.flat_corrected)
data = []
scales = []
masks = []
# handle the FITS file to close it finally
desc = []
try:
for i in skyframes:
filename = i.flat_corrected
hdulist = fits.open(filename, mode='readonly', memmap=True)
data.append(hdulist['primary'].data[i.valid_region])
desc.append(hdulist)
scales.append(numpy.median(data[-1]))
if i.objmask_data is not None:
masks.append(i.objmask_data)
_logger.debug('object mask is shared')
elif i.objmask is not None:
hdulistmask = fits.open(i.objmask,
mode='readonly',
memmap=True)
masks.append(hdulistmask['primary'].data)
desc.append(hdulistmask)
_logger.debug('object mask is particular')
else:
_logger.warn('no object mask for %s', filename)
_logger.debug('computing background with %d frames', len(data))
sky, _, num = median(data, masks, scales=scales)
finally:
# Closing all FITS files
for hdl in desc:
hdl.close()
if numpy.any(num == 0):
# We have pixels without
# sky background information
_logger.warn('pixels without sky information when correcting %s',
frame.flat_corrected)
binmask = num == 0
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
fixpix2(sky, binmask, out=sky, iterations=1)
name = name_skybackground(frame.baselabel, step)
fits.writeto(name, sky, clobber=True)
name = name_skybackgroundmask(frame.baselabel, step)
fits.writeto(name, binmask.astype('int16'), clobber=True)
dst = name_skysub_proc(frame.baselabel, step)
prev = frame.lastname
shutil.copyfile(prev, dst)
frame.lastname = dst
with fits.open(frame.lastname, mode='update') as hdulist:
data = hdulist['primary'].data
valid = data[frame.valid_region]
valid -= sky
def combine_frames(self, frames, extinction, out=None, step=0):
self.logger.debug('Step %d, opening sky-subtracted frames', step)
def fits_open(name):
"""Open FITS with memmap in readonly mode"""
return fits.open(name, mode='readonly', memmap=True)
frameslll = [
fits_open(frame.lastname) for frame in frames if frame.valid_target
]
self.logger.debug('Step %d, opening mask frames', step)
mskslll = [
fits_open(frame.resized_mask) for frame in frames
if frame.valid_target
]
self.logger.debug('Step %d, combining %d frames', step, len(frameslll))
try:
extinc = [
pow(10, -0.4 * frame.metadata['airmass'] * extinction)
for frame in frames if frame.valid_target
]
data = [i['primary'].data for i in frameslll]
masks = [i['primary'].data for i in mskslll]
headers = [i['primary'].header for i in frameslll]
out = nacom.median(data,
masks,
scales=extinc,
dtype='float32',
out=out)
base_header = headers[0]
hdu = fits.PrimaryHDU(out[0], header=base_header)
hdu.header['history'] = "Combined %d images using '%s'" % (
len(frameslll), 'median')
hdu.header['history'] = 'Combination time {}'.format(
datetime.datetime.utcnow().isoformat())
for img in frameslll:
hdu.header['history'] = "Image {}".format(
img[0].header['uuid'])
prevnum = base_header.get('NUM-NCOM', 1)
hdu.header['NUM-NCOM'] = prevnum * len(frameslll)
hdu.header['NUMRNAM'] = 'FullDitheredImagesRecipe'
hdu.header['UUID'] = str(uuid.uuid1())
hdu.header['OBSMODE'] = 'FULL_DITHERED_IMAGE'
# Headers of last image
hdu.header['TSUTC2'] = headers[-1]['TSUTC2']
varhdu = fits.ImageHDU(out[1], name='VARIANCE')
num = fits.ImageHDU(out[2].astype('uint8'), name='MAP')
result = fits.HDUList([hdu, varhdu, num])
# saving the three extensions
fits.writeto('result_i%0d.fits' % step, out[0], overwrite=True)
fits.writeto('result_i%0d_var.fits' % step, out[1], overwrite=True)
fits.writeto('result_i%0d_npix.fits' % step,
out[2],
overwrite=True)
result.writeto('result_i%0d_full.fits' % step, overwrite=True)
return result
finally:
self.logger.debug('Step %d, closing sky-subtracted frames', step)
for f in frameslll:
f.close()
self.logger.debug('Step %d, closing mask frames', step)
for f in mskslll:
f.close()
def run_single(self, rinput):
# FIXME: remove this, is deprecated
obresult = rinput.obresult
# just in case images are in result, instead of frames
if not obresult.frames:
frames = obresult.results
else:
frames = obresult.frames
img_info = []
data_hdul = []
for f in frames:
img = f.open()
data_hdul.append(img)
info = {}
info['tstamp'] = img[0].header['tstamp']
info['airmass'] = img[0].header['airmass']
img_info.append(info)
channels = FULL
use_errors = True
# Initial checks
baseimg = data_hdul[0]
has_num_ext = 'NUM' in baseimg
has_bpm_ext = 'BPM' in baseimg
baseshape = baseimg[0].shape
subpixshape = baseshape
base_header = baseimg[0].header
compute_sky = 'NUM-SK' not in base_header
compute_sky_advanced = False
self.logger.debug('base image is: %s',
self.datamodel.get_imgid(baseimg))
self.logger.debug('images have NUM extension: %s', has_num_ext)
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
self.logger.debug('compute sky is needed: %s', compute_sky)
if compute_sky:
self.logger.info('compute sky simple')
sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
self.save_intermediate_img(sky_result, 'sky_init.fits')
sky_result.writeto('sky_init.fits', overwrite=True)
sky_data = sky_result[0].data
self.logger.debug('sky image has shape %s', sky_data.shape)
self.logger.info('sky correction in individual images')
corrector = proc.SkyCorrector(
sky_data,
self.datamodel,
calibid=self.datamodel.get_imgid(sky_result))
# If we do not update keyword SKYADD
# there is no sky subtraction
for m in data_hdul:
m[0].header['SKYADD'] = True
# this is a little hackish
# sky corrected
data_hdul_s = [corrector(m) for m in data_hdul]
base_header = data_hdul_s[0][0].header
else:
sky_result = None
data_hdul_s = data_hdul
self.logger.info('Computing offsets from WCS information')
finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
data_hdul_s, baseshape, subpixshape)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
# Resizing target imgs
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if has_num_ext:
self.logger.debug('Using NUM extension')
masks = [
numpy.where(m['NUM'].data, 0, 1).astype('int16')
for m in data_hdul
]
elif has_bpm_ext:
self.logger.debug('Using BPM extension')
#
masks = [
numpy.where(m['BPM'].data, 1, 0).astype('int16')
for m in data_hdul
]
else:
self.logger.warning('BPM missing, use zeros instead')
false_mask = numpy.zeros(baseshape, dtype='int16')
masks = [false_mask for _ in data_arr_sr]
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp,
use_errors)
self.save_intermediate_img(hdulist, 'result_initial1.fits')
compute_cross_offsets = True
if compute_cross_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions_c = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions_c = self.compute_regions_from_objs(hdulist[0].data,
finalshape,
box=20)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
data_arr_sr, regions_c, refine=True, tol=1)
#
# Combined offsets
# Offsets in numpy order, swaping
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = offsets_xy_t[:, ::-1]
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape, offsetsp = narray.combine_shape(
subpixshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
self.logger.info('Shape of resized array (crosscorr) is %s',
finalshape)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate2 img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r,
'interm2_%03d.fits' % idx)
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks,
subpixshape,
offsetsp,
finalshape,
fill=1)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_initial2.fits')
except Exception as error:
self.logger.warning('Error during cross-correlation, %s',
error)
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
data_arr_0 = [(d[r] + s)
for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
data_arr_r = [d.copy() for d in data_arr_sr]
for inum in range(1, rinput.iterations + 1):
# superflat
sf_data = self.compute_superflat(data_arr_0, objmask, regions,
channels)
fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
# apply superflat
data_arr_rf = data_arr_r
for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
arr_f = arr / sf_data
#arr_f = arr
base[reg] = arr_f
# compute sky advanced
data_arr_sky = []
data_arr_rfs = []
self.logger.info('Step %d, SC: computing advanced sky', inum)
scale = rinput.sky_images_sep_time * 60
tstamps = numpy.array([info['tstamp'] for info in img_info])
for idx, hdu in enumerate(data_hdul):
diff1 = tstamps - tstamps[idx]
idxs1 = (diff1 > 0) & (diff1 < scale)
idxs2 = (diff1 < 0) & (diff1 > -scale)
l1, = numpy.nonzero(idxs1)
l2, = numpy.nonzero(idxs2)
limit1 = l1[-rinput.sky_images:]
limit2 = l2[:rinput.sky_images]
len_l1 = len(limit1)
len_l2 = len(limit2)
self.logger.info('For image %s, using %d-%d images)', idx,
len_l1, len_l2)
if len_l1 + len_l2 == 0:
self.logger.error('No sky image available for frame %d',
idx)
raise ValueError('No sky image')
skydata = []
skymasks = []
skyscales = []
my_region = regions[idx]
my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
for i in numpy.concatenate((limit1, limit2)):
region_s = regions[i]
data_s = data_arr_rf[i][region_s]
mask_s = objmask[region_s]
scale_s = numpy.median(data_s)
skydata.append(data_s)
skymasks.append(mask_s)
skyscales.append(scale_s)
self.logger.debug('computing background with %d frames',
len(skydata))
sky, _, num = nacom.median(skydata, skymasks, scales=skyscales)
# rescale
sky *= my_sky_scale
binmask = num == 0
if numpy.any(binmask):
# We have pixels without
# sky background information
self.logger.warn(
'pixels without sky information when correcting %d',
idx)
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
narray.fixpix2(sky, binmask, out=sky, iterations=1)
name = 'sky_%d_%03d.fits' % (inum, idx)
fits.writeto(name, sky, overwrite=True)
name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
data_arr_sky.append(sky)
arr = numpy.copy(data_arr_rf[idx])
arr[my_region] = data_arr_rf[idx][my_region] - sky
data_arr_rfs.append(arr)
# subtract sky advanced
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_rfs):
self.save_intermediate_array(
arr_r, 'interm_%d_%03d.fits' % (inum, idx))
hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul,
offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)
# For next step
catalog, objmask = self.create_object_catalog(hdulist[0].data,
border=50)
data_arr_0 = [
(d[r] + s)
for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)
]
data_arr_r = [d.copy() for d in data_arr_rfs]
result = self.create_result(frame=hdulist)
self.logger.info('end of dither recipe')
return result
def run_single(self, rinput):
# FIXME: remove this, is deprecated
obresult = rinput.obresult
# just in case images are in result, instead of frames
if not obresult.frames:
frames = obresult.results
else:
frames = obresult.frames
img_info = []
data_hdul = []
for f in frames:
img = f.open()
data_hdul.append(img)
info = {}
info['tstamp'] = img[0].header['tstamp']
info['airmass'] = img[0].header['airmass']
img_info.append(info)
channels = FULL
use_errors = True
# Initial checks
baseimg = data_hdul[0]
has_num_ext = 'NUM' in baseimg
has_bpm_ext = 'BPM' in baseimg
baseshape = baseimg[0].shape
subpixshape = baseshape
base_header = baseimg[0].header
compute_sky = 'NUM-SK' not in base_header
compute_sky_advanced = False
self.logger.debug('base image is: %s', self.datamodel.get_imgid(baseimg))
self.logger.debug('images have NUM extension: %s', has_num_ext)
self.logger.debug('images have BPM extension: %s', has_bpm_ext)
self.logger.debug('compute sky is needed: %s', compute_sky)
if compute_sky:
self.logger.info('compute sky simple')
sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
self.save_intermediate_img(sky_result, 'sky_init.fits')
sky_result.writeto('sky_init.fits', overwrite=True)
sky_data = sky_result[0].data
self.logger.debug('sky image has shape %s', sky_data.shape)
self.logger.info('sky correction in individual images')
corrector = proc.SkyCorrector(
sky_data,
self.datamodel,
calibid=self.datamodel.get_imgid(sky_result)
)
# If we do not update keyword SKYADD
# there is no sky subtraction
for m in data_hdul:
m[0].header['SKYADD'] = True
# this is a little hackish
# sky corrected
data_hdul_s = [corrector(m) for m in data_hdul]
base_header = data_hdul_s[0][0].header
else:
sky_result = None
data_hdul_s = data_hdul
self.logger.info('Computing offsets from WCS information')
finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
data_hdul_s,
baseshape,
subpixshape
)
self.logger.debug("Relative offsetsp %s", offsetsp)
self.logger.info('Shape of resized array is %s', finalshape)
# Resizing target imgs
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1
)
if has_num_ext:
self.logger.debug('Using NUM extension')
masks = [numpy.where(m['NUM'].data, 0, 1).astype('int16') for m in data_hdul]
elif has_bpm_ext:
self.logger.debug('Using BPM extension')
#
masks = [numpy.where(m['BPM'].data, 1, 0).astype('int16') for m in data_hdul]
else:
self.logger.warning('BPM missing, use zeros instead')
false_mask = numpy.zeros(baseshape, dtype='int16')
masks = [false_mask for _ in data_arr_sr]
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks, subpixshape, offsetsp, finalshape, fill=1)
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_initial1.fits')
compute_cross_offsets = True
if compute_cross_offsets:
self.logger.debug("Compute cross-correlation of images")
# regions_c = self.compute_regions(finalshape, box=200, corners=True)
# Regions frm bright objects
regions_c = self.compute_regions_from_objs(hdulist[0].data, finalshape, box=20)
try:
offsets_xy_c = self.compute_offset_xy_crosscor_regions(
data_arr_sr, regions_c, refine=True, tol=1
)
#
# Combined offsets
# Offsets in numpy order, swaping
offsets_xy_t = offset_xy0 - offsets_xy_c
offsets_fc = offsets_xy_t[:, ::-1]
offsets_fc_t = numpy.round(offsets_fc).astype('int')
self.logger.debug('Total offsets: %s', offsets_xy_t)
self.logger.info('Computing relative offsets from cross-corr')
finalshape, offsetsp = narray.combine_shape(subpixshape, offsets_fc_t)
#
self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
self.logger.info('Shape of resized array (crosscorr) is %s', finalshape)
# Resizing target imgs
self.logger.debug("Resize to final offsets")
data_arr_sr, regions = narray.resize_arrays(
[m[0].data for m in data_hdul_s],
subpixshape,
offsetsp,
finalshape,
fill=1
)
if self.intermediate_results:
self.logger.debug('save resized intermediate2 img')
for idx, arr_r in enumerate(data_arr_sr):
self.save_intermediate_array(arr_r, 'interm2_%03d.fits' % idx)
self.logger.debug('resize bad pixel masks')
mask_arr_r, _ = narray.resize_arrays(masks, subpixshape, offsetsp, finalshape, fill=1)
hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_initial2.fits')
except Exception as error:
self.logger.warning('Error during cross-correlation, %s', error)
catalog, objmask = self.create_object_catalog(hdulist[0].data, border=50)
data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
data_arr_0 = [(d[r] + s) for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
data_arr_r = [d.copy() for d in data_arr_sr]
for inum in range(1, rinput.iterations + 1):
# superflat
sf_data = self.compute_superflat(data_arr_0, objmask, regions, channels)
fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
# apply superflat
data_arr_rf = data_arr_r
for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
arr_f = arr / sf_data
#arr_f = arr
base[reg] = arr_f
# compute sky advanced
data_arr_sky = []
data_arr_rfs = []
self.logger.info('Step %d, SC: computing advanced sky', inum)
scale = rinput.sky_images_sep_time * 60
tstamps = numpy.array([info['tstamp'] for info in img_info])
for idx, hdu in enumerate(data_hdul):
diff1 = tstamps - tstamps[idx]
idxs1 = (diff1 > 0) & (diff1 < scale)
idxs2 = (diff1 < 0) & (diff1 > -scale)
l1, = numpy.nonzero(idxs1)
l2, = numpy.nonzero(idxs2)
limit1 = l1[-rinput.sky_images:]
limit2 = l2[:rinput.sky_images]
len_l1 =len(limit1)
len_l2 = len(limit2)
self.logger.info('For image %s, using %d-%d images)', idx,
len_l1, len_l2)
if len_l1 + len_l2 == 0:
self.logger.error(
'No sky image available for frame %d', idx)
raise ValueError('No sky image')
skydata = []
skymasks = []
skyscales = []
my_region = regions[idx]
my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
for i in numpy.concatenate((limit1, limit2)):
region_s = regions[i]
data_s = data_arr_rf[i][region_s]
mask_s = objmask[region_s]
scale_s = numpy.median(data_s)
skydata.append(data_s)
skymasks.append(mask_s)
skyscales.append(scale_s)
self.logger.debug('computing background with %d frames', len(skydata))
sky, _, num = nacom.median(skydata, skymasks, scales=skyscales)
# rescale
sky *= my_sky_scale
binmask = num == 0
if numpy.any(binmask):
# We have pixels without
# sky background information
self.logger.warn('pixels without sky information when correcting %d',
idx)
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
narray.fixpix2(sky, binmask, out=sky, iterations=1)
name = 'sky_%d_%03d.fits' % (inum, idx)
fits.writeto(name, sky, overwrite=True)
name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
data_arr_sky.append(sky)
arr = numpy.copy(data_arr_rf[idx])
arr[my_region] = data_arr_rf[idx][my_region] - sky
data_arr_rfs.append(arr)
# subtract sky advanced
if self.intermediate_results:
self.logger.debug('save resized intermediate img')
for idx, arr_r in enumerate(data_arr_rfs):
self.save_intermediate_array(arr_r, 'interm_%d_%03d.fits' % (inum, idx))
hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul, offsetsp, use_errors)
self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)
# For next step
catalog, objmask = self.create_object_catalog(hdulist[0].data, border=50)
data_arr_0 = [(d[r] + s) for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)]
data_arr_r = [d.copy() for d in data_arr_rfs]
result = self.create_result(frame=hdulist)
self.logger.info('end of dither recipe')
return result
def compute_advanced_sky_for_frame(self, frame, skyframes,
step=0, save=True):
self.logger.info('Correcting sky in frame %s', frame.lastname)
self.logger.info('with sky computed from frames')
for i in skyframes:
self.logger.info('%s', i.flat_corrected)
data = []
scales = []
masks = []
# handle the FITS file to close it finally
desc = []
try:
for i in skyframes:
filename = i.flat_corrected
hdulist = fits.open(filename, mode='readonly', memmap=True)
data.append(hdulist['primary'].data[i.valid_region])
desc.append(hdulist)
#scales.append(numpy.median(data[-1]))
if i.objmask_data is not None:
masks.append(i.objmask_data)
self.logger.debug('object mask is shared')
elif i.objmask is not None:
hdulistmask = fits.open(
i.objmask, mode='readonly', memmap=True)
masks.append(hdulistmask['primary'].data)
desc.append(hdulistmask)
self.logger.debug('object mask is particular')
else:
self.logger.warn('no object mask for %s', filename)
self.logger.debug('computing background with %d frames', len(data))
sky, _, num = nacom.median(data, masks)#, scales=scales)
finally:
# Closing all FITS files
for hdl in desc:
hdl.close()
if numpy.any(num == 0):
# We have pixels without
# sky background information
self.logger.warn('pixels without sky information when correcting %s',
frame.flat_corrected)
binmask = num == 0
# FIXME: during development, this is faster
# sky[binmask] = sky[num != 0].mean()
# To continue we interpolate over the patches
narray.fixpix2(sky, binmask, out=sky, iterations=1)
name = name_skybackgroundmask(frame.label, step)
fits.writeto(name, binmask.astype('int16'), overwrite=True)
name_sky = name_skybackground(frame.label, step)
fits.writeto(name_sky, sky, overwrite=True)
dst = name_skysub_proc(frame.label, step)
prev = frame.lastname
shutil.copyfile(prev, dst)
frame.lastname = dst
with fits.open(frame.lastname, mode='update') as hdulist:
data = hdulist['primary'].data
valid = data[frame.valid_region]
valid -= sky
