Пример #1
0
    def test_median(self):
        '''Median combine: combination of integer arrays.'''
        # Inputs
        input1 = numpy.array([[1, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
        input2 = numpy.array([[1, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
        input3 = numpy.array([[7, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
        input4 = numpy.array([[7, 2, 3, 4], [1, 2, 3, 4], [9, 2, 0, 4]])
        input5 = numpy.array([[7, 2, 1, 4], [1, 2, 0, 4], [44, 2, 2, 0]])
        inputs = [input1, input2, input3, input4, input5]

        out = median(inputs)

        rres = input3
        rvar = [16.954474097331239, 0.0, 1.2558869701726849,
                0.0, 0.0, 0.0, 2.8257456828885403, 0.0,
                384.61538461538458, 0.0, 1.2558869701726847,
                5.0235478806907397]

        # Checking
        for cal, precal in zip(out[0].flat, rres.flat):
            self.assertAlmostEqual(cal, precal)
        for cal, precal in zip(out[1].flat, rvar):
            self.assertAlmostEqual(cal, precal)
        for cal, precal in zip(out[2].flat, itertools.repeat(5)):
            self.assertEqual(cal, precal)
Пример #2
0
    def hdu_creation(self, obresult, params=None):

        if params is None:
            params = {}

        basicflow = self.__generate_flow(params, obresult.configuration)
        cdata = []
        headers = []
        hdulist = []
        try:
            for frame in obresult.images:
                hdulist = frame.open()
                hdulist = basicflow(hdulist)
                cdata.append(hdulist)

            self.logger.info('stacking %d images using median', len(cdata))

            data = combine.median([d[0].data for d in cdata], dtype='float32')
            template_header = cdata[0][0].header

            for header in hdulist:
                if 'PRIMARY' in header.name:
                    headers.append(fits.PrimaryHDU(data[0], header=template_header))
                else:
                    headers.append(header)

        finally:
            for hdulist in cdata:
                hdulist.close()

        return fits.HDUList(headers), data
Пример #3
0
    def combine_frames(self, frames, extinction, out=None, step=0):
        self.logger.debug('Step %d, opening sky-subtracted frames', step)

        def fits_open(name):
            """Open FITS with memmap in readonly mode"""
            return fits.open(name, mode='readonly', memmap=True)

        frameslll = [fits_open(frame.lastname)
                     for frame in frames if frame.valid_target]
        self.logger.debug('Step %d, opening mask frames', step)
        mskslll = [fits_open(frame.resized_mask)
                   for frame in frames if frame.valid_target]

        self.logger.debug('Step %d, combining %d frames', step, len(frameslll))
        try:
            extinc = [pow(10, -0.4 * frame.metadata['airmass'] * extinction)
                      for frame in frames if frame.valid_target]
            data = [i['primary'].data for i in frameslll]
            masks = [i['primary'].data for i in mskslll]
            headers = [i['primary'].header for i in frameslll]

            out = nacom.median(data, masks, scales=extinc, dtype='float32', out=out)

            base_header = headers[0]
            hdu = fits.PrimaryHDU(out[0], header=base_header)
            hdu.header['history'] = "Combined %d images using '%s'" % (len(frameslll), 'median')
            hdu.header['history'] = 'Combination time {}'.format(datetime.datetime.utcnow().isoformat())
            for img in frameslll:
                hdu.header['history'] = "Image {}".format(img[0].header['uuid'])
            prevnum = base_header.get('NUM-NCOM', 1)
            hdu.header['NUM-NCOM'] = prevnum * len(frameslll)
            hdu.header['NUMRNAM'] = 'FullDitheredImagesRecipe'
            hdu.header['UUID'] = str(uuid.uuid1())
            hdu.header['OBSMODE'] = 'FULL_DITHERED_IMAGE'
            # Headers of last image
            hdu.header['TSUTC2'] = headers[-1]['TSUTC2']

            varhdu = fits.ImageHDU(out[1], name='VARIANCE')
            num = fits.ImageHDU(out[2].astype('uint8'), name='MAP')

            result = fits.HDUList([hdu, varhdu, num])
            # saving the three extensions
            fits.writeto('result_i%0d.fits' % step, out[0], overwrite=True)
            fits.writeto('result_i%0d_var.fits' % step, out[1], overwrite=True)
            fits.writeto('result_i%0d_npix.fits' % step, out[2], overwrite=True)

            result.writeto('result_i%0d_full.fits' % step, overwrite=True)
            return result

        finally:
            self.logger.debug('Step %d, closing sky-subtracted frames', step)
            for f in frameslll:
                f.close()
            self.logger.debug('Step %d, closing mask frames', step)
            for f in mskslll:
                f.close()
Пример #4
0
def compute_sky_advanced(data, omasks):
    from numina.array.combine import median
    d = data[0]
    m = omasks[0]
    median_sky = numpy.median(d[m == 0])
    result = numpy.zeros(data[0].shape)
    result += median_sky
    return result

    result = median(data, omasks)
    return result[0]
Пример #5
0
    def compute_superflat(self, images_info, segmask=None, step=0):

        self.logger.info("Step %d, SF: combining the frames without offsets",
                         step)

        base_imgs = [img.resized_base for img in images_info]
        with nfcom.manage_fits(base_imgs) as imgs:

            data = []
            masks = []

            for img, img_info in zip(imgs, images_info):
                self.logger.debug('Step %d, opening resized frame %s', step,
                                  img_info.resized_base)
                data.append(img['primary'].data[img_info.valid_region])

            scales = [numpy.median(d) for d in data]

            if segmask is not None:
                masks = [segmask[frame.valid_region] for frame in images_info]
            else:
                for frame in images_info:
                    self.logger.debug('Step %d, opening resized mask %s', step,
                                      frame.resized_mask)
                    hdulist = fits.open(frame.resized_mask,
                                        memmap=True,
                                        mode='readonly')
                    #filelist.append(hdulist)
                    masks.append(hdulist['primary'].data[frame.valid_region])
                masks = None

            self.logger.debug('Step %d, combining %d frames', step, len(data))
            sf_data, _sf_var, sf_num = nacom.median(
                data,
                masks,
                scales=scales,
                dtype='float32',
                #blank=1.0 / scales[0]
            )

        # Normalize, flat has mean = 1
        sf_data[sf_data == 0] = 1e-5
        sf_data /= sf_data.mean()
        #sf_data[sf_data <= 0] = 1.0

        # Auxiliary data
        sfhdu = fits.PrimaryHDU(sf_data)
        self.save_intermediate_img(sfhdu, name_skyflat('comb', step))
        return sf_data
Пример #6
0
    def compute_superflat(self, images_info, segmask=None, step=0):

        self.logger.info("Step %d, SF: combining the frames without offsets", step)

        base_imgs = [img.resized_base for img in images_info]
        with nfcom.manage_fits(base_imgs) as imgs:

            data = []
            masks = []

            for img, img_info in zip(imgs, images_info):
                self.logger.debug('Step %d, opening resized frame %s',
                              step, img_info.resized_base)
                data.append(img['primary'].data[img_info.valid_region])

            scales = [numpy.median(d) for d in data]

            if segmask is not None:
                masks = [segmask[frame.valid_region] for frame in images_info]
            else:
                for frame in images_info:
                    self.logger.debug('Step %d, opening resized mask %s',
                                      step, frame.resized_mask)
                    hdulist = fits.open(
                         frame.resized_mask, memmap=True, mode='readonly')
                    #filelist.append(hdulist)
                    masks.append(hdulist['primary'].data[frame.valid_region])
                masks = None

            self.logger.debug('Step %d, combining %d frames', step, len(data))
            sf_data, _sf_var, sf_num = nacom.median(data, masks, scales=scales,
                                                 dtype='float32',
                                                 #blank=1.0 / scales[0]
                                                 )

        # Normalize, flat has mean = 1
        sf_data[sf_data == 0] = 1e-5
        sf_data /= sf_data.mean()
        #sf_data[sf_data <= 0] = 1.0

        # Auxiliary data
        sfhdu = fits.PrimaryHDU(sf_data)
        self.save_intermediate_img(sfhdu, name_skyflat('comb', step))
        return sf_data
Пример #7
0
    def test_median2(self):
        '''Median combine: combination an even number of integer arrays.'''
        # Inputs
        input1 = numpy.array([[1, 2, 3, -4]])
        input2 = numpy.array([[1, 2, 6, 4]])
        input3 = numpy.array([[7, 3, 8, -4]])
        input4 = numpy.array([[7, 2, 3, 4]])
        inputs = [input1, input2, input3, input4]

        out = median(inputs)

        rres = numpy.array([[4, 2, 4.5, 0.0]], dtype='float')
        rvar = [18.838304552590266, 0.39246467817896391, 9.419152276295133,
                33.490319204604916]

        # Checking
        for cal, precal in zip(out[0].flat, rres.flat):
            self.assertAlmostEqual(cal, precal)
        for cal, precal in zip(out[1].flat, rvar):
            self.assertAlmostEqual(cal, precal)
        for cal, precal in zip(out[2].flat, itertools.repeat(4)):
            self.assertEqual(cal, precal)
Пример #8
0
    def compute_advanced_sky_for_frame(self,
                                       frame,
                                       skyframes,
                                       step=0,
                                       save=True):
        _logger.info('Correcting sky in frame %s', frame.lastname)
        _logger.info('with sky computed from frames')
        for i in skyframes:
            _logger.info('%s', i.flat_corrected)

        data = []
        scales = []
        masks = []
        # handle the FITS file to close it finally
        desc = []
        try:
            for i in skyframes:
                filename = i.flat_corrected
                hdulist = fits.open(filename, mode='readonly', memmap=True)

                data.append(hdulist['primary'].data[i.valid_region])
                desc.append(hdulist)
                scales.append(numpy.median(data[-1]))
                if i.objmask_data is not None:
                    masks.append(i.objmask_data)
                    _logger.debug('object mask is shared')
                elif i.objmask is not None:
                    hdulistmask = fits.open(i.objmask,
                                            mode='readonly',
                                            memmap=True)
                    masks.append(hdulistmask['primary'].data)
                    desc.append(hdulistmask)
                    _logger.debug('object mask is particular')
                else:
                    _logger.warn('no object mask for %s', filename)

            _logger.debug('computing background with %d frames', len(data))
            sky, _, num = median(data, masks, scales=scales)

        finally:
            # Closing all FITS files
            for hdl in desc:
                hdl.close()

        if numpy.any(num == 0):
            # We have pixels without
            # sky background information
            _logger.warn('pixels without sky information when correcting %s',
                         frame.flat_corrected)
            binmask = num == 0
            # FIXME: during development, this is faster
            # sky[binmask] = sky[num != 0].mean()

            # To continue we interpolate over the patches
            fixpix2(sky, binmask, out=sky, iterations=1)
            name = name_skybackground(frame.baselabel, step)
            fits.writeto(name, sky, clobber=True)
            name = name_skybackgroundmask(frame.baselabel, step)
            fits.writeto(name, binmask.astype('int16'), clobber=True)

        dst = name_skysub_proc(frame.baselabel, step)
        prev = frame.lastname
        shutil.copyfile(prev, dst)
        frame.lastname = dst

        with fits.open(frame.lastname, mode='update') as hdulist:
            data = hdulist['primary'].data
            valid = data[frame.valid_region]
            valid -= sky
Пример #9
0
    def combine_frames(self, frames, extinction, out=None, step=0):
        self.logger.debug('Step %d, opening sky-subtracted frames', step)

        def fits_open(name):
            """Open FITS with memmap in readonly mode"""
            return fits.open(name, mode='readonly', memmap=True)

        frameslll = [
            fits_open(frame.lastname) for frame in frames if frame.valid_target
        ]
        self.logger.debug('Step %d, opening mask frames', step)
        mskslll = [
            fits_open(frame.resized_mask) for frame in frames
            if frame.valid_target
        ]

        self.logger.debug('Step %d, combining %d frames', step, len(frameslll))
        try:
            extinc = [
                pow(10, -0.4 * frame.metadata['airmass'] * extinction)
                for frame in frames if frame.valid_target
            ]
            data = [i['primary'].data for i in frameslll]
            masks = [i['primary'].data for i in mskslll]
            headers = [i['primary'].header for i in frameslll]

            out = nacom.median(data,
                               masks,
                               scales=extinc,
                               dtype='float32',
                               out=out)

            base_header = headers[0]
            hdu = fits.PrimaryHDU(out[0], header=base_header)
            hdu.header['history'] = "Combined %d images using '%s'" % (
                len(frameslll), 'median')
            hdu.header['history'] = 'Combination time {}'.format(
                datetime.datetime.utcnow().isoformat())
            for img in frameslll:
                hdu.header['history'] = "Image {}".format(
                    img[0].header['uuid'])
            prevnum = base_header.get('NUM-NCOM', 1)
            hdu.header['NUM-NCOM'] = prevnum * len(frameslll)
            hdu.header['NUMRNAM'] = 'FullDitheredImagesRecipe'
            hdu.header['UUID'] = str(uuid.uuid1())
            hdu.header['OBSMODE'] = 'FULL_DITHERED_IMAGE'
            # Headers of last image
            hdu.header['TSUTC2'] = headers[-1]['TSUTC2']

            varhdu = fits.ImageHDU(out[1], name='VARIANCE')
            num = fits.ImageHDU(out[2].astype('uint8'), name='MAP')

            result = fits.HDUList([hdu, varhdu, num])
            # saving the three extensions
            fits.writeto('result_i%0d.fits' % step, out[0], overwrite=True)
            fits.writeto('result_i%0d_var.fits' % step, out[1], overwrite=True)
            fits.writeto('result_i%0d_npix.fits' % step,
                         out[2],
                         overwrite=True)

            result.writeto('result_i%0d_full.fits' % step, overwrite=True)
            return result

        finally:
            self.logger.debug('Step %d, closing sky-subtracted frames', step)
            for f in frameslll:
                f.close()
            self.logger.debug('Step %d, closing mask frames', step)
            for f in mskslll:
                f.close()
Пример #10
0
    def run_single(self, rinput):

        # FIXME: remove this, is deprecated

        obresult = rinput.obresult

        # just in case images are in result, instead of frames
        if not obresult.frames:
            frames = obresult.results
        else:
            frames = obresult.frames

        img_info = []
        data_hdul = []
        for f in frames:
            img = f.open()
            data_hdul.append(img)
            info = {}
            info['tstamp'] = img[0].header['tstamp']
            info['airmass'] = img[0].header['airmass']
            img_info.append(info)

        channels = FULL

        use_errors = True
        # Initial checks
        baseimg = data_hdul[0]
        has_num_ext = 'NUM' in baseimg
        has_bpm_ext = 'BPM' in baseimg
        baseshape = baseimg[0].shape
        subpixshape = baseshape
        base_header = baseimg[0].header
        compute_sky = 'NUM-SK' not in base_header
        compute_sky_advanced = False

        self.logger.debug('base image is: %s',
                          self.datamodel.get_imgid(baseimg))
        self.logger.debug('images have NUM extension: %s', has_num_ext)
        self.logger.debug('images have BPM extension: %s', has_bpm_ext)
        self.logger.debug('compute sky is needed: %s', compute_sky)

        if compute_sky:
            self.logger.info('compute sky simple')
            sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
            self.save_intermediate_img(sky_result, 'sky_init.fits')
            sky_result.writeto('sky_init.fits', overwrite=True)
            sky_data = sky_result[0].data
            self.logger.debug('sky image has shape %s', sky_data.shape)

            self.logger.info('sky correction in individual images')
            corrector = proc.SkyCorrector(
                sky_data,
                self.datamodel,
                calibid=self.datamodel.get_imgid(sky_result))
            # If we do not update keyword SKYADD
            # there is no sky subtraction
            for m in data_hdul:
                m[0].header['SKYADD'] = True
            # this is a little hackish
            # sky corrected
            data_hdul_s = [corrector(m) for m in data_hdul]
            base_header = data_hdul_s[0][0].header
        else:
            sky_result = None
            data_hdul_s = data_hdul

        self.logger.info('Computing offsets from WCS information')

        finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
            data_hdul_s, baseshape, subpixshape)

        self.logger.debug("Relative offsetsp %s", offsetsp)
        self.logger.info('Shape of resized array is %s', finalshape)

        # Resizing target imgs
        data_arr_sr, regions = narray.resize_arrays(
            [m[0].data for m in data_hdul_s],
            subpixshape,
            offsetsp,
            finalshape,
            fill=1)

        if has_num_ext:
            self.logger.debug('Using NUM extension')
            masks = [
                numpy.where(m['NUM'].data, 0, 1).astype('int16')
                for m in data_hdul
            ]
        elif has_bpm_ext:
            self.logger.debug('Using BPM extension')
            #
            masks = [
                numpy.where(m['BPM'].data, 1, 0).astype('int16')
                for m in data_hdul
            ]
        else:
            self.logger.warning('BPM missing, use zeros instead')
            false_mask = numpy.zeros(baseshape, dtype='int16')
            masks = [false_mask for _ in data_arr_sr]

        self.logger.debug('resize bad pixel masks')
        mask_arr_r, _ = narray.resize_arrays(masks,
                                             subpixshape,
                                             offsetsp,
                                             finalshape,
                                             fill=1)

        if self.intermediate_results:
            self.logger.debug('save resized intermediate img')
            for idx, arr_r in enumerate(data_arr_sr):
                self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)

        hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp,
                                use_errors)

        self.save_intermediate_img(hdulist, 'result_initial1.fits')

        compute_cross_offsets = True
        if compute_cross_offsets:

            self.logger.debug("Compute cross-correlation of images")
            # regions_c = self.compute_regions(finalshape, box=200, corners=True)

            # Regions frm bright objects
            regions_c = self.compute_regions_from_objs(hdulist[0].data,
                                                       finalshape,
                                                       box=20)

            try:

                offsets_xy_c = self.compute_offset_xy_crosscor_regions(
                    data_arr_sr, regions_c, refine=True, tol=1)
                #
                # Combined offsets
                # Offsets in numpy order, swaping
                offsets_xy_t = offset_xy0 - offsets_xy_c
                offsets_fc = offsets_xy_t[:, ::-1]
                offsets_fc_t = numpy.round(offsets_fc).astype('int')
                self.logger.debug('Total offsets: %s', offsets_xy_t)
                self.logger.info('Computing relative offsets from cross-corr')
                finalshape, offsetsp = narray.combine_shape(
                    subpixshape, offsets_fc_t)
                #
                self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
                self.logger.info('Shape of resized array (crosscorr) is %s',
                                 finalshape)

                # Resizing target imgs
                self.logger.debug("Resize to final offsets")
                data_arr_sr, regions = narray.resize_arrays(
                    [m[0].data for m in data_hdul_s],
                    subpixshape,
                    offsetsp,
                    finalshape,
                    fill=1)

                if self.intermediate_results:
                    self.logger.debug('save resized intermediate2 img')
                    for idx, arr_r in enumerate(data_arr_sr):
                        self.save_intermediate_array(arr_r,
                                                     'interm2_%03d.fits' % idx)

                self.logger.debug('resize bad pixel masks')
                mask_arr_r, _ = narray.resize_arrays(masks,
                                                     subpixshape,
                                                     offsetsp,
                                                     finalshape,
                                                     fill=1)

                hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul,
                                        offsetsp, use_errors)

                self.save_intermediate_img(hdulist, 'result_initial2.fits')
            except Exception as error:
                self.logger.warning('Error during cross-correlation, %s',
                                    error)

        catalog, objmask = self.create_object_catalog(hdulist[0].data,
                                                      border=50)

        data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
        data_arr_0 = [(d[r] + s)
                      for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
        data_arr_r = [d.copy() for d in data_arr_sr]

        for inum in range(1, rinput.iterations + 1):
            # superflat
            sf_data = self.compute_superflat(data_arr_0, objmask, regions,
                                             channels)
            fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
            # apply superflat
            data_arr_rf = data_arr_r
            for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
                arr_f = arr / sf_data
                #arr_f = arr
                base[reg] = arr_f

            # compute sky advanced
            data_arr_sky = []
            data_arr_rfs = []
            self.logger.info('Step %d, SC: computing advanced sky', inum)
            scale = rinput.sky_images_sep_time * 60
            tstamps = numpy.array([info['tstamp'] for info in img_info])
            for idx, hdu in enumerate(data_hdul):
                diff1 = tstamps - tstamps[idx]
                idxs1 = (diff1 > 0) & (diff1 < scale)
                idxs2 = (diff1 < 0) & (diff1 > -scale)
                l1, = numpy.nonzero(idxs1)
                l2, = numpy.nonzero(idxs2)
                limit1 = l1[-rinput.sky_images:]
                limit2 = l2[:rinput.sky_images]
                len_l1 = len(limit1)
                len_l2 = len(limit2)
                self.logger.info('For image %s, using %d-%d images)', idx,
                                 len_l1, len_l2)
                if len_l1 + len_l2 == 0:
                    self.logger.error('No sky image available for frame %d',
                                      idx)
                    raise ValueError('No sky image')
                skydata = []
                skymasks = []
                skyscales = []
                my_region = regions[idx]
                my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
                for i in numpy.concatenate((limit1, limit2)):
                    region_s = regions[i]
                    data_s = data_arr_rf[i][region_s]
                    mask_s = objmask[region_s]
                    scale_s = numpy.median(data_s)
                    skydata.append(data_s)
                    skymasks.append(mask_s)
                    skyscales.append(scale_s)
                self.logger.debug('computing background with %d frames',
                                  len(skydata))
                sky, _, num = nacom.median(skydata, skymasks, scales=skyscales)
                # rescale
                sky *= my_sky_scale

                binmask = num == 0

                if numpy.any(binmask):
                    # We have pixels without
                    # sky background information
                    self.logger.warn(
                        'pixels without sky information when correcting %d',
                        idx)

                    # FIXME: during development, this is faster
                    # sky[binmask] = sky[num != 0].mean()
                    # To continue we interpolate over the patches
                    narray.fixpix2(sky, binmask, out=sky, iterations=1)

                name = 'sky_%d_%03d.fits' % (inum, idx)
                fits.writeto(name, sky, overwrite=True)
                name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
                fits.writeto(name, binmask.astype('int16'), overwrite=True)

                data_arr_sky.append(sky)
                arr = numpy.copy(data_arr_rf[idx])
                arr[my_region] = data_arr_rf[idx][my_region] - sky
                data_arr_rfs.append(arr)
                # subtract sky advanced

            if self.intermediate_results:
                self.logger.debug('save resized intermediate img')
                for idx, arr_r in enumerate(data_arr_rfs):
                    self.save_intermediate_array(
                        arr_r, 'interm_%d_%03d.fits' % (inum, idx))

            hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul,
                                    offsetsp, use_errors)

            self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)

            # For next step
            catalog, objmask = self.create_object_catalog(hdulist[0].data,
                                                          border=50)

            data_arr_0 = [
                (d[r] + s)
                for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)
            ]
            data_arr_r = [d.copy() for d in data_arr_rfs]

        result = self.create_result(frame=hdulist)
        self.logger.info('end of dither recipe')
        return result
Пример #11
0
    def run_single(self, rinput):

        # FIXME: remove this, is deprecated

        obresult = rinput.obresult

        # just in case images are in result, instead of frames
        if not obresult.frames:
            frames = obresult.results
        else:
            frames = obresult.frames

        img_info = []
        data_hdul = []
        for f in frames:
            img = f.open()
            data_hdul.append(img)
            info = {}
            info['tstamp'] = img[0].header['tstamp']
            info['airmass'] = img[0].header['airmass']
            img_info.append(info)

        channels = FULL

        use_errors = True
        # Initial checks
        baseimg = data_hdul[0]
        has_num_ext = 'NUM' in baseimg
        has_bpm_ext = 'BPM' in baseimg
        baseshape = baseimg[0].shape
        subpixshape = baseshape
        base_header = baseimg[0].header
        compute_sky = 'NUM-SK' not in base_header
        compute_sky_advanced = False

        self.logger.debug('base image is: %s', self.datamodel.get_imgid(baseimg))
        self.logger.debug('images have NUM extension: %s', has_num_ext)
        self.logger.debug('images have BPM extension: %s', has_bpm_ext)
        self.logger.debug('compute sky is needed: %s', compute_sky)

        if compute_sky:
            self.logger.info('compute sky simple')
            sky_result = self.compute_sky_simple(data_hdul, use_errors=False)
            self.save_intermediate_img(sky_result, 'sky_init.fits')
            sky_result.writeto('sky_init.fits', overwrite=True)
            sky_data = sky_result[0].data
            self.logger.debug('sky image has shape %s', sky_data.shape)

            self.logger.info('sky correction in individual images')
            corrector = proc.SkyCorrector(
                sky_data,
                self.datamodel,
                calibid=self.datamodel.get_imgid(sky_result)
            )
            # If we do not update keyword SKYADD
            # there is no sky subtraction
            for m in data_hdul:
                m[0].header['SKYADD'] = True
            # this is a little hackish
            # sky corrected
            data_hdul_s = [corrector(m) for m in data_hdul]
            base_header = data_hdul_s[0][0].header
        else:
            sky_result = None
            data_hdul_s = data_hdul

        self.logger.info('Computing offsets from WCS information')

        finalshape, offsetsp, refpix, offset_xy0 = self.compute_offset_wcs_imgs(
            data_hdul_s,
            baseshape,
            subpixshape
        )

        self.logger.debug("Relative offsetsp %s", offsetsp)
        self.logger.info('Shape of resized array is %s', finalshape)

        # Resizing target imgs
        data_arr_sr, regions = narray.resize_arrays(
            [m[0].data for m in data_hdul_s],
            subpixshape,
            offsetsp,
            finalshape,
            fill=1
        )

        if has_num_ext:
            self.logger.debug('Using NUM extension')
            masks = [numpy.where(m['NUM'].data, 0, 1).astype('int16') for m in data_hdul]
        elif has_bpm_ext:
            self.logger.debug('Using BPM extension')
            #
            masks = [numpy.where(m['BPM'].data, 1, 0).astype('int16') for m in data_hdul]
        else:
            self.logger.warning('BPM missing, use zeros instead')
            false_mask = numpy.zeros(baseshape, dtype='int16')
            masks = [false_mask for _ in data_arr_sr]

        self.logger.debug('resize bad pixel masks')
        mask_arr_r, _ = narray.resize_arrays(masks, subpixshape, offsetsp, finalshape, fill=1)

        if self.intermediate_results:
            self.logger.debug('save resized intermediate img')
            for idx, arr_r in enumerate(data_arr_sr):
                self.save_intermediate_array(arr_r, 'interm1_%03d.fits' % idx)

        hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp, use_errors)

        self.save_intermediate_img(hdulist, 'result_initial1.fits')

        compute_cross_offsets = True
        if compute_cross_offsets:

            self.logger.debug("Compute cross-correlation of images")
            # regions_c = self.compute_regions(finalshape, box=200, corners=True)

            # Regions frm bright objects
            regions_c = self.compute_regions_from_objs(hdulist[0].data, finalshape, box=20)

            try:

                offsets_xy_c = self.compute_offset_xy_crosscor_regions(
                    data_arr_sr, regions_c, refine=True, tol=1
                )
                #
                # Combined offsets
                # Offsets in numpy order, swaping
                offsets_xy_t = offset_xy0 - offsets_xy_c
                offsets_fc = offsets_xy_t[:, ::-1]
                offsets_fc_t = numpy.round(offsets_fc).astype('int')
                self.logger.debug('Total offsets: %s', offsets_xy_t)
                self.logger.info('Computing relative offsets from cross-corr')
                finalshape, offsetsp = narray.combine_shape(subpixshape, offsets_fc_t)
                #
                self.logger.debug("Relative offsetsp (crosscorr) %s", offsetsp)
                self.logger.info('Shape of resized array (crosscorr) is %s', finalshape)

                # Resizing target imgs
                self.logger.debug("Resize to final offsets")
                data_arr_sr, regions = narray.resize_arrays(
                    [m[0].data for m in data_hdul_s],
                    subpixshape,
                    offsetsp,
                    finalshape,
                    fill=1
                )

                if self.intermediate_results:
                    self.logger.debug('save resized intermediate2 img')
                    for idx, arr_r in enumerate(data_arr_sr):
                        self.save_intermediate_array(arr_r, 'interm2_%03d.fits' % idx)

                self.logger.debug('resize bad pixel masks')
                mask_arr_r, _ = narray.resize_arrays(masks, subpixshape, offsetsp, finalshape, fill=1)

                hdulist = self.combine2(data_arr_sr, mask_arr_r, data_hdul, offsetsp, use_errors)

                self.save_intermediate_img(hdulist, 'result_initial2.fits')
            except Exception as error:
                self.logger.warning('Error during cross-correlation, %s', error)


        catalog, objmask = self.create_object_catalog(hdulist[0].data, border=50)

        data_arr_sky = [sky_result[0].data for _ in data_arr_sr]
        data_arr_0 = [(d[r] + s) for d, r, s in zip(data_arr_sr, regions, data_arr_sky)]
        data_arr_r = [d.copy() for d in data_arr_sr]

        for inum in range(1, rinput.iterations + 1):
            # superflat
            sf_data = self.compute_superflat(data_arr_0, objmask, regions, channels)
            fits.writeto('superflat_%d.fits' % inum, sf_data, overwrite=True)
            # apply superflat
            data_arr_rf = data_arr_r
            for base, arr, reg in zip(data_arr_rf, data_arr_0, regions):
                arr_f = arr / sf_data
                #arr_f = arr
                base[reg] = arr_f

            # compute sky advanced
            data_arr_sky = []
            data_arr_rfs = []
            self.logger.info('Step %d, SC: computing advanced sky', inum)
            scale = rinput.sky_images_sep_time * 60
            tstamps = numpy.array([info['tstamp'] for info in img_info])
            for idx, hdu in enumerate(data_hdul):
                diff1 = tstamps - tstamps[idx]
                idxs1 = (diff1 > 0) & (diff1 < scale)
                idxs2 = (diff1 < 0) & (diff1 > -scale)
                l1, = numpy.nonzero(idxs1)
                l2, = numpy.nonzero(idxs2)
                limit1 = l1[-rinput.sky_images:]
                limit2 = l2[:rinput.sky_images]
                len_l1 =len(limit1)
                len_l2 = len(limit2)
                self.logger.info('For image %s, using %d-%d images)', idx,
                                 len_l1, len_l2)
                if len_l1 + len_l2 == 0:
                    self.logger.error(
                        'No sky image available for frame %d', idx)
                    raise ValueError('No sky image')
                skydata = []
                skymasks = []
                skyscales = []
                my_region = regions[idx]
                my_sky_scale = numpy.median(data_arr_rf[idx][my_region])
                for i in numpy.concatenate((limit1, limit2)):
                    region_s = regions[i]
                    data_s = data_arr_rf[i][region_s]
                    mask_s = objmask[region_s]
                    scale_s = numpy.median(data_s)
                    skydata.append(data_s)
                    skymasks.append(mask_s)
                    skyscales.append(scale_s)
                self.logger.debug('computing background with %d frames', len(skydata))
                sky, _, num = nacom.median(skydata, skymasks, scales=skyscales)
                # rescale
                sky *= my_sky_scale

                binmask = num == 0

                if numpy.any(binmask):
                    # We have pixels without
                    # sky background information
                    self.logger.warn('pixels without sky information when correcting %d',
                                 idx)

                    # FIXME: during development, this is faster
                    # sky[binmask] = sky[num != 0].mean()
                    # To continue we interpolate over the patches
                    narray.fixpix2(sky, binmask, out=sky, iterations=1)

                name = 'sky_%d_%03d.fits' % (inum, idx)
                fits.writeto(name, sky, overwrite=True)
                name = 'sky_binmask_%d_%03d.fits' % (inum, idx)
                fits.writeto(name, binmask.astype('int16'), overwrite=True)

                data_arr_sky.append(sky)
                arr = numpy.copy(data_arr_rf[idx])
                arr[my_region] = data_arr_rf[idx][my_region] - sky
                data_arr_rfs.append(arr)
                # subtract sky advanced

            if self.intermediate_results:
                self.logger.debug('save resized intermediate img')
                for idx, arr_r in enumerate(data_arr_rfs):
                    self.save_intermediate_array(arr_r, 'interm_%d_%03d.fits' % (inum, idx))


            hdulist = self.combine2(data_arr_rfs, mask_arr_r, data_hdul, offsetsp, use_errors)

            self.save_intermediate_img(hdulist, 'result_%d.fits' % inum)

            # For next step
            catalog, objmask = self.create_object_catalog(hdulist[0].data, border=50)

            data_arr_0 = [(d[r] + s) for d, r, s in zip(data_arr_rfs, regions, data_arr_sky)]
            data_arr_r = [d.copy() for d in data_arr_rfs]

        result = self.create_result(frame=hdulist)
        self.logger.info('end of dither recipe')
        return result
Пример #12
0
    def compute_advanced_sky_for_frame(self, frame, skyframes,
                                       step=0, save=True):
        self.logger.info('Correcting sky in frame %s', frame.lastname)
        self.logger.info('with sky computed from frames')
        for i in skyframes:
            self.logger.info('%s', i.flat_corrected)

        data = []
        scales = []
        masks = []
        # handle the FITS file to close it finally
        desc = []
        try:
            for i in skyframes:
                filename = i.flat_corrected
                hdulist = fits.open(filename, mode='readonly', memmap=True)

                data.append(hdulist['primary'].data[i.valid_region])
                desc.append(hdulist)
                #scales.append(numpy.median(data[-1]))
                if i.objmask_data is not None:
                    masks.append(i.objmask_data)
                    self.logger.debug('object mask is shared')
                elif i.objmask is not None:
                    hdulistmask = fits.open(
                        i.objmask, mode='readonly', memmap=True)
                    masks.append(hdulistmask['primary'].data)
                    desc.append(hdulistmask)
                    self.logger.debug('object mask is particular')
                else:
                    self.logger.warn('no object mask for %s', filename)

            self.logger.debug('computing background with %d frames', len(data))
            sky, _, num = nacom.median(data, masks)#, scales=scales)

        finally:
            # Closing all FITS files
            for hdl in desc:
                hdl.close()

        if numpy.any(num == 0):
            # We have pixels without
            # sky background information
            self.logger.warn('pixels without sky information when correcting %s',
                         frame.flat_corrected)
            binmask = num == 0
            # FIXME: during development, this is faster
            # sky[binmask] = sky[num != 0].mean()

            # To continue we interpolate over the patches
            narray.fixpix2(sky, binmask, out=sky, iterations=1)

            name = name_skybackgroundmask(frame.label, step)
            fits.writeto(name, binmask.astype('int16'), overwrite=True)

        name_sky = name_skybackground(frame.label, step)
        fits.writeto(name_sky, sky, overwrite=True)

        dst = name_skysub_proc(frame.label, step)
        prev = frame.lastname
        shutil.copyfile(prev, dst)
        frame.lastname = dst

        with fits.open(frame.lastname, mode='update') as hdulist:
            data = hdulist['primary'].data
            valid = data[frame.valid_region]
            valid -= sky