Beispiel #1
0
    def _s2_aerosol(self, ):

        self.s2_logger.propagate = False
        self.s2_logger.info('Start to retrieve atmospheric parameters.')
        self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
                          self.day, self.s2_u_bands)
        self.s2_logger.info('Reading in TOA reflectance.')
        selected_img = self.s2.get_s2_toa()
        self.s2_file_dir = self.s2.s2_file_dir
        self.s2.get_s2_cloud()
        self.s2_logger.info(
            'Find corresponding pixels between S2 and MODIS tiles')
        tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2',
                                          destination_res=500)
        if len(tiles.keys()) > 1:
            self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' %
                                len(tiles.keys()))
        self.mcd43_files = []
        szas, vzas, saas, vaas, raas = [], [], [], [], []
        boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], []
        for key in tiles.keys():
            #h,v = int(key[1:3]), int(key[-2:])
            self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key)
            mcd43_file = glob(self.mcd43_tmp %
                              (self.mcd43_dir, self.year, self.doy, key))[0]
            self.mcd43_files.append(mcd43_file)
            self.H_inds, self.L_inds = tiles[key]
            Lx, Ly = self.L_inds
            Hx, Hy = self.H_inds
            Hxs.append(Hx)
            Hys.append(Hy)
            self.s2_logger.info(
                'Getting the angles and simulated surface reflectance.')
            self.s2.get_s2_angles(self.reconstruct_s2_angle)

            self.s2_angles = np.zeros((4, 6, len(Hx)))
            for j, band in enumerate(self.s2_u_bands[:-2]):
                self.s2_angles[[0,2],j,:] = self.s2.angles['vza'][band][Hx, Hy], \
                                            self.s2.angles['vaa'][band][Hx, Hy]
                self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \
                                                          self.s2.angles['saa'][Hx, Hy]

#use mean value to fill bad values
            for i in range(4):
                mask = ~np.isfinite(self.s2_angles[i])
                if mask.sum() > 0:
                    self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \
                     np.flatnonzero(~mask),  self.s2_angles[i][~mask]) # simple interpolation
            vza, sza = self.s2_angles[:2]
            vaa, saa = self.s2_angles[2:]
            raa = vaa - saa
            szas.append(sza)
            vzas.append(vza)
            raas.append(raa)
            vaas.append(vaa)
            saas.append(saa)
            # get the simulated surface reflectance
            s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\
                                                              bands=(3,4,1,2,6,7), Linds= [Lx, Ly])
            boas.append(s2_boa)
            boa_qas.append(s2_boa_qa)
            brdf_stds.append(brdf_std)

        self.s2_boa = np.hstack(boas)
        self.s2_boa_qa = np.hstack(boa_qas)
        self.brdf_stds = np.hstack(brdf_stds)
        self.Hx = np.hstack(Hxs)
        self.Hy = np.hstack(Hys)
        vza = np.hstack(vzas)
        sza = np.hstack(szas)
        vaa = np.hstack(vaas)
        saa = np.hstack(saas)
        raa = np.hstack(raas)
        self.s2_angles = np.array([vza, sza, vaa, saa])
        #self.s2_boa, self.s2_boa_qa = self.s2_boa.flatten(), self.s2_boa_qa.flatten()
        self.s2_logger.info('Applying spectral transform.')
        self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
                                  np.array(self.s2_spectral_transform)[1,:-1][...,None]
        self.s2_logger.info('Getting elevation.')
        ele_data = reproject_data(self.global_dem,
                                  self.s2.s2_file_dir + '/B04.jp2',
                                  outputType=gdal.GDT_Float32).data
        mask = ~np.isfinite(ele_data)
        ele_data = np.ma.array(ele_data, mask=mask) / 1000.
        self.elevation = ele_data[self.Hx, self.Hy]

        self.s2_logger.info('Getting pripors from ECMWF forcasts.')
        sen_time_str = json.load(
            open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
        self.sen_time = datetime.datetime.strptime(sen_time_str,
                                                   u'%Y-%m-%dT%H:%M:%S.%fZ')
        example_file = self.s2.s2_file_dir + '/B04.jp2'
        aod, tcwv, tco3 = np.array(self._read_cams(example_file))[:, self.Hx,
                                                                  self.Hy]
        self.s2_aod550 = aod  #* (1-0.14) # validation of +14% biase
        self.s2_tco3 = tco3 * 46.698  #* (1 - 0.05)
        tcwv = tcwv / 10.
        self.s2_tco3_unc = np.ones(self.s2_tco3.shape) * 0.2
        self.s2_aod550_unc = np.ones(self.s2_aod550.shape) * 0.5

        self.s2_logger.info(
            'Trying to get the tcwv from the emulation of sen2cor look up table.'
        )

        #try:
        self._get_tcwv(selected_img, vza, sza, raa, ele_data)
        #except:
        #    self.s2_logger.warning('Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.')
        #    self.s2_tcwv     = tcwv
        #    self.s2_tcwv_unc = np.ones(self.s2_tcwv.shape) * 0.2

        self.s2_logger.info('Trying to get the aod from ddv method.')
        try:
            solved = self._get_ddv_aot(self.s2.angles, example_file,
                                       self.s2_tcwv, ele_data, selected_img)
            if solved[0] < 0:
                self.s2_logger.warning(
                    'DDV failed and only cams data used for the prior.')
            else:
                self.s2_logger.info(
                    'DDV solved aod is %.02f, and it will used as the mean value of cams prediction.'
                    % solved[0])
                self.s2_aod550 += (solved[0] - self.s2_aod550.mean())
        except:
            self.s2_logger.warning('Getting aod from ddv failed.')
        self.s2_logger.info('Applying PSF model.')
        if self.s2_psf is None:
            self.s2_logger.info('No PSF parameters specified, start solving.')
            high_img = np.repeat(
                np.repeat(selected_img['B11'], 2, axis=0), 2, axis=1) * 0.0001
            high_indexs = self.Hx, self.Hy
            low_img = self.s2_boa[4]
            qa, cloud = self.s2_boa_qa[4], self.s2.cloud
            psf = psf_optimize(high_img, high_indexs, low_img, qa, cloud, 2)
            xs, ys = psf.fire_shift_optimize()
            xstd, ystd = 29.75, 39
            ang = 0
            self.s2_logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \
                                 %(xstd, ystd, 0, xs, ys, 1-psf.costs.min()))
        else:
            xstd, ystd, ang, xs, ys = self.s2_psf
        # apply psf shifts without going out of the image extend
        shifted_mask = np.logical_and.reduce(
            ((self.Hx + int(xs) >= 0),
             (self.Hx + int(xs) < self.s2_full_res[0]),
             (self.Hy + int(ys) >= 0),
             (self.Hy + int(ys) < self.s2_full_res[0])))

        self.Hx, self.Hy = self.Hx[shifted_mask] + int(
            xs), self.Hy[shifted_mask] + int(ys)
        #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
        self.s2_boa = self.s2_boa[:, shifted_mask]
        self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]
        self.s2_angles = self.s2_angles[:, :, shifted_mask]
        self.elevation = self.elevation[shifted_mask]
        self.s2_aod550 = self.s2_aod550[shifted_mask]
        self.s2_tcwv = self.s2_tcwv[shifted_mask]
        self.s2_tco3 = self.s2_tco3[shifted_mask]

        self.s2_aod550_unc = self.s2_aod550_unc[shifted_mask]
        self.s2_tcwv_unc = self.s2_tcwv_unc[shifted_mask]
        self.s2_tco3_unc = self.s2_tco3_unc[shifted_mask]
        self.brdf_stds = self.brdf_stds[:, shifted_mask]

        self.s2_logger.info('Getting the convolved TOA reflectance.')
        self.valid_pixs = sum(
            shifted_mask)  # count how many pixels is still within the s2 tile
        ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
        self.bad_pixs = np.zeros(self.valid_pixs).astype(bool)
        imgs = []
        for i, band in enumerate(self.s2_u_bands[:-2]):
            if selected_img[band].shape != self.s2_full_res:
                selected_img[band] = self.repeat_extend(selected_img[band],
                                                        shape=self.s2_full_res)
            else:
                pass
            selected_img[band][0, :] = -9999
            selected_img[band][-1, :] = -9999
            selected_img[band][:, 0] = -9999
            selected_img[band][:, -1] = -9999
            imgs.append(selected_img[band])
            # filter out the bad pixels
            self.bad_pixs |= cloud_dilation(self.s2.cloud |\
                                            (selected_img[band] <= 0) | \
                                            (selected_img[band] >= 10000),\
                                            iteration= ker_size/2)[self.Hx, self.Hy]
        del selected_img
        del self.s2.selected_img
        del high_img
        del self.s2.angles
        del self.s2.sza
        del self.s2.saa
        del self.s2
        ker = self.gaussian(xstd, ystd, ang)
        f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
                                                                  .Hy] * 0.0001
        half = parmap(f, imgs[:3])
        self.s2_toa = np.array(half + parmap(f, imgs[3:]))
        #self.s2_toa = np.array(parmap(f,imgs))
        del imgs
        # get the valid value masks
        qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)

        boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
                          np.all(self.s2_boa > 0, axis = 0) &\
                          np.all(self.s2_boa < 1, axis = 0)
        toa_mask = (~self.bad_pixs) &\
                    np.all(self.s2_toa > 0, axis = 0) &\
                    np.all(self.s2_toa < 1, axis = 0)
        self.s2_mask = boa_mask & toa_mask & qua_mask & (~self.elevation.mask)
        self.s2_AEE, self.s2_bounds = self._load_emus(self.s2_sensor)
Beispiel #2
0
    f_dat[mask] = np.nan
    unc_dat[mask] = np.nan
    return f_dat, unc_dat


if __name__ == '__main__':
    example_file = '/store/S2_data/50/S/LH/2017/2/24/0/B04.jp2'
    mcd43_dir = '/data/selene/ucfajlg/Hebei/MCD43/'
    bands = [3, 4, 1, 2, 6, 7]
    va_files = [
        '/'.join(example_file.split('/')[:-1]) +
        '/angles/VAA_VZA_B%02d.img' % i for i in [2, 3, 4, 8, 11, 12]
    ]
    sza = np.zeros((10980, 10980))
    saa = sza.copy()
    from grab_s2_toa import read_s2
    s2 = read_s2('/store/S2_data/',
                 '50SLH',
                 2017,
                 2,
                 24,
                 bands=['B02', 'B03', 'B04', 'B08', 'B11', 'B12'])
    s2.get_s2_angles()
    sa_files = [s2.angles['saa'], s2.angles['sza']]
    f_data, unc_data = MCD43_SurRef(mcd43_dir,
                                    example_file,
                                    2017,
                                    55, [va_files, sa_files],
                                    sun_view_ang_scale=[1., 0.01],
                                    bands=bands)
Beispiel #3
0
    def atmospheric_correction(self, ):

        self.logger.propagate = False
        self.s2_sensor = 'msi'
        self.logger.info('Loading emulators.')
        #self.s2_inv_AEE = self._load_inverse_emus(self.s2_sensor)
        self._load_xa_xb_xc_emus()
        self.s2   = read_s2(self.s2_toa_dir, self.s2_tile, \
                            self.year, self.month, self.day, bands=None)
        self.logger.info('Reading in the reflectance.')
        all_refs = self.s2.get_s2_toa()
        self.logger.info('Reading in the angles')
        self.s2.get_s2_angles(self.reconstruct_s2_angle)
        self.sza, self.saa = self.s2.angles['sza'], self.s2.angles['saa']

        self.logger.info('Doing 10 meter bands')
        self._10meter_ref = np.array([all_refs[band].astype(float)/10000. for band \
                                      in ['B02', 'B03', 'B04', 'B08']])
        self._10meter_vza = np.array([
            self.s2.angles['vza'][band]
            for band in ['B02', 'B03', 'B04', 'B08']
        ])
        self._10meter_vaa = np.array([
            self.s2.angles['vaa'][band]
            for band in ['B02', 'B03', 'B04', 'B08']
        ])
        self.logger.info('Getting control variables for 10 meters bands.')
        self._10meter_aod, self._10meter_tcwv, self._10meter_tco3,\
                           self._10meter_ele = self.get_control_variables('B04')
        self._block_size = 3660
        self._num_blocks = 10980 / self._block_size
        self._mean_size = 60
        self._10meter_band_indexs = [1, 2, 3, 7]
        self.rsr = [PredefinedWavelengths.S2A_MSI_02, PredefinedWavelengths.S2A_MSI_03, \
                    PredefinedWavelengths.S2A_MSI_04, PredefinedWavelengths.S2A_MSI_08 ]
        self.logger.info('Fire correction and splited into %d blocks.' %
                         self._num_blocks**2)
        self.fire_correction(self._10meter_ref, self.sza, self._10meter_vza,\
                             self.saa, self._10meter_vaa, self._10meter_aod,\
                             self._10meter_tcwv, self._10meter_tco3, self._10meter_ele,\
                             self._10meter_band_indexs)
        self.toa_rgb = clip(
            self._10meter_ref[[2, 1, 0], ...].transpose(1, 2, 0) * 255 / 0.255,
            0., 255.).astype(uint8)
        self.boa_rgb = clip(
            self.boa[[2, 1, 0], ...].transpose(1, 2, 0) * 255 / 0.255, 0.,
            255.).astype(uint8)
        self._save_rgb(self.toa_rgb, 'TOA_RGB.tif',
                       self.s2.s2_file_dir + '/B04.jp2')
        self._save_rgb(self.boa_rgb, 'BOA_RGB.tif',
                       self.s2.s2_file_dir + '/B04.jp2')

        del self._10meter_ref
        del self._10meter_vza
        del self._10meter_vaa
        del self._10meter_aod         \
                del self._10meter_tcwv
        del self._10meter_tco3
        del self._10meter_ele

        #self.sur_refs.update(dict(zip(['B02', 'B03', 'B04', 'B08'], self.boa)))
        self._save_img(self.boa, ['B02', 'B03', 'B04', 'B08'])
        del self.boa

        self.logger.info('Doing 20 meter bands')
        self._20meter_ref = np.array([all_refs[band].astype(float)/10000. for band \
                                      in ['B05', 'B06', 'B07', 'B8A', 'B11', 'B12']])

        self._20meter_vza = np.array([self.s2.angles['vza'][band] for band \
                                      in ['B05', 'B06', 'B07', 'B8A', 'B11', 'B12']]).reshape(6, 5490, 2, 5490, 2).mean(axis=(4,2))
        self._20meter_vaa = np.array([self.s2.angles['vaa'][band] for band \
                                      in ['B05', 'B06', 'B07', 'B8A', 'B11', 'B12']]).reshape(6, 5490, 2, 5490, 2).mean(axis=(4,2))

        self._20meter_sza = self.sza.reshape(5490, 2, 5490,
                                             2).mean(axis=(3, 1))
        self._20meter_saa = self.saa.reshape(5490, 2, 5490,
                                             2).mean(axis=(3, 1))

        self.logger.info('Getting control variables for 20 meters bands.')
        self._20meter_aod, self._20meter_tcwv, self._20meter_tco3,\
                           self._20meter_ele = self.get_control_variables('B05')
        self._block_size = 1830
        self._num_blocks = 5490 / self._block_size
        self._mean_size = 30
        self._20meter_band_indexs = [4, 5, 6, 8, 11, 12]
        self.rsr = [PredefinedWavelengths.S2A_MSI_05, PredefinedWavelengths.S2A_MSI_06, \
                    PredefinedWavelengths.S2A_MSI_07, PredefinedWavelengths.S2A_MSI_09, \
                    PredefinedWavelengths.S2A_MSI_12, PredefinedWavelengths.S2A_MSI_13]

        self.logger.info('Fire correction and splited into %d blocks.' %
                         self._num_blocks**2)
        self.fire_correction(self._20meter_ref, self._20meter_sza, self._20meter_vza,\
                             self._20meter_saa, self._20meter_vaa, self._20meter_aod,\
                             self._20meter_tcwv, self._20meter_tco3, self._20meter_ele,\
                             self._20meter_band_indexs)

        del self._20meter_ref
        del self._20meter_vza
        del self._20meter_vaa
        del self._20meter_sza
        del self._20meter_saa
        del self._20meter_aod
        del self._20meter_tcwv
        del self._20meter_tco3
        del self._20meter_ele

        #self.sur_refs.update(dict(zip(['B05', 'B06', 'B07', 'B8A', 'B11', 'B12'], self.boa)))
        self._save_img(self.boa, ['B05', 'B06', 'B07', 'B8A', 'B11', 'B12'])
        del self.boa

        self.logger.info('Doing 60 meter bands')
        self._60meter_ref = np.array([all_refs[band].astype(float)/10000. for band \
                                      in ['B01', 'B09', 'B10']])
        self._60meter_vza = np.array([self.s2.angles['vza'][band] for band \
                                      in ['B01', 'B09', 'B10']]).reshape(3, 1830, 6, 1830, 6).mean(axis=(4, 2))
        self._60meter_vaa = np.array([self.s2.angles['vaa'][band] for band \
                                      in ['B01', 'B09', 'B10']]).reshape(3, 1830, 6, 1830, 6).mean(axis=(4, 2))
        self._60meter_sza = self.sza.reshape(1830, 6, 1830,
                                             6).mean(axis=(3, 1))
        self._60meter_saa = self.saa.reshape(1830, 6, 1830,
                                             6).mean(axis=(3, 1))

        self.logger.info('Getting control variables for 60 meters bands.')
        self._60meter_aod, self._60meter_tcwv, self._60meter_tco3,\
                           self._60meter_ele = self.get_control_variables('B09')
        self._block_size = 610
        self._num_blocks = 1830 / self._block_size
        self._mean_size = 10
        self._60meter_band_indexs = [0, 9, 10]
        self.rsr = [
            PredefinedWavelengths.S2A_MSI_01, PredefinedWavelengths.S2A_MSI_10,
            PredefinedWavelengths.S2A_MSI_11
        ]

        self.logger.info('Fire correction and splited into %d blocks.' %
                         self._num_blocks**2)
        self.fire_correction(self._60meter_ref, self._60meter_sza, self._60meter_vza,\
                             self._60meter_saa, self._60meter_vaa, self._60meter_aod,\
                             self._60meter_tcwv, self._60meter_tco3, self._60meter_ele,\
                             self._60meter_band_indexs)

        del self._60meter_ref
        del self._60meter_vza
        del self._60meter_vaa
        del self._60meter_sza
        del self._60meter_saa
        del self._60meter_aod
        del self._60meter_tcwv
        del self._60meter_tco3
        del self._60meter_ele

        #self.sur_refs.update(dict(zip(['B01', 'B09', 'B10'], self.boa)))
        self._save_img(self.boa, ['B01', 'B09', 'B10'])
        del self.boa
        del all_refs
        del self.s2.selected_img
        del self.s2.angles
        self.logger.info('Done!')
Beispiel #4
0
    def _s2_aerosol(self, ):

        self.s2_logger.propagate = False
        self.s2_logger.info('Start to retrieve atmospheric parameters.')
        self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
                          self.day, self.s2_u_bands)
        self.s2_logger.info('Reading in TOA reflectance.')
        selected_img = self.s2.get_s2_toa()
        self.s2_file_dir = self.s2.s2_file_dir
        self.s2.get_s2_cloud()
        self.s2_logger.info('Loading emulators.')
        self._load_xa_xb_xc_emus()
        self.s2_logger.info(
            'Find corresponding pixels between S2 and MODIS tiles')
        tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2',
                                          destination_res=500)
        if len(tiles.keys()) > 1:
            self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' %
                                len(tiles.keys()))
        self.mcd43_files = []
        boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], []
        self.s2_logger.info(
            'Getting the angles and simulated surface reflectance.')
        for key in tiles.keys():
            self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key)
            mcd43_file = glob(self.mcd43_tmp %
                              (self.mcd43_dir, self.year, self.doy, key))[0]
            self.mcd43_files.append(mcd43_file)
            self.H_inds, self.L_inds = tiles[key]
            Lx, Ly = self.L_inds
            Hx, Hy = self.H_inds
            Hxs.append(Hx)
            Hys.append(Hy)
            self.s2.get_s2_angles(self.reconstruct_s2_angle)

            self.s2_angles = np.zeros((4, 6, len(Hx)))
            for j, band in enumerate(self.s2_u_bands[:-2]):
                self.s2_angles[[0,2],j,:] = (self.s2.angles['vza'][band])[Hx, Hy], \
                                            (self.s2.angles['vaa'][band])[Hx, Hy]

                self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \
                                                          self.s2.angles['saa'][Hx, Hy]

#use mean value to fill bad values
            for i in range(4):
                mask = ~np.isfinite(self.s2_angles[i])
                if mask.sum() > 0:
                    self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \
                     np.flatnonzero(~mask), \
                                                                      self.s2_angles[i][~mask]) # simple interpolation
            vza, sza = self.s2_angles[:2]
            vaa, saa = self.s2_angles[2:]
            raa = vaa - saa
            # get the simulated surface reflectance
            s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\
                                                              bands=(3,4,1,2,6,7), Linds= [Lx, Ly])
            boas.append(s2_boa)
            boa_qas.append(s2_boa_qa)
            brdf_stds.append(brdf_std)
        self.s2_boa = np.hstack(boas)
        self.s2_boa_qa = np.hstack(boa_qas)
        self.brdf_stds = np.hstack(brdf_stds)
        self.s2_logger.info('Applying spectral transform.')
        self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
                                  np.array(self.s2_spectral_transform)[1,:-1][...,None]
        self.Hx = np.hstack(Hxs)
        self.Hy = np.hstack(Hys)
        del sza
        del vza
        del saa
        del vaa
        del raa
        del mask
        del boas
        del boa_qas
        del brdf_stds
        del Hxs
        del Hys
        shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \
                 self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks)
        self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1))
        self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1))
        self.vza = []
        self.vaa = []
        for band in self.s2_u_bands[:-2]:
            self.vza.append(
                self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1)))
            self.vaa.append(
                self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1)))
        self.vza = np.array(self.vza)
        self.vaa = np.array(self.vaa)
        self.raa = self.saa[None, ...] - self.vaa
        self.s2_logger.info('Getting elevation.')
        example_file = self.s2.s2_file_dir + '/B04.jp2'
        ele_data = reproject_data(self.global_dem,
                                  example_file,
                                  outputType=gdal.GDT_Float32).data
        mask = ~np.isfinite(ele_data)
        ele_data = np.ma.array(ele_data, mask=mask) / 1000.
        self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \
                                           self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1))

        self.s2_logger.info('Getting pripors from ECMWF forcasts.')
        sen_time_str = json.load(
            open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
        self.sen_time = datetime.datetime.strptime(sen_time_str,
                                                   u'%Y-%m-%dT%H:%M:%S.%fZ')
        aot, tcwv, tco3 = np.array(self._read_cams(example_file)).reshape((3, self.num_blocks, \
                                   self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2))
        self.aot = aot  #* (1-0.14) # validation of +14% biase
        self.tco3 = tco3 * 46.698  #* (1 - 0.05)
        tcwv = tcwv / 10.
        self.tco3_unc = np.ones(self.tco3.shape) * 0.2
        self.aot_unc = np.ones(self.aot.shape) * 0.5

        self.s2_logger.info(
            'Trying to get the tcwv from the emulation of sen2cor look up table.'
        )
        try:
            self._get_tcwv(selected_img, self.s2.angles['vza'],
                           self.s2.angles['vaa'], self.s2.angles['sza'],
                           self.s2.angles['saa'], ele_data)
        except:
            self.s2_logger.warning(
                'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.'
            )
            self.tcwv = tcwv
            self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2

        self.s2_logger.info('Trying to get the aot from ddv method.')
        try:
            solved = self._get_ddv_aot(selected_img)
            if solved[0] < 0:
                self.s2_logger.warning(
                    'DDV failed and only cams data used for the prior.')
            else:
                self.s2_logger.info(
                    'DDV solved aot is %.02f, and it will used as the mean value of cams prediction.'
                    % solved[0])
                self.aot += (solved[0] - self.aot.mean())
        except:
            self.s2_logger.warning('Getting aot from ddv failed.')
        self.s2_logger.info('Applying PSF model.')
        if self.s2_psf is None:
            xstd, ystd, ang, xs, ys = self._get_psf(selected_img)
        else:
            xstd, ystd, ang, xs, ys = self.s2_psf
        # apply psf shifts without going out of the image extend
        shifted_mask = np.logical_and.reduce(
            ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]),
             (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0])))

        self.Hx, self.Hy = self.Hx[shifted_mask] + int(
            xs), self.Hy[shifted_mask] + int(ys)
        #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
        self.s2_boa = self.s2_boa[:, shifted_mask]
        self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]
        self.brdf_stds = self.brdf_stds[:, shifted_mask]

        self.s2_logger.info('Getting the convolved TOA reflectance.')
        self.valid_pixs = sum(
            shifted_mask)  # count how many pixels is still within the s2 tile
        ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
        self.bad_pixs = np.zeros(self.valid_pixs).astype(bool)
        imgs = []
        for i, band in enumerate(self.s2_u_bands[:-2]):
            if selected_img[band].shape != self.full_res:
                imgs.append(
                    self.repeat_extend(selected_img[band],
                                       shape=self.full_res))
            else:
                imgs.append(selected_img[band])

        border_mask = np.zeros(self.full_res).astype(bool)
        border_mask[[0, -1], :] = True
        border_mask[:, [0, -1]] = True
        self.bad_pixs = cloud_dilation(self.s2.cloud | border_mask,
                                       iteration=ker_size / 2)[self.Hx,
                                                               self.Hy]
        del selected_img
        del self.s2.selected_img
        del self.s2.angles['vza']
        del self.s2.angles['vaa']
        del self.s2.angles['sza']
        del self.s2.angles['saa']
        del self.s2.sza
        del self.s2.saa
        del self.s2
        ker = self.gaussian(xstd, ystd, ang)
        f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
                                                                  .Hy] * 0.0001
        half = parmap(f, imgs[:3])
        self.s2_toa = np.array(half + parmap(f, imgs[3:]))
        del imgs
        # get the valid value masks
        qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)
        boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
                          np.all(self.s2_boa > 0, axis = 0) &\
                          np.all(self.s2_boa < 1, axis = 0)
        toa_mask =       (~self.bad_pixs) &\
                          np.all(self.s2_toa > 0, axis = 0) &\
                          np.all(self.s2_toa < 1, axis = 0)
        self.s2_mask = boa_mask & toa_mask & qua_mask
        self.Hx = self.Hx[self.s2_mask]
        self.Hy = self.Hy[self.s2_mask]
        self.s2_toa = self.s2_toa[:, self.s2_mask]
        self.s2_boa = self.s2_boa[:, self.s2_mask]
        self.s2_boa_qa = self.s2_boa_qa[:, self.s2_mask]
        self.brdf_stds = self.brdf_stds[:, self.s2_mask]
        self.s2_boa_unc = grab_uncertainty(self.s2_boa, self.boa_bands,
                                           self.s2_boa_qa,
                                           self.brdf_stds).get_boa_unc()
        self.s2_logger.info('Solving...')
        self.aero = solving_atmo_paras(
            self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa,
            self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc,
            self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy,
            self.full_res, self.aero_res, self.emus, self.band_indexs,
            self.boa_bands)
        solved = self.aero._optimization()
        return solved
Beispiel #5
0
    def _s2_aerosol(self, ):

        self.s2_logger.propagate = False
        self.s2_logger.info('Start to retrieve atmospheric parameters.')
        self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
                          self.day, self.s2_u_bands)
        self.s2_logger.info('Reading in TOA reflectance.')
        selected_img = self.s2.get_s2_toa()
        self.s2_file_dir = self.s2.s2_file_dir
        self.example_file = self.s2.s2_file_dir + '/B04.jp2'
        self.s2.get_s2_cloud()
        self.s2_logger.info('Loading emulators.')
        self._load_xa_xb_xc_emus()
        self.s2_logger.info(
            'Getting the angles and simulated surface reflectance.')
        self.s2.get_s2_angles()
        sa_files = [self.s2.angles['saa'], self.s2.angles['sza']]
        if os.path.exists(self.s2_file_dir + '/angles/'):
            va_files = [
                self.s2_file_dir + '/angles/VAA_VZA_B%02d.img' % i
                for i in [2, 3, 4, 8, 11, 12]
            ]
        else:
            va_files = [
                np.array(
                    [self.s2.angles['vaa'][band], self.s2.angles['vza'][band]])
                for band in self.s2_u_bands[:-2]
            ]
        if len(glob(self.s2_file_dir + '/MCD43.npz')) == 0:
            boa, unc, hx, hy, lx, ly, flist = MCD43_SurRef(self.mcd43_dir, self.example_file, \
                                                           self.year, self.doy, [sa_files, va_files],
                                                           sun_view_ang_scale=[1, 0.01], bands = [3,4,1,2,6,7], tolz=0.001, reproject=False)
            np.savez(self.s2_file_dir + '/MCD43.npz',
                     boa=boa,
                     unc=unc,
                     hx=hx,
                     hy=hy,
                     lx=lx,
                     ly=ly,
                     flist=flist)
        else:
            f = np.load(self.s2_file_dir + '/MCD43.npz')
            boa, unc, hx, hy, lx, ly, flist = f['boa'], f['unc'], f['hx'], f[
                'hy'], f['lx'], f['ly'], f['flist']
        self.Hx, self.Hy = hx, hy
        self.s2_logger.info('Update cloud mask.')
        flist = [
            f.split('/')[0] + self.mcd43_dir + '/' + f.split('/')[-1]
            for f in flist
        ]
        self._mcd43_cloud(flist, lx, ly, self.example_file, boa,
                          selected_img['B12'])
        self.s2_logger.info('Applying spectral transform.')
        self.s2_boa_qa = np.ma.array(unc)
        self.s2_boa = np.ma.array(boa)*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
                                      np.array(self.s2_spectral_transform)[1,:-1][...,None]
        shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \
                 self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks)
        self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1))
        self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1))
        self.vza = []
        self.vaa = []
        for band in self.s2_u_bands[:-2]:
            self.vza.append(
                self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1)))
            self.vaa.append(
                self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1)))
        self.vza = np.array(self.vza)
        self.vaa = np.array(self.vaa)
        self.raa = self.saa[None, ...] - self.vaa
        self.s2_logger.info('Getting elevation.')
        ele_data = reproject_data(self.global_dem,
                                  self.example_file,
                                  outputType=gdal.GDT_Float32).data
        mask = ~np.isfinite(ele_data)
        ele_data = np.ma.array(ele_data, mask=mask) / 1000.
        self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \
                                           self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1))

        self.s2_logger.info('Getting pripors from ECMWF forcasts.')
        sen_time_str = json.load(
            open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
        self.sen_time = datetime.datetime.strptime(sen_time_str,
                                                   u'%Y-%m-%dT%H:%M:%S.%fZ')
        aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)).reshape((3, self.num_blocks, \
                                   self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2))
        mod08_aot = self._mod08_aot()
        #if np.isnan(mod08_aot):
        self.aot = aot - (aot.mean() - np.nanmean([mod08_aot, aot.mean()]))
        #else:
        #    temp        = np.zeros_like(aot)
        #    temp[:]     = mod08_aot
        #     self.aot    = temp
        self.tco3 = tco3 * 46.698
        tcwv = tcwv / 10.
        self.tco3_unc = np.ones(self.tco3.shape) * 0.2
        self.aot_unc = np.ones(self.aot.shape) * 2.

        self.s2_logger.info(
            'Trying to get the tcwv from the emulation of sen2cor look up table.'
        )
        try:
            self._get_tcwv(selected_img, self.s2.angles['vza'],
                           self.s2.angles['vaa'], self.s2.angles['sza'],
                           self.s2.angles['saa'], ele_data)
        except:
            self.s2_logger.warning(
                'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.'
            )
            self.tcwv = tcwv
            self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2
        '''
        self.s2_logger.info('Trying to get the aot from ddv method.')
        try:
            solved = self._get_ddv_aot(selected_img)
            if solved[0] < 0:
                self.s2_logger.warning('DDV failed and only cams data used for the prior.')
            else:
                self.s2_logger.info('DDV solved aot is %.02f, and it will used as the mean value of cams prediction.'%solved[0])
                self.aot[:] = solved[0] #self.aot + (solved[0]-self.aot.mean())
        except:
            self.s2_logger.warning('Getting aot from ddv failed.')
        '''
        self.s2_logger.info(
            'Mean values for priors are: %.02f, %.02f, %.02f and mod08 aot is %.02f'
            % (aot.mean(), self.tcwv.mean(), self.tco3.mean(), mod08_aot))
        self.s2_logger.info('Applying PSF model.')
        if self.s2_psf is None:
            xstd, ystd, ang, xs, ys = self._get_psf(selected_img)
        else:
            xstd, ystd, ang, xs, ys = self.s2_psf
        # apply psf shifts without going out of the image extend
        shifted_mask = np.logical_and.reduce(
            ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]),
             (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0])))

        self.Hx, self.Hy = self.Hx[shifted_mask] + int(
            xs), self.Hy[shifted_mask] + int(ys)
        #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
        self.s2_boa = self.s2_boa[:, shifted_mask]
        self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]

        self.s2_logger.info('Getting the convolved TOA reflectance.')
        #self.valid_pixs = sum(shifted_mask) # count how many pixels is still within the s2 tile
        ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
        #self.bad_pixs   = np.zeros(self.valid_pixs).astype(bool)
        imgs = []
        for i, band in enumerate(self.s2_u_bands[:-2]):
            if selected_img[band].shape != self.full_res:
                imgs.append(
                    self.repeat_extend(selected_img[band],
                                       shape=self.full_res))
            else:
                imgs.append(selected_img[band])

        border_mask = np.zeros(self.full_res).astype(bool)
        border_mask[[0, -1], :] = True
        border_mask[:, [0, -1]] = True
        self.dcloud = binary_dilation(self.cloud | border_mask,
                                      structure=np.ones((3, 3)).astype(bool),
                                      iterations=ker_size / 2).astype(bool)
        self.bad_pixs = self.dcloud[self.Hx, self.Hy]
        del selected_img
        del self.s2.selected_img
        del self.s2.angles['vza']
        del self.s2.angles['vaa']
        del self.s2.angles['sza']
        del self.s2.angles['saa']
        del self.s2.sza
        del self.s2.saa
        del self.s2
        ker = self.gaussian(xstd, ystd, ang)
        f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
                                                                  .Hy] * 0.0001
        half = parmap(f, imgs[:3])
        self.s2_toa = np.array(half + parmap(f, imgs[3:]))
        del imgs
        # get the valid value masks
        qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)
        boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
                          np.all(self.s2_boa >= 0.001, axis = 0) &\
                          np.all(self.s2_boa < 1, axis = 0)
        toa_mask =       (~self.bad_pixs) &\
                          np.all(self.s2_toa >= 0.0001, axis = 0) &\
                          np.all(self.s2_toa < 1., axis = 0)
        self.s2_mask = boa_mask & toa_mask & qua_mask
        self.Hx = self.Hx[self.s2_mask]
        self.Hy = self.Hy[self.s2_mask]
        self.s2_toa = self.s2_toa[:, self.s2_mask]
        self.s2_boa = self.s2_boa[:, self.s2_mask]
        self.s2_boa_unc = self.s2_boa_qa[:, self.s2_mask]
        self.s2_logger.info('Solving...')
        tempm = np.zeros(self.full_res)
        tempm[self.Hx, self.Hy] = 1
        tempm = tempm.reshape(self.num_blocks, self.block_size, \
                              self.num_blocks, self.block_size).astype(int).sum(axis=(3,1))
        mask = ~self.dcloud
        self.mask = mask.reshape(self.num_blocks, self.block_size, \
                                 self.num_blocks, self.block_size).astype(int).sum(axis=(3,1))
        self.mask = ((self.mask / ((1. * self.block_size)**2)) >= 0.5) & (
            (tempm / ((self.aero_res / 500.)**2)) >= 0.5)
        self.mask = binary_erosion(self.mask,
                                   structure=np.ones((3, 3)).astype(bool))
        self.aero = solving_atmo_paras(
            self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa,
            self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc,
            self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy,
            self.mask, self.full_res, self.aero_res, self.emus,
            self.band_indexs, self.boa_bands)
        solved = self.aero._optimization()
        return solved
Beispiel #6
0
    from datetime import datetime
    example_file = '/store/S2_data/50/S/MG/2016/1/24/0/B04.jp2'
    date = datetime.strptime('/'.join(example_file.split('/')[-5:-2]),
                             '%Y/%m/%d')
    doy = date.timetuple().tm_yday
    mcd43_dir = '/data/selene/ucfajlg/Hebei/MCD43/'
    bands = [3, 4, 1, 2, 6, 7]
    va_files = [
        '/'.join(example_file.split('/')[:-1]) +
        '/angles/VAA_VZA_B%02d.img' % i for i in [2, 3, 4, 8, 11, 12]
    ]
    sza = np.zeros((10980, 10980))
    saa = sza.copy()
    from grab_s2_toa import read_s2
    tile = ''.join(example_file.split('/')[-8:-5])
    s2 = read_s2('/store/S2_data/',
                 tile,
                 date.year,
                 date.month,
                 date.day,
                 bands=['B02', 'B03', 'B04', 'B08', 'B11', 'B12'])
    s2.get_s2_angles()
    sa_files = [s2.angles['saa'], s2.angles['sza']]
    ret = MCD43_SurRef(mcd43_dir,
                       example_file,
                       date.year,
                       doy, [sa_files, va_files],
                       sun_view_ang_scale=[1., 0.01],
                       bands=bands,
                       reproject=False)