def _get_psf(self, ):
'''
Get the PSF parameters
'''
toa = self._toa_bands[-1].ReadAsArray() * self.ref_scale + self.ref_off
index = [self.hx, self.hy]
boa = np.ma.array(self.boa[-1])
boa_unc = self.boa_unc[-1]
mask = self.bad_pix
thresh = 0.1
ang = 0
psf = psf_optimize(toa,
index,
boa,
boa_unc,
mask,
thresh,
xstd=self.psf_xstd,
ystd=self.psf_ystd)
xs, ys = psf.fire_shift_optimize()
self.logger.info('Solved PSF: %.02f, %.02f, %d, %d, %d, and R value is: %.03f.' \
%(self.psf_xstd, self.psf_ystd, 0, xs, ys, 1-psf.costs.min()))
shifted_mask = np.logical_and.reduce(
((self.hx + int(xs) >= 0), (self.hx + int(xs) < self.full_res[0]),
(self.hy + int(ys) >= 0), (self.hy + int(ys) < self.full_res[1])))
self.hx, self.hy = self.hx[shifted_mask] + int(
xs), self.hy[shifted_mask] + int(ys)
self.boa = self.boa[:, shifted_mask]
self.boa_unc = self.boa_unc[:, shifted_mask]
def _get_psf(self,):
self.logger.info('No PSF parameters specified, start solving.')
xstd, ystd = 12., 20.
psf = psf_optimize(self.toa[-1].data, [self.Hx, self.Hy], np.ma.array(self.boa[-1]), self.boa_qa[-1], self.ecloud, 0.1, xstd=xstd, ystd= ystd)
xs, ys = psf.fire_shift_optimize()
ang = 0
self.logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \
%(xstd, ystd, 0, xs, ys, 1-psf.costs.min()))
return xstd, ystd, ang, xs, ys
def _get_psf(self, selected_img):
self.s2_logger.info('No PSF parameters specified, start solving.')
high_img = np.repeat(
np.repeat(selected_img['B11'], 2, axis=0), 2, axis=1) * 0.0001
high_indexs = self.Hx, self.Hy
low_img = self.s2_boa[4]
qa, cloud = self.s2_boa_qa[4], self.s2.cloud
psf = psf_optimize(high_img, high_indexs, low_img, qa, cloud, 2)
xs, ys = psf.fire_shift_optimize()
xstd, ystd = 29.75, 39
ang = 0
self.s2_logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \
%(xstd, ystd, 0, xs, ys, 1-psf.costs.min()))
return xstd, ystd, ang, xs, ys
def _s2_aerosol(self, ):
self.s2_logger.propagate = False
self.s2_logger.info('Start to retrieve atmospheric parameters.')
self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
self.day, self.s2_u_bands)
self.s2_logger.info('Reading in TOA reflectance.')
selected_img = self.s2.get_s2_toa()
self.s2_file_dir = self.s2.s2_file_dir
self.s2.get_s2_cloud()
self.s2_logger.info(
'Find corresponding pixels between S2 and MODIS tiles')
tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2',
destination_res=500)
if len(tiles.keys()) > 1:
self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' %
len(tiles.keys()))
self.mcd43_files = []
szas, vzas, saas, vaas, raas = [], [], [], [], []
boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], []
for key in tiles.keys():
#h,v = int(key[1:3]), int(key[-2:])
self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key)
mcd43_file = glob(self.mcd43_tmp %
(self.mcd43_dir, self.year, self.doy, key))[0]
self.mcd43_files.append(mcd43_file)
self.H_inds, self.L_inds = tiles[key]
Lx, Ly = self.L_inds
Hx, Hy = self.H_inds
Hxs.append(Hx)
Hys.append(Hy)
self.s2_logger.info(
'Getting the angles and simulated surface reflectance.')
self.s2.get_s2_angles(self.reconstruct_s2_angle)
self.s2_angles = np.zeros((4, 6, len(Hx)))
for j, band in enumerate(self.s2_u_bands[:-2]):
self.s2_angles[[0,2],j,:] = self.s2.angles['vza'][band][Hx, Hy], \
self.s2.angles['vaa'][band][Hx, Hy]
self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \
self.s2.angles['saa'][Hx, Hy]
#use mean value to fill bad values
for i in range(4):
mask = ~np.isfinite(self.s2_angles[i])
if mask.sum() > 0:
self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \
np.flatnonzero(~mask), self.s2_angles[i][~mask]) # simple interpolation
vza, sza = self.s2_angles[:2]
vaa, saa = self.s2_angles[2:]
raa = vaa - saa
szas.append(sza)
vzas.append(vza)
raas.append(raa)
vaas.append(vaa)
saas.append(saa)
# get the simulated surface reflectance
s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\
bands=(3,4,1,2,6,7), Linds= [Lx, Ly])
boas.append(s2_boa)
boa_qas.append(s2_boa_qa)
brdf_stds.append(brdf_std)
self.s2_boa = np.hstack(boas)
self.s2_boa_qa = np.hstack(boa_qas)
self.brdf_stds = np.hstack(brdf_stds)
self.Hx = np.hstack(Hxs)
self.Hy = np.hstack(Hys)
vza = np.hstack(vzas)
sza = np.hstack(szas)
vaa = np.hstack(vaas)
saa = np.hstack(saas)
raa = np.hstack(raas)
self.s2_angles = np.array([vza, sza, vaa, saa])
#self.s2_boa, self.s2_boa_qa = self.s2_boa.flatten(), self.s2_boa_qa.flatten()
self.s2_logger.info('Applying spectral transform.')
self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
np.array(self.s2_spectral_transform)[1,:-1][...,None]
self.s2_logger.info('Getting elevation.')
ele_data = reproject_data(self.global_dem,
self.s2.s2_file_dir + '/B04.jp2',
outputType=gdal.GDT_Float32).data
mask = ~np.isfinite(ele_data)
ele_data = np.ma.array(ele_data, mask=mask) / 1000.
self.elevation = ele_data[self.Hx, self.Hy]
self.s2_logger.info('Getting pripors from ECMWF forcasts.')
sen_time_str = json.load(
open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
self.sen_time = datetime.datetime.strptime(sen_time_str,
u'%Y-%m-%dT%H:%M:%S.%fZ')
example_file = self.s2.s2_file_dir + '/B04.jp2'
aod, tcwv, tco3 = np.array(self._read_cams(example_file))[:, self.Hx,
self.Hy]
self.s2_aod550 = aod #* (1-0.14) # validation of +14% biase
self.s2_tco3 = tco3 * 46.698 #* (1 - 0.05)
tcwv = tcwv / 10.
self.s2_tco3_unc = np.ones(self.s2_tco3.shape) * 0.2
self.s2_aod550_unc = np.ones(self.s2_aod550.shape) * 0.5
self.s2_logger.info(
'Trying to get the tcwv from the emulation of sen2cor look up table.'
)
#try:
self._get_tcwv(selected_img, vza, sza, raa, ele_data)
#except:
# self.s2_logger.warning('Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.')
# self.s2_tcwv = tcwv
# self.s2_tcwv_unc = np.ones(self.s2_tcwv.shape) * 0.2
self.s2_logger.info('Trying to get the aod from ddv method.')
try:
solved = self._get_ddv_aot(self.s2.angles, example_file,
self.s2_tcwv, ele_data, selected_img)
if solved[0] < 0:
self.s2_logger.warning(
'DDV failed and only cams data used for the prior.')
else:
self.s2_logger.info(
'DDV solved aod is %.02f, and it will used as the mean value of cams prediction.'
% solved[0])
self.s2_aod550 += (solved[0] - self.s2_aod550.mean())
except:
self.s2_logger.warning('Getting aod from ddv failed.')
self.s2_logger.info('Applying PSF model.')
if self.s2_psf is None:
self.s2_logger.info('No PSF parameters specified, start solving.')
high_img = np.repeat(
np.repeat(selected_img['B11'], 2, axis=0), 2, axis=1) * 0.0001
high_indexs = self.Hx, self.Hy
low_img = self.s2_boa[4]
qa, cloud = self.s2_boa_qa[4], self.s2.cloud
psf = psf_optimize(high_img, high_indexs, low_img, qa, cloud, 2)
xs, ys = psf.fire_shift_optimize()
xstd, ystd = 29.75, 39
ang = 0
self.s2_logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \
%(xstd, ystd, 0, xs, ys, 1-psf.costs.min()))
else:
xstd, ystd, ang, xs, ys = self.s2_psf
# apply psf shifts without going out of the image extend
shifted_mask = np.logical_and.reduce(
((self.Hx + int(xs) >= 0),
(self.Hx + int(xs) < self.s2_full_res[0]),
(self.Hy + int(ys) >= 0),
(self.Hy + int(ys) < self.s2_full_res[0])))
self.Hx, self.Hy = self.Hx[shifted_mask] + int(
xs), self.Hy[shifted_mask] + int(ys)
#self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
self.s2_boa = self.s2_boa[:, shifted_mask]
self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]
self.s2_angles = self.s2_angles[:, :, shifted_mask]
self.elevation = self.elevation[shifted_mask]
self.s2_aod550 = self.s2_aod550[shifted_mask]
self.s2_tcwv = self.s2_tcwv[shifted_mask]
self.s2_tco3 = self.s2_tco3[shifted_mask]
self.s2_aod550_unc = self.s2_aod550_unc[shifted_mask]
self.s2_tcwv_unc = self.s2_tcwv_unc[shifted_mask]
self.s2_tco3_unc = self.s2_tco3_unc[shifted_mask]
self.brdf_stds = self.brdf_stds[:, shifted_mask]
self.s2_logger.info('Getting the convolved TOA reflectance.')
self.valid_pixs = sum(
shifted_mask) # count how many pixels is still within the s2 tile
ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
self.bad_pixs = np.zeros(self.valid_pixs).astype(bool)
imgs = []
for i, band in enumerate(self.s2_u_bands[:-2]):
if selected_img[band].shape != self.s2_full_res:
selected_img[band] = self.repeat_extend(selected_img[band],
shape=self.s2_full_res)
else:
pass
selected_img[band][0, :] = -9999
selected_img[band][-1, :] = -9999
selected_img[band][:, 0] = -9999
selected_img[band][:, -1] = -9999
imgs.append(selected_img[band])
# filter out the bad pixels
self.bad_pixs |= cloud_dilation(self.s2.cloud |\
(selected_img[band] <= 0) | \
(selected_img[band] >= 10000),\
iteration= ker_size/2)[self.Hx, self.Hy]
del selected_img
del self.s2.selected_img
del high_img
del self.s2.angles
del self.s2.sza
del self.s2.saa
del self.s2
ker = self.gaussian(xstd, ystd, ang)
f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
.Hy] * 0.0001
half = parmap(f, imgs[:3])
self.s2_toa = np.array(half + parmap(f, imgs[3:]))
#self.s2_toa = np.array(parmap(f,imgs))
del imgs
# get the valid value masks
qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)
boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
np.all(self.s2_boa > 0, axis = 0) &\
np.all(self.s2_boa < 1, axis = 0)
toa_mask = (~self.bad_pixs) &\
np.all(self.s2_toa > 0, axis = 0) &\
np.all(self.s2_toa < 1, axis = 0)
self.s2_mask = boa_mask & toa_mask & qua_mask & (~self.elevation.mask)
self.s2_AEE, self.s2_bounds = self._load_emus(self.s2_sensor)