def _get_boa(self, ):
pix_mem = 180.
if 'jasmin_memory_limit' in os.environ:
jasmin_memory_limit = float(
os.environ['jasmin_memory_limit']) - 10000
else:
jasmin_memory_limit = psutil.virtual_memory().available + 10000
av_ram = min(psutil.virtual_memory().available - 10000,
jasmin_memory_limit)
needed = np.array(
[i.RasterXSize * i.RasterYSize * pix_mem for i in self._toa_bands])
u_need = np.unique(needed)
procs = av_ram / u_need
if av_ram > sum(needed):
#ret = parmap(self._do_band, range(len(self.toa_bands)))
self._chunks = 1
ret = parmap(self._do_chunk, range(len(self.toa_bands)))
else:
ret = []
index = []
for i, proc in enumerate(procs):
bands_to_do = np.arange(len(
self.toa_bands))[needed == u_need[i]]
if int(proc) >= 1:
self._chunks = 1
re = parmap(self._do_chunk, bands_to_do,
min(int(proc), len(bands_to_do)))
else:
self._chunks = int(np.ceil(1. / proc))
re = list(map(self._do_chunk, bands_to_do))
ret += re
index += (np.where(needed == u_need[i])[0]).tolist()
ret = list(zip(*sorted(zip(index, ret))))[1]
self.boa_rgb = ret[self.ri], ret[self.gi], ret[self.bi]
def _fill_nan(self, ):
self._vza = np.array(parmap(fill_nan, list(self._vza)))
self._vaa = np.array(parmap(fill_nan, list(self._vaa)))
self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3 = \
parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3])
self._aot = self._aot
self._aot = np.maximum(self._aot, 0.01)
def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None,
global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', \
cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', jasmin = False):
if not os.path.exists(file_path + '/emus/'):
os.mkdir(file_path + '/emus/')
if len(glob(file_path + '/emus/' +
'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12:
url = 'http://www2.geog.ucl.ac.uk/~ucfafyi/emus/'
req = requests.get(url)
to_down = []
for line in req.text.split():
if 'MSI' in line:
fname = line.split('"')[1].split('<')[0]
if 'MSI' in fname:
to_down.append([fname, url])
f = lambda fname_url: downloader(fname_url[0], fname_url[1], file_path
+ '/emus/')
parmap(f, to_down)
rets = s2_pre_processing(s2_t)
aero_atmos = []
for ret in rets:
ret += (mcd43, vrt_dir, aoi, global_dem, cams_dir, jasmin)
aero_atmo = do_correction(*ret)
if send_back:
aero_atmos.append(aero_atmo)
if send_back:
return aero_atmos
def SIAC_L8(l8_dir,
send_back=False,
mcd43=home + '/MCD43/',
vrt_dir=home + '/MCD43_VRT/',
aoi=None):
file_path = os.path.dirname(os.path.realpath(__file__))
if not os.path.exists(file_path + '/emus/'):
os.mkdir(file_path + '/emus/')
#print(file_path)
if len(glob(file_path + '/emus/' +
'isotropic_OLI_emulators_*_x?p_L8.pkl')) < 6:
url = 'http://www2.geog.ucl.ac.uk/~ucfafyi/emus/'
req = requests.get(url)
to_down = []
for line in req.text.split():
if 'OLI' in line:
fname = line.split('"')[1].split('<')[0]
if ('OLI' in fname) & ('L8' in fname):
to_down.append([fname, url])
f = lambda fname_url: downloader(fname_url[0], fname_url[1], file_path
+ '/emus/')
parmap(f, to_down)
rets = l8_pre_processing(l8_dir)
aero_atmos = []
for ret in rets:
ret += (mcd43, vrt_dir, aoi)
#sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret
aero_atmo = do_correction(*ret)
if send_back:
aero_atmos.append(aero_atmo)
if send_back:
return aero_atmos
def _fill_nan(self,):
self._vza = np.array(parmap(fill_nan, list(self._vza)))
self._vaa = np.array(parmap(fill_nan, list(self._vaa)))
self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc = \
parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc])
self._aot_unc = array_to_raster(self._aot_unc, self.example_file)
self._tcwv_unc = array_to_raster(self._tcwv_unc, self.example_file)
self._tco3_unc = array_to_raster(self._tco3_unc, self.example_file)
def _get_convolved_toa(self, ):
imgs = [band_g.ReadAsArray() for band_g in self._toa_bands]
self.bad_pixs = self.bad_pix[self.hx, self.hy]
if self.full_res[0] % 2 != 0:
xgaus = np.exp(-2. * (np.pi**2) * (self.psf_xstd**2) *
((0.5 * np.arange(self.full_res[0] + 1) /
(self.full_res[0] + 1))**2))
else:
xgaus = np.exp(
-2. * (np.pi**2) * (self.psf_xstd**2) *
((0.5 * np.arange(self.full_res[0]) / self.full_res[0])**2))
if self.full_res[1] % 2 != 0:
ygaus = np.exp(-2. * (np.pi**2) * (self.psf_ystd**2) *
((0.5 * np.arange(self.full_res[1] + 1) /
(self.full_res[1] + 1))**2))
else:
ygaus = np.exp(
-2. * (np.pi**2) * (self.psf_ystd**2) *
((0.5 * np.arange(self.full_res[1]) / self.full_res[1])**2))
gaus_2d = np.outer(xgaus, ygaus)
par = partial(convolve, gaus_2d=gaus_2d, hx=self.hx, hy=self.hy)
if np.array(self.ref_scale).ndim == 2:
self.ref_scale = self.ref_scale[self.hx, self.hy]
if np.array(self.ref_off).ndim == 2:
self.ref_off = self.ref_off[self.hx, self.hy]
self.toa = np.array(parmap(par, imgs)) * self.ref_scale + self.ref_off
def _fill_nan(self,):
def fill_nan(array):
x_shp, y_shp = array.shape
mask = ~np.isnan(array)
valid = np.array(np.where(mask)).T
value = array[mask]
mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \
np.tile (range(y_shp), x_shp).reshape(x_shp, y_shp)
array = griddata(valid, value, mesh, method='nearest')
return array
self._vza = np.array(parmap(fill_nan, list(self._vza)))
self._vaa = np.array(parmap(fill_nan, list(self._vaa)))
self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3 = \
parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3])
self._aot = self._aot
self._aot = np.maximum(self._aot, 0.01)
def _get_convolved_toa(self,):
imgs = [band_g.ReadAsArray() for band_g in self._toa_bands]
self.bad_pixs = self.bad_pix[self.hx, self.hy]
if self.full_res[0] %2 != 0:
xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0] + 1) /(self.full_res[0] + 1))**2))
else:
xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0]) /self.full_res[0])**2))
if self.full_res[1] %2 != 0:
ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1] + 1) /(self.full_res[1] + 1))**2))
else:
ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1]) /self.full_res[1])**2))
gaus_2d = np.outer(xgaus, ygaus)
def convolve(img, gaus_2d, hx, hy):
x_size, y_size = img.shape
if x_size % 2 != 0:
img = np.insert(img, -1, img[-1, :], axis=0)
if y_size % 2 != 0:
img = np.insert(img, -1, img[:, -1], axis=1)
dat = idct(idct(dct(dct(img, axis=0, norm = 'ortho'), axis=1, \
norm='ortho') * gaus_2d, axis=1, norm='ortho'), axis=0, norm='ortho')[hx, hy]
return dat
par = partial(convolve, gaus_2d = gaus_2d, hx = self.hx, hy = self.hy)
if np.array(self.ref_scale).ndim ==2:
self.ref_scale = self.ref_scale[self.hx, self.hy]
if np.array(self.ref_off).ndim == 2:
self.ref_off = self.ref_off[self.hx, self.hy]
self.toa = np.array(parmap(par,imgs)) * self.ref_scale+self.ref_off
def fire_shift_optimize(self, ):
#self.S2_PSF_optimization()
self._preprocess()
if self.lh_mask.sum() == 0:
self.costs = np.array([
100000000000.,
])
return 0, 0
min_val = [-50, -50]
max_val = [50, 50]
ps, distributions = create_training_set(['xs', 'ys'],
min_val,
max_val,
n_train=50)
self.shift_solved = parmap(self.shift_optimize, ps)
self.paras, self.costs = np.array([i[0] for i in self.shift_solved]), \
np.array([i[1] for i in self.shift_solved])
if (1 - self.costs.min()) >= 0.6:
xs, ys = self.paras[self.costs == np.nanmin(self.costs)][0].astype(
int)
else:
xs, ys = 0, 0
#print 'Best shift is ', xs, ys, 'with the correlation of', 1-self.costs.min()
return xs, ys
def fire_gaus_optimize(self, ):
xs, ys = self.fire_shift_optimize()
if self.costs.min() < 0.1:
min_val = [4, 4, -15, xs - 2, ys - 2]
max_val = [40, 40, 15, xs + 2, ys + 2]
self.bounds = [4, 40], [4, 40], [-15,
15], [xs - 2,
xs + 2], [ys - 2, ys + 2]
ps, distributions = create_training_set(self.parameters,
min_val,
max_val,
n_train=50)
print('Start solving...')
self.gaus_solved = parmap(self.gaus_optimize, ps, nprocs=5)
result = np.array(
[np.hstack((i[0], i[1])) for i in self.gaus_solved])
print(
'solved psf',
dict(
zip(self.parameters + [
'cost',
], result[np.argmin(result[:, -1])])))
return result[np.argmin(result[:, -1]), :]
else:
print('Cost is too large, plese check!')
return []
def _fill_nan(self, ):
def fill_nan(array):
x_shp, y_shp = array.shape
mask = ~np.isnan(array)
valid = np.array(np.where(mask)).T
value = array[mask]
mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \
np.tile (range(y_shp), x_shp).reshape(x_shp, y_shp)
array = griddata(valid, value, mesh, method='nearest')
return array
self._vza = np.array(parmap(fill_nan, list(self._vza)))
self._vaa = np.array(parmap(fill_nan, list(self._vaa)))
self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc = \
parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc])
self._aot_unc = array_to_raster(self._aot_unc, self.example_file)
self._tcwv_unc = array_to_raster(self._tcwv_unc, self.example_file)
self._tco3_unc = array_to_raster(self._tco3_unc, self.example_file)
def _load_xa_xb_xc_emus(self, ):
xap_emu = glob(self.emus_dir +
'/isotropic_%s_emulators_correction_xap_%s.pkl' %
(self.sensor, self.satellite))[0]
xbp_emu = glob(self.emus_dir +
'/isotropic_%s_emulators_correction_xbp_%s.pkl' %
(self.sensor, self.satellite))[0]
xcp_emu = glob(self.emus_dir +
'/isotropic_%s_emulators_correction_xcp_%s.pkl' %
(self.sensor, self.satellite))[0]
if sys.version_info >= (3, 0):
f = lambda em: pkl.load(open(em, 'rb'), encoding='latin1')
else:
f = lambda em: pkl.load(open(str(em), 'rb'))
self.emus = parmap(f, [xap_emu, xbp_emu, xcp_emu])
def _read_MCD43(self, fnames):
def warp_data(fname, aoi, xRes, yRes):
g = gdal.Warp('',fname, format = 'MEM', srcNodata = 32767, dstNodata=0, \
cutlineDSName=aoi, xRes = xRes, yRes = yRes, cropToCutline=True, resampleAlg = 0) # weird adaptation for gdal 2.3, this should be a bug in gdal 2.3
return g.ReadAsArray(
) # no reason you have to specify the srcNodata to use dstNodata
par = partial(warp_data,
aoi=self.aoi,
xRes=self.aero_res * 0.5,
yRes=self.aero_res * 0.5)
n_files = int(len(fnames) / 2)
ret = parmap(par, fnames)
das = np.array(ret[:n_files])
qas = np.array(ret[n_files:])
ws = 0.618034**qas
ws[qas == 255] = 0
das[das == 32767] = 0
return das, ws
def _get_convolved_toa(self, ):
imgs = [band_g.ReadAsArray() for band_g in self._toa_bands]
self.bad_pixs = self.bad_pix[self.hx, self.hy]
xgaus = np.exp(
-2. * (np.pi**2) * (self.psf_xstd**2) *
((0.5 * np.arange(self.full_res[0]) / self.full_res[0])**2))
ygaus = np.exp(
-2. * (np.pi**2) * (self.psf_ystd**2) *
((0.5 * np.arange(self.full_res[1]) / self.full_res[1])**2))
gaus_2d = np.outer(xgaus, ygaus)
def convolve(img, gaus_2d, hx, hy):
dat = idct(idct(dct(dct(img, axis=0, norm = 'ortho'), axis=1, \
norm='ortho') * gaus_2d, axis=1, norm='ortho'), axis=0, norm='ortho')[hx, hy]
return dat
par = partial(convolve, gaus_2d=gaus_2d, hx=self.hx, hy=self.hy)
if np.array(self.ref_scale).ndim == 2:
self.ref_scale = self.ref_scale[self.hx, self.hy]
if np.array(self.ref_off).ndim == 2:
self.ref_off = self.ref_off[self.hx, self.hy]
self.toa = np.array(parmap(par, imgs)) * self.ref_scale + self.ref_off
def _solving(self, ):
self.logger.propagate = False
self.logger.info('Set AOI.')
self._create_base_map()
self.logger.info('Get corresponding bands.')
self._find_boa_bands()
self.logger.info('Slice TOA bands based on AOI.')
self._create_band_gs()
self._resamplers()
self.logger.info('Parsing angles.')
self._parse_angles()
self.logger.info('Mask bad pixeles.')
self._mask_bad_pix()
if np.sum(~self.bad_pix) > 10:
self.logger.info('Get simulated BOA.')
self._get_boa()
self.logger.info('Get PSF.')
self._get_psf()
self.logger.info('Get simulated TOA reflectance.')
self._get_convolved_toa()
self.logger.info('Filtering data.')
self._re_mask()
if self.mask is not False:
self.logger.info('Loading emulators.')
self._load_xa_xb_xc_emus()
self.logger.info('Reading priors and elevation.')
self._read_aux()
self._fill_nan()
self.logger.info(
'Mean values for prior AOT: %.02f and TCWV: %.02f' %
(self._aot.mean(), self._tcwv.mean()))
if self.mask.sum() == 0:
self.logger.info(
'No valid value is found for retrieval of atmospheric parameters and priors are stored.'
)
ret = np.array(
[[self._aot, self._tcwv, self._tco3],
[self._aot_unc, self._tcwv_unc, self._tco3_unc]])
self.aero_res /= 2
#self.ySize *=2
#self.xSize *=2
self.ySize, self.xSize = self._aot.shape
else:
self.aero = solving_atmo_paras(self.boa,
self.toa,
self._sza,
self._vza,
self._saa,
self._vaa,
self._aot,
self._tcwv,
self._tco3,
self._ele,
self._aot_unc,
self._tcwv_unc,
self._tco3_unc,
self.boa_unc,
self.hx,
self.hy,
self.mask,
self.full_res,
self.aero_res,
self.emus,
self.band_index,
self.boa_wv,
pix_res=self.pixel_res,
gamma=self.gamma,
log_file=self.log_file)
ret = self.aero._multi_grid_solver()
else:
self.logger.info(
'No valid value is found for retrieval of atmospheric parameters and priors are stored.'
)
self._read_aux()
self._fill_nan()
self.logger.info(
'Mean values for prior AOT: %.02f and TCWV: %.02f' %
(self._aot.mean(), self._tcwv.mean()))
ret = np.array(
[[self._aot, self._tcwv, self._tco3],
[self._aot_unc, self._tcwv_unc, self._tco3_unc]])
self.aero_res /= 2
#self.ySize *=2
#self.xSize *=2
self.ySize, self.xSize = self._aot.shape
else:
self.logger.info(
'No valid value is found for retrieval of atmospheric parameters and priors are stored.'
)
self._read_aux()
self._fill_nan()
self.logger.info(
'Mean values for prior AOT: %.02f and TCWV: %.02f' %
(self._aot.mean(), self._tcwv.mean()))
ret = np.array([[self._aot, self._tcwv, self._tco3],
[self._aot_unc, self._tcwv_unc, self._tco3_unc]])
self.aero_res /= 2
self.ySize, self.xSize = self._aot.shape
solved = ret[0].reshape(3, self.ySize, self.xSize)
unc = ret[1].reshape(3, self.ySize, self.xSize)
self.logger.info(
'Finished retrieval and saving them into local files.')
para_names = 'aot', 'tcwv', 'tco3', 'aot_unc', 'tcwv_unc', 'tco3_unc'
toa_dir = self.toa_dir + '/' + 'B'.join(
self.toa_bands[0].split('/')[-1].split('B')[:-1])
name_arrays = zip(para_names, list(solved) + list(unc))
par = partial(save_posterior,
g=self.example_file,
aero_res=self.aero_res,
toa_dir=toa_dir)
parmap(par, name_arrays)
self.post_aot, self.post_tcwv, self.post_tco3, = solved
self.post_aot_unc, self.post_tcwv_unc, self.post_tco3_unc = unc
handlers = self.logger.handlers[:]
for handler in handlers:
handler.close()
self.logger.removeHandler(handler)
aot_unc[:] = 0.5
tcwv_unc[:] = 0.2
tco3_unc[:] = 0.2
toa = np.random.rand(6, 50000)
y = toa * 2.639794 - 0.038705
boa = y / (1 + 0.068196 * y)
boa_unc = np.ones(50000) * 0.05
Hx = np.random.choice(10980, 50000)
Hy = np.random.choice(10980, 50000)
full_res = (10980, 10980)
aero_res = 600
emus_dir = '~/DATA/Multiply/emus/'
sensor = 'MSI'
xap_emu = glob(emus_dir + '/isotropic_%s_emulators_*xap*.pkl' %
(sensor))[0]
xbp_emu = glob(emus_dir + '/isotropic_%s_emulators_*xbp*.pkl' %
(sensor))[0]
xcp_emu = glob(emus_dir + '/isotropic_%s_emulators_*xcp*.pkl' %
(sensor))[0]
f = lambda em: pkl.load(open(em, 'rb'))
emus = parmap(f, [xap_emu, xbp_emu, xcp_emu])
band_indexs = [1, 2, 3, 7, 11, 12]
band_wavelength = [469, 555, 645, 869, 1640, 2130]
mask = np.zeros((10980, 10980)).astype(bool)
mask[1, 1] = True
aero = solving_atmo_paras(boa, toa, sza, vza, saa, vaa, aot, tcwv, tco3,
ele, aot_unc, tcwv_unc, tco3_unc, boa_unc, Hx,
Hy, mask, full_res, aero_res, emus, band_indexs,
band_wavelength)
solved = aero._multi_grid_solver()
def _obs_cost_test(self, p, is_full=True, do_unc=False):
p = np.array(p).reshape(2, -1)
X = self.control_variables.reshape(self.boa.shape[0], 7, -1)
X[:, 3:5, :] = np.array(p)
xap_H, xbp_H, xcp_H = [], [], []
xap_dH, xbp_dH, xcp_dH = [], [], []
emus = list(self.xap_emus) + list(self.xbp_emus) + list(self.xcp_emus)
Xs = list(X) + list(X) + list(X)
inps = list(zip(emus, Xs))
if self._coarse_mask.sum() > 1000:
ret = np.array(parmap(self._helper, inps))
else:
ret = np.array(list(map(self._helper, inps)))
xap_H, xbp_H, xcp_H = ret[:, :, 0].reshape(3, self.boa.shape[0],
len(self.Hx))
xap_dH, xbp_dH, xcp_dH = ret[:, :, 1:].reshape(3, self.boa.shape[0],
len(self.Hx), 2)
y = xap_H * self.toa - xbp_H
sur_ref = y / (1 + xcp_H * y)
diff = sur_ref - self.boa
full_J = np.nansum(0.5 * self.band_weights[..., None] * (diff)**2 /
self.boa_unc**2,
axis=0)
J = np.zeros(self.full_res)
dH = -1 * (-self.toa[...,None] * xap_dH - \
2 * self.toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \
self.toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \
xbp_dH + \
xbp_H[...,None]**2 * xcp_dH) / \
(self.toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \
xbp_H[...,None] * xcp_H[...,None] + 1)**2
full_dJ = [
self.band_weights[..., None] * dH[:, :, i] * diff /
(self.boa_unc**2) for i in range(2)
]
if is_full:
dJ = np.nansum(np.array(full_dJ), axis=(1, ))
dJ[np.isnan(dJ)] = 0
full_J[np.isnan(full_J)] = 0
#J_ = np.zeros((2,) + self.full_res)
#J_[:, self.Hx, self.Hy] = dJ
#subs1 = [np.array_split(sub, self.num_blocks_y, axis=2) for sub in np.array_split(J_, self.num_blocks_x, axis=1)]
#J = np.zeros(self.full_res)
#J[self.Hx, self.Hy] = full_J
#subs2 = [np.array_split(sub, self.num_blocks_y, axis=1) for sub in np.array_split(J, self.num_blocks_x, axis=0)]
#J_ = np.zeros((2, self.num_blocks_x, self.num_blocks_y))
#J = np.zeros(( self.num_blocks_x, self.num_blocks_y))
nx, ny = (np.ceil(np.array(self.full_res) / np.array([self.num_blocks_x, self.num_blocks_y])) \
* np.array([self.num_blocks_x, self.num_blocks_y])).astype(int)
#end_x, end_y = np.array(self.full_res) - np.array([nx, ny])
x_size, y_size = int(nx / self.num_blocks_x), int(
ny / self.num_blocks_y)
J_ = np.zeros((2, nx, ny))
J = np.zeros((nx, ny))
#J_[:], J[:] = np.nan, np.nan
J_[:, self.Hx, self.Hy] = dJ
J[self.Hx, self.Hy] = full_J
J_ = J_.reshape(2, self.num_blocks_x, x_size, self.num_blocks_y,
y_size)
J = J.reshape(self.num_blocks_x, x_size, self.num_blocks_y, y_size)
J_ = np.sum(J_, axis=(2, 4))
J = np.sum(J, axis=(1, 3))
#for i in range(self.num_blocks_x):
# for j in range(self.num_blocks_y):
# J_[:, i,j] = np.nansum(subs1[i][j], axis=(1,2))
# J [ i,j] = np.nansum(subs2[i][j], axis=(0,1))
J_[:, ~self._coarse_mask] = 0
J[~self._coarse_mask] = 0
J_ = J_.reshape(2, -1)
if do_unc:
#comb_unc = np.nansum([self.band_weights[...,None] * (dH[:, :, i] ** 2) * (self.boa_unc ** -2) for i in range(2)], axis = 1)
#comb_unc[comb_unc==0] = np.nan
#self.obs_unc = np.zeros((2,) + self.full_res)
#self.obs_unc[:] = np.nan
#self.obs_unc[:, self.Hx, self.Hy] = comb_unc
comb_unc = np.nansum([
self.band_weights[..., None] * (dH[:, :, i]**2) *
(self.boa_unc**-2) for i in range(2)
],
axis=1)
comb_unc[comb_unc == 0] = np.nan
self.obs_unc = np.zeros((2, nx, ny))
self.obs_unc[:] = np.nan
self.obs_unc[:, self.Hx, self.Hy] = comb_unc
self.obs_unc = self.obs_unc.reshape(2, self.num_blocks_x,
x_size, self.num_blocks_y,
y_size)
self.obs_unc = np.nanmean(self.obs_unc, axis=(2, 4))
#subs = [np.array_split(sub, self.num_blocks_y, axis=2) for sub in np.array_split(self.obs_unc, self.num_blocks_x, axis=1)]
#self.obs_unc = np.zeros((2, self.num_blocks_x, self.num_blocks_y))
#for i in range(self.num_blocks_x):
# for j in range(self.num_blocks_y):
# self.obs_unc[:, i,j] = np.nanmean(subs[i][j], axis=(1,2))
self.obs_unc[:, ~self._coarse_mask] = np.nan
return self.obs_unc
else:
J = np.nansum(np.array(full_J))
J_ = np.nansum(np.array(full_dJ), axis=(1, 2))
return J, J_
def _get_boa(self, temporal_filling=16):
qa_temp = 'MCD43_%s_BRDF_Albedo_Band_Mandatory_Quality_Band%d.vrt'
da_temp = 'MCD43_%s_BRDF_Albedo_Parameters_Band%d.vrt'
doy = self.obs_time.strftime('%Y%j')
if temporal_filling == True:
temporal_filling = 16
if temporal_filling:
days = [(self.obs_time - timedelta(days = int(i))) for i in np.arange(temporal_filling, 0, -1)] + \
[(self.obs_time + timedelta(days = int(i))) for i in np.arange(0, temporal_filling+1, 1)]
fnames = []
for temp in [da_temp, qa_temp]:
for day in days:
for band in self.boa_bands:
fname = self.mcd43_dir + '/'.join([
day.strftime('%Y-%m-%d'), temp %
(day.strftime('%Y%j'), band)
])
fnames.append(fname)
else:
fnames = []
for temp in [da_temp, qa_temp]:
for band in self.boa_bands:
fname = self.MCD43_dir + '/'.join([
datetime.strftime(self.obs_time, '%Y-%m-%d'), temp %
(doy, band)
])
fnames.append(fname)
das, ws = self._read_MCD43(fnames)
mg = gdal.Warp('',fnames[0], format = 'MEM', dstNodata= None, xRes = self.aero_res*0.5, yRes = \
self.aero_res*0.5, cutlineDSName=self.aoi, cropToCutline=True, resampleAlg = 0)
hg = self.example_file
self.hx, self.hy, hmask, rmask = self._get_index(mg, hg)
No_band = len(self.toa_bands)
mask = ~(mg.ReadAsArray()[0] == 0)
self._annoying_angles(mg)
sza = np.repeat(self._sza[None, ...], len(self._vza),
axis=0)[:, mask][:, hmask][:, rmask]
saa = np.repeat(self._saa[None, ...], len(self._vza),
axis=0)[:, mask][:, hmask][:, rmask]
angles = self._vza[:,
mask][:,
hmask][:,
rmask], sza, self._vaa[:,
mask][:,
hmask][:,
rmask] - saa
kk = get_kk(angles)
k_vol = kk.Ross
k_geo = kk.Li
kers = np.array([np.ones(k_vol.shape), k_vol, k_geo])
surs = []
for i in range(No_band):
surs.append(
(das[i::No_band][:, :, mask][:, :, hmask][:, :, rmask] *
kers[:, i] * 0.001).sum(axis=1))
if temporal_filling:
def smooth(da_w):
da, w = da_w
mid = int(da.shape[0] / 2)
if (da.shape[-1] == 0) | (w.shape[-1] == 0):
return da[mid], w[mid]
data = np.array(
smoothn(da,
s=10.,
smoothOrder=1.,
axis=0,
TolZ=0.001,
verbose=False,
isrobust=True,
W=w))[[0, 3], ]
return data[0][mid], data[1][mid]
boa = []
w = []
for i in range(No_band):
das = surs[i]
Ws = ws[i::No_band][:, mask][:, hmask][:, rmask]
chunks = zip(np.array_split(das, 18, axis=1),
np.array_split(Ws, 18, axis=1))
ret = parmap(smooth, chunks)
_b = np.hstack([i[0] for i in ret])
_w = np.hstack([i[1] for i in ret])
boa.append(_b)
w.append(_w)
boa = np.array(boa)
w = np.array(w)
unc = 0.015 / w
else:
boa = np.array(surs)
unc = 0.015 / ws
self.boa = boa
self.boa_unc = np.minimum(unc, 0.5)
