def _parse_atmo(self,):
atmos = [self.aot, self.tcwv, self.tco3]
atmo_uncs = [self.aot_unc, self.tcwv_unc, self.tco3_unc]
atmo_scale = self.atmo_scale
cams_names = ['aod550', 'tcwv', 'gtco3']
defalt_uncs = [0.4, 0.1, 0.05]
if self.obs_time is not None:
time_ind = np.abs((self.obs_time.hour + self.obs_time.minute/60. + \
self.obs_time.second/3600.) - np.arange(0,25,3)).argmin()
for i in range(3):
if atmos[i] is None:
atmos[i] = self.cams_dir + '/'.join([datetime.strftime(self.obs_time, '%Y_%m_%d'),\
datetime.strftime(self.obs_time, '%Y_%m_%d')+'_%s.tif'%cams_names[i]])
var_g = self._var_parser(atmos[i])
_g = reproject_data(var_g, self.mg, srcNodata = None, resample = \
gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).g
offset = var_g.GetOffset()
scale = var_g.GetScale()
data = _g.GetRasterBand(int(time_ind+1)).ReadAsArray() * scale + offset
atmos[i] = data * self.cams_scale[i]
if atmo_uncs[i] is None:
atmo_uncs[i] = defalt_uncs[i]
else:
for i in range(3):
var_g = self._var_parser(atmos[i])
data = reproject_data(var_g, self.mg, srcNodata = np.nan, resample = \
gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
atmos[i] = data * self.atmo_scale[i]
unc_g = self._var_parser(atmo_uncs[i])
ug = reproject_data(unc_g, self.mg, srcNodata = np.nan, resample = \
gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
atmo_uncs[i] = ug
self._aot, self._tcwv, self._tco3 = atmos
self._aot_unc, self._tcwv_unc, self._tco3_unc = atmo_uncs
def get_control_variables(self, ):
aod = reproject_data(self.mod_l1b_dir+'/atmo_paras/' + \
self.example_file.split('/')[-1].split('_EV_')[0]+'_EV_aod550.tif', self.example_file).data
tcwv = reproject_data(self.mod_l1b_dir+'/atmo_paras/' + \
self.example_file.split('/')[-1].split('_EV_')[0]+'_EV_tcwv.tif', self.example_file).data
tco3 = reproject_data(self.mod_l1b_dir+'/atmo_paras/' +\
self.example_file.split('/')[-1].split('_EV_')[0]+'_EV_tco3.tif', self.example_file).data
ele = reproject_data(self.global_dem, self.example_file).data
mask = ~np.isfinite(ele)
if mask.sum() > 0:
ele[mask] = np.interp(np.flatnonzero(mask), \
np.flatnonzero(~mask), ele[~mask]) # simple interpolation
return aod, tcwv, tco3, ele
def do_cloud(cloud_bands, cloud_name=None):
toas = [
reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data
for band in cloud_bands
]
toas = np.array(toas) / 10000.
mask = np.all(toas >= 0.0001, axis=0)
valid_pixel = mask.sum()
if valid_pixel > 0:
cloud_proba = cl.predict_proba(toas[:, mask].T)
cloud_mask = np.zeros_like(toas[0])
cloud_mask[mask] = cloud_proba[:, 1]
else:
cloud_mask = np.zeros_like(toas[0])
cloud_mask[~mask] = 2.56
if cloud_name is None:
return cloud_mask
else:
g = gdal.Open(cloud_bands[0])
dst = gdal.GetDriverByName('GTiff').Create(
cloud_name,
g.RasterXSize,
g.RasterYSize,
1,
gdal.GDT_Byte,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((cloud_mask * 100).astype(int))
dst.GetRasterBand(1).SetNoDataValue(256)
dst = None
g = None
return cloud_mask
def _read_cams(self, example_file, parameters=['aod550', 'tcwv', 'gtco3']):
netcdf_file = datetime.datetime(self.sen_time.year, self.sen_time.month, \
self.sen_time.day).strftime("%Y-%m-%d.nc")
template = 'NETCDF:"%s":%s'
ind = np.abs((self.sen_time.hour + self.sen_time.minute/60. + \
self.sen_time.second/3600.) - np.arange(0,25,3)).argmin()
sr = osr.SpatialReference()
sr.ImportFromEPSG(4326)
proj = sr.ExportToWkt()
results = []
for para in parameters:
fname = template % (self.cams_dir + '/' + netcdf_file, para)
g = gdal.Open(fname)
g.SetProjection(proj)
sub = g.GetRasterBand(ind + 1)
offset = sub.GetOffset()
scale = sub.GetScale()
bad_pix = int(sub.GetNoDataValue())
rep_g = reproject_data(g,
example_file,
outputType=gdal.GDT_Float32).g
data = rep_g.GetRasterBand(ind + 1).ReadAsArray()
data = data * scale + offset
mask = (data == (bad_pix * scale + offset))
if mask.sum() >= 1:
data[mask] = np.interp(np.flatnonzero(mask),
np.flatnonzero(~mask), data[~mask])
results.append(data)
return results
def _get_control_variables(self, ):
aot = reproject_data(self.l8_toa_dir + '/%s_%s.tif'%(self.l8_header, 'aot'), \
self.example_file, outputType= gdal.GDT_Float32).data
tcwv = reproject_data(self.l8_toa_dir + '/%s_%s.tif'%(self.l8_header, 'tcwv'), \
self.example_file, outputType= gdal.GDT_Float32).data
tco3 = reproject_data(self.l8_toa_dir + '/%s_%s.tif'%(self.l8_header, 'tco3'), \
self.example_file, outputType= gdal.GDT_Float32).data
ele = reproject_data(self.global_dem,
self.example_file,
outputType=gdal.GDT_Float32).data
mask = ~np.isfinite(ele)
ele[mask] = np.interp(np.flatnonzero(mask), \
np.flatnonzero(~mask), ele[~mask]) # simple interpolation
return aot, tcwv, tco3, ele
def get_control_variables(self, target_band):
aod = reproject_data(self.s2.s2_file_dir+'/aot.tif', \
self.s2.s2_file_dir+'/%s.jp2'%target_band, outputType= gdal.GDT_Float32).data
tcwv = reproject_data(self.s2.s2_file_dir+'/tcwv.tif', \
self.s2.s2_file_dir+'/%s.jp2'%target_band, outputType= gdal.GDT_Float32).data
tco3 = reproject_data(self.s2.s2_file_dir+'/tco3.tif', \
self.s2.s2_file_dir+'/%s.jp2'%target_band, outputType= gdal.GDT_Float32).data
ele = reproject_data(self.global_dem,
self.s2.s2_file_dir + '/%s.jp2' % target_band,
outputType=gdal.GDT_Float32).data
mask = ~np.isfinite(ele)
ele[mask] = np.interp(np.flatnonzero(mask), \
np.flatnonzero(~mask), ele[~mask]) # simple interpolation
return aod, tcwv, tco3, ele.astype(float)
def _resamplers(self, ):
'''
Define the sresamplers used for resampling different gdal files or objects to required resolution and size.
'''
self.nearest_resampler = lambda fname: reproject_data(fname, \
self.example_file, \
xRes=self.aero_res*0.5, \
yRes=self.aero_res*0.5, \
srcNodata = None,\
outputType= gdal.GDT_Float32, \
resample = gdal.GRIORA_NearestNeighbour).g
self.bilinear_resampler = lambda fname: reproject_data(fname, \
self.example_file, \
xRes=self.aero_res*0.5, \
yRes=self.aero_res*0.5, \
srcNodata = 0,\
outputType= gdal.GDT_Float32,\
resample = gdal.GRIORA_Bilinear).g
def _mcd08_aot(self,):
temp = 'HDF4_EOS:EOS_GRID:"%s":mod08:Aerosol_Optical_Depth_Land_Ocean_Mean'
try:
g = gdal.Open(temp%glob('%s/MOD08_D3.A2016%03d.006.*.hdf'%(self.mod08_dir, self.doy))[0])
dat = reproject_data(g, self.example_file, outputType= gdal.GDT_Float32).data * g.GetRasterBand(1).GetScale() + g.GetRasterBand(1).GetOffset()
dat[dat<=0] = np.nan
dat[dat>1.7] = np.nan
mod08_aot = np.nanmean(dat)
except:
mod08_aot = np.nan
try:
g1 = gdal.Open(temp%glob('%s/MYD08_D3.A2016%03d.006.*.hdf'%(self.myd08_dir, self.doy))[0])
dat1 = reproject_data(g1, self.example_file, outputType= gdal.GDT_Float32).data * g1.GetRasterBand(1).GetScale() + g1.GetRasterBand(1).GetOffset()
dat1[dat1<=0] = np.nan
dat1[dat1>1.7] = np.nan
myd08_aot = np.nanmean(dat1)
except:
myd08_aot = np.nan
return mod08_aot, myd08_aot
def _parse_aux(self,):
auxs = [self.global_dem, self.mask.astype(float)]
scales = [self.ele_scale, 1]
for i in range(len(auxs)):
var_g = self._var_parser(auxs[i])
dat = reproject_data(var_g, self.mg, srcNodata = 0, resample = \
gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
auxs[i] = dat * scales[i]
self._ele, self._cmask = auxs
self._ele[np.isnan(self._ele)] = 0
self._cmask[np.isnan(self._cmask)] = 0
self._cmask = binary_dilation(self._cmask, structure = np.ones((3,3)).astype(bool)).astype(bool)
def _annoying_angles(self,):
_sun_angles = []
_view_angles = []
for fname in self._sun_angles:
ang = reproject_data(fname, self.mg, srcNodata = None, resample = \
gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_sun_angles.append(ang)
for fname in self._view_angles:
ang = reproject_data(fname, self.mg, srcNodata = None, resample = \
gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_view_angles.append(ang)
_view_angles = np.array(_view_angles)
_sun_angles = np.squeeze(np.array(_sun_angles))
if len(self._view_angles) == len(self.toa_bands):
if self.a_z_order == 1:
self._vaa = _view_angles[:,0]
self._vza = _view_angles[:,1]
else:
self._vaa = _view_angles[:,1]
self._vza = _view_angles[:,0]
elif len(self._view_angles) == 2*len(self.toa_bands):
self._vaa = _view_angles[:len(self.toa_bands)]
self._vza = _view_angles[len(self.toa_bands):]
if os.path.exists(str(self.sun_angles)):
if self.a_z_order == 1:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
else:
self._saa = _sun_angles[1]
self._sza = _sun_angles[0]
elif len(self.sun_angles) == 2:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
self._sza = self._sza * self.ang_scale
self._saa = self._saa * self.ang_scale
self._vza = self._vza * self.ang_scale
self._vaa = self._vaa * self.ang_scale
self._vaa, self._vza, self._saa, self._sza = map(np.deg2rad, [self._vaa, self._vza, self._saa, self._sza])
def _mod08_aot(self, ):
try:
temp = 'HDF4_EOS:EOS_GRID:"%s":mod08:Aerosol_Optical_Depth_Land_Ocean_Mean'
g = gdal.Open(temp % glob('%s/MOD08_D3.A2016%03d.006.*.hdf' %
(self.mod08_dir, self.doy))[0])
dat = reproject_data(
g, self.s2_file_dir + '/B01.jp2',
outputType=gdal.GDT_Float32).data * g.GetRasterBand(
1).GetScale() + g.GetRasterBand(1).GetOffset()
dat[dat <= 0] = np.nan
dat[dat > 1.5] = np.nan
mod08_aot = np.nanmean(dat)
except:
mod08_aot = np.nan
return mod08_aot
def _mcd43_cloud(self, flist, lx, ly, example_file, boa, b12):
g = gdal.BuildVRT('', list(flist))
temp_data = np.zeros((g.RasterYSize, g.RasterXSize))
temp_data[lx, ly] = boa[5, :]
self.boa_b12 = reproject_data(array_to_raster(temp_data, g),
example_file,
outputType=gdal.GDT_Float32).data
toa_b12 = np.repeat(np.repeat(b12 / 10000., 2, axis=0), 2, axis=1)
mask = abs(self.boa_b12 - toa_b12) > 0.1
emask = binary_erosion(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=15)
dmask = binary_dilation(emask,
structure=np.ones((3, 3)).astype(bool),
iterations=100)
self.cloud = self.s2.cloud | dmask
def _modis_aerosol(self, ):
self.modis_logger.info('Getting emualated surface reflectance.')
vza, sza, vaa, saa = self.modis_angle
self.modis_boa, self.modis_boa_qa, self.brdf_stds = get_brdf_six(
self.mcd43_file,
angles=[vza, sza, vaa - saa],
bands=(1, 2, 3, 4, 5, 6, 7),
Linds=None)
if self.mod_cloud is None:
self.modis_cloud = np.zeros_like(self.modis_toa[0]).astype(bool)
self.modis_logger.info('Getting elevation.')
ele = reproject_data(self.global_dem, self.example_file)
ele.get_it()
mask = ~np.isfinite(ele.data)
ele.data = np.ma.array(ele.data, mask=mask)
self.elevation = ele.data / 1000.
self.modis_logger.info('Getting pripors from ECMWF forcasts.')
aod, tcwv, tco3 = self._read_cams(self.example_file)
self.aod550 = aod * (1 - 0.14) # validation of +14% biase
self.tco3 = tco3 * 46.698 * (1 - 0.05)
self.tcwv = tcwv / 10.
self.tco3_unc = np.ones(self.tco3.shape) * 0.2
self.aod550_unc = np.ones(self.aod550.shape) * 0.5
self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2
qua_mask = np.all(self.modis_boa_qa <= self.qa_thresh, axis=0)
boa_mask = np.all(~self.modis_boa.mask, axis = 0) &\
np.all(self.modis_boa>0, axis=0) &\
np.all(self.modis_boa<1, axis=0)
toa_mask = np.all(np.isfinite(self.modis_toa), axis=0) &\
np.all(self.modis_toa>0, axis=0) & \
np.all(self.modis_toa<1, axis=0)
self.modis_mask = qua_mask & boa_mask & toa_mask & (~self.modis_cloud)
self.modis_AEE, self.modis_bounds = self._load_emus(self.modis_sensor)
self.modis_solved = []
def _do_chunk(self, ind):
toa_g = self._toa_bands[ind]
geo = toa_g.GetGeoTransform()
x_size, y_size = toa_g.RasterXSize, toa_g.RasterYSize
boas = []
uncs = []
for i in range(self._chunks):
r = i * 1. / self._chunks
x_start = int(r * x_size)
xmin = geo[0] + r * x_size * geo[1]
r = (i+1.)/ self._chunks
x_end = int(r * x_size)
xmax = geo[0] + r * x_size * geo[1]
x_off = x_end - x_start
toa = toa_g.ReadAsArray(x_start, 0, x_off, int(y_size))
_g = gdal.Warp('', toa_g, format = 'MEM', outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = gdal.GRIORA_NearestNeighbour)
xRes = yRes = int(geo[1])
xps = [self.xap_H[ind], self.xbp_H[ind], self.xcp_H[ind]]
for j in range(3):
xps[j] = reproject_data(xps[j], _g,xRes=xRes, yRes = yRes,xSize=x_off, \
ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, outputType= gdal.GDT_Float32).data
xap_H, xbp_H, xcp_H = xps
toa = toa * self.ref_scale + self.ref_off
y = xap_H * toa - xbp_H
boa = y / (1 + xcp_H * y)
xhs = [self.xap_dH[ind], self.xbp_dH[ind], self.xcp_dH[ind]]
for j in range(3):
#var_g = xhs[j]
xhs[j] = reproject_data(xhs[j], _g, xRes=xRes, yRes = yRes, xSize=x_off, \
ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, \
outputType= gdal.GDT_Float32).data.transpose(1,2,0)
xap_dH, xbp_dH, xcp_dH = xhs
del xps; del xhs
dH = -1 * (-toa[...,None] * xap_dH - \
2 * toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \
toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \
xbp_dH + \
xbp_H[...,None]**2 * xcp_dH) / \
(toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \
xbp_H[...,None] * xcp_H[...,None] + 1)**2
toa_dH = xap_H / (xcp_H*(toa * xap_H - xbp_H) + 1)**2
del xap_H; del xbp_H; del xcp_H; del xap_dH; del xbp_dH; del xcp_dH
aot_dH, tcwv_dH, tco3_dH = [dH[:, :,i] for i in range(3)]
del dH
tmp = []
for unc_g in [self._aot_unc, self._tcwv_unc, self._tco3_unc]:
unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, outputType= gdal.GDT_Float32).data
tmp.append(unc)
aot_unc, tcwv_unc, tco3_unc = tmp; del tmp
unc = np.sqrt(aot_dH ** 2 * aot_unc**2 + tcwv_dH ** 2 * tcwv_unc**2 + tco3_dH ** 2 * tco3_unc**2 + toa_dH**2 * 0.015**2)
del aot_unc; del tcwv_unc; del tco3_unc
boas.append(boa)
uncs.append(unc)
boas, uncs = np.hstack(boas), np.hstack(uncs)
boa_name = self.toa_dir + '/' + '.'.join(self.toa_bands[ind].split('/')[-1].split('.')[0:-1]) + '_sur.tif'
unc_name = boa_name.replace('_sur.tif', '_sur_unc.tif')
projectionRef = toa_g.GetProjectionRef()
geotransform = toa_g.GetGeoTransform()
self._save_band(boas, boa_name, projectionRef, geotransform)
self._save_band(uncs, unc_name, projectionRef, geotransform)
if ind in [self.ri, self.gi, self.bi]:
return boas
else:
return None
def _s2_aerosol(self, ):
self.s2_logger.propagate = False
self.s2_logger.info('Start to retrieve atmospheric parameters.')
self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
self.day, self.s2_u_bands)
self.s2_logger.info('Reading in TOA reflectance.')
selected_img = self.s2.get_s2_toa()
self.s2_file_dir = self.s2.s2_file_dir
self.example_file = self.s2.s2_file_dir + '/B04.jp2'
self.s2.get_s2_cloud()
self.s2_logger.info('Loading emulators.')
self._load_xa_xb_xc_emus()
self.s2_logger.info(
'Getting the angles and simulated surface reflectance.')
self.s2.get_s2_angles()
sa_files = [self.s2.angles['saa'], self.s2.angles['sza']]
if os.path.exists(self.s2_file_dir + '/angles/'):
va_files = [
self.s2_file_dir + '/angles/VAA_VZA_B%02d.img' % i
for i in [2, 3, 4, 8, 11, 12]
]
else:
va_files = [
np.array(
[self.s2.angles['vaa'][band], self.s2.angles['vza'][band]])
for band in self.s2_u_bands[:-2]
]
if len(glob(self.s2_file_dir + '/MCD43.npz')) == 0:
boa, unc, hx, hy, lx, ly, flist = MCD43_SurRef(self.mcd43_dir, self.example_file, \
self.year, self.doy, [sa_files, va_files],
sun_view_ang_scale=[1, 0.01], bands = [3,4,1,2,6,7], tolz=0.001, reproject=False)
np.savez(self.s2_file_dir + '/MCD43.npz',
boa=boa,
unc=unc,
hx=hx,
hy=hy,
lx=lx,
ly=ly,
flist=flist)
else:
f = np.load(self.s2_file_dir + '/MCD43.npz')
boa, unc, hx, hy, lx, ly, flist = f['boa'], f['unc'], f['hx'], f[
'hy'], f['lx'], f['ly'], f['flist']
self.Hx, self.Hy = hx, hy
self.s2_logger.info('Update cloud mask.')
flist = [
f.split('/')[0] + self.mcd43_dir + '/' + f.split('/')[-1]
for f in flist
]
self._mcd43_cloud(flist, lx, ly, self.example_file, boa,
selected_img['B12'])
self.s2_logger.info('Applying spectral transform.')
self.s2_boa_qa = np.ma.array(unc)
self.s2_boa = np.ma.array(boa)*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
np.array(self.s2_spectral_transform)[1,:-1][...,None]
shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \
self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks)
self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1))
self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1))
self.vza = []
self.vaa = []
for band in self.s2_u_bands[:-2]:
self.vza.append(
self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1)))
self.vaa.append(
self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1)))
self.vza = np.array(self.vza)
self.vaa = np.array(self.vaa)
self.raa = self.saa[None, ...] - self.vaa
self.s2_logger.info('Getting elevation.')
ele_data = reproject_data(self.global_dem,
self.example_file,
outputType=gdal.GDT_Float32).data
mask = ~np.isfinite(ele_data)
ele_data = np.ma.array(ele_data, mask=mask) / 1000.
self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \
self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1))
self.s2_logger.info('Getting pripors from ECMWF forcasts.')
sen_time_str = json.load(
open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
self.sen_time = datetime.datetime.strptime(sen_time_str,
u'%Y-%m-%dT%H:%M:%S.%fZ')
aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)).reshape((3, self.num_blocks, \
self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2))
mod08_aot = self._mod08_aot()
#if np.isnan(mod08_aot):
self.aot = aot - (aot.mean() - np.nanmean([mod08_aot, aot.mean()]))
#else:
# temp = np.zeros_like(aot)
# temp[:] = mod08_aot
# self.aot = temp
self.tco3 = tco3 * 46.698
tcwv = tcwv / 10.
self.tco3_unc = np.ones(self.tco3.shape) * 0.2
self.aot_unc = np.ones(self.aot.shape) * 2.
self.s2_logger.info(
'Trying to get the tcwv from the emulation of sen2cor look up table.'
)
try:
self._get_tcwv(selected_img, self.s2.angles['vza'],
self.s2.angles['vaa'], self.s2.angles['sza'],
self.s2.angles['saa'], ele_data)
except:
self.s2_logger.warning(
'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.'
)
self.tcwv = tcwv
self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2
'''
self.s2_logger.info('Trying to get the aot from ddv method.')
try:
solved = self._get_ddv_aot(selected_img)
if solved[0] < 0:
self.s2_logger.warning('DDV failed and only cams data used for the prior.')
else:
self.s2_logger.info('DDV solved aot is %.02f, and it will used as the mean value of cams prediction.'%solved[0])
self.aot[:] = solved[0] #self.aot + (solved[0]-self.aot.mean())
except:
self.s2_logger.warning('Getting aot from ddv failed.')
'''
self.s2_logger.info(
'Mean values for priors are: %.02f, %.02f, %.02f and mod08 aot is %.02f'
% (aot.mean(), self.tcwv.mean(), self.tco3.mean(), mod08_aot))
self.s2_logger.info('Applying PSF model.')
if self.s2_psf is None:
xstd, ystd, ang, xs, ys = self._get_psf(selected_img)
else:
xstd, ystd, ang, xs, ys = self.s2_psf
# apply psf shifts without going out of the image extend
shifted_mask = np.logical_and.reduce(
((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]),
(self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0])))
self.Hx, self.Hy = self.Hx[shifted_mask] + int(
xs), self.Hy[shifted_mask] + int(ys)
#self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
self.s2_boa = self.s2_boa[:, shifted_mask]
self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]
self.s2_logger.info('Getting the convolved TOA reflectance.')
#self.valid_pixs = sum(shifted_mask) # count how many pixels is still within the s2 tile
ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
#self.bad_pixs = np.zeros(self.valid_pixs).astype(bool)
imgs = []
for i, band in enumerate(self.s2_u_bands[:-2]):
if selected_img[band].shape != self.full_res:
imgs.append(
self.repeat_extend(selected_img[band],
shape=self.full_res))
else:
imgs.append(selected_img[band])
border_mask = np.zeros(self.full_res).astype(bool)
border_mask[[0, -1], :] = True
border_mask[:, [0, -1]] = True
self.dcloud = binary_dilation(self.cloud | border_mask,
structure=np.ones((3, 3)).astype(bool),
iterations=ker_size / 2).astype(bool)
self.bad_pixs = self.dcloud[self.Hx, self.Hy]
del selected_img
del self.s2.selected_img
del self.s2.angles['vza']
del self.s2.angles['vaa']
del self.s2.angles['sza']
del self.s2.angles['saa']
del self.s2.sza
del self.s2.saa
del self.s2
ker = self.gaussian(xstd, ystd, ang)
f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
.Hy] * 0.0001
half = parmap(f, imgs[:3])
self.s2_toa = np.array(half + parmap(f, imgs[3:]))
del imgs
# get the valid value masks
qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)
boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
np.all(self.s2_boa >= 0.001, axis = 0) &\
np.all(self.s2_boa < 1, axis = 0)
toa_mask = (~self.bad_pixs) &\
np.all(self.s2_toa >= 0.0001, axis = 0) &\
np.all(self.s2_toa < 1., axis = 0)
self.s2_mask = boa_mask & toa_mask & qua_mask
self.Hx = self.Hx[self.s2_mask]
self.Hy = self.Hy[self.s2_mask]
self.s2_toa = self.s2_toa[:, self.s2_mask]
self.s2_boa = self.s2_boa[:, self.s2_mask]
self.s2_boa_unc = self.s2_boa_qa[:, self.s2_mask]
self.s2_logger.info('Solving...')
tempm = np.zeros(self.full_res)
tempm[self.Hx, self.Hy] = 1
tempm = tempm.reshape(self.num_blocks, self.block_size, \
self.num_blocks, self.block_size).astype(int).sum(axis=(3,1))
mask = ~self.dcloud
self.mask = mask.reshape(self.num_blocks, self.block_size, \
self.num_blocks, self.block_size).astype(int).sum(axis=(3,1))
self.mask = ((self.mask / ((1. * self.block_size)**2)) >= 0.5) & (
(tempm / ((self.aero_res / 500.)**2)) >= 0.5)
self.mask = binary_erosion(self.mask,
structure=np.ones((3, 3)).astype(bool))
self.aero = solving_atmo_paras(
self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa,
self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc,
self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy,
self.mask, self.full_res, self.aero_res, self.emus,
self.band_indexs, self.boa_bands)
solved = self.aero._optimization()
return solved
def _l8_aerosol(self, ):
self.logger.propagate = False
self.logger.info('Start to retrieve atmospheric parameters.')
l8 = read_l8(self.l8_toa_dir,
self.l8_tile,
self.year,
self.month,
self.day,
bands=self.bands)
l8._get_angles()
self.logger.info('Loading emulators.')
self._load_xa_xb_xc_emus()
self.logger.info(
'Find corresponding pixels between L8 and MODIS tiles')
self.example_file = self.l8_toa_dir + '/%s_b%d.tif' % (l8.header, 1)
tiles = Find_corresponding_pixels(self.example_file,
destination_res=500)
if len(tiles.keys()) > 1:
self.logger.info('This Landsat 8 tile covers %d MODIS tile.' %
len(tiles.keys()))
self.mcd43_files = []
boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], []
for key in tiles.keys()[1:]:
self.logger.info('Getting BOA from MODIS tile: %s.' % key)
mcd43_file = glob(self.mcd43_tmp %
(self.mcd43_dir, self.year, self.doy, key))[0]
self.mcd43_files.append(mcd43_file)
self.H_inds, self.L_inds = tiles[key]
Lx, Ly = self.L_inds
Hx, Hy = self.H_inds
Hxs.append(Hx)
Hys.append(Hy)
vza, sza = l8.vza[:, Hx, Hy], l8.sza[:, Hx, Hy]
vaa, saa = l8.vaa[:, Hx, Hy], l8.saa[:, Hx, Hy]
raa = vaa - saa
boa, boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\
bands=(3,4,1,2,6,7), Linds= [Lx, Ly])
boas.append(boa)
boa_qas.append(boa_qa)
brdf_stds.append(brdf_std)
self.boa = np.hstack(boas)
self.boa_qa = np.hstack(boa_qas)
self.brdf_stds = np.hstack(brdf_stds)
self.logger.info('Applying spectral transform.')
self.boa = self.boa*np.array(self.spectral_transform)[0][...,None] + \
np.array(self.spectral_transform)[1][...,None]
self.Hx = np.hstack(Hxs)
self.Hy = np.hstack(Hys)
self.sza = l8.sza[:, self.Hx, self.Hy]
self.vza = l8.vza[:, self.Hx, self.Hy]
self.saa = l8.saa[:, self.Hx, self.Hy]
self.vaa = l8.vaa[:, self.Hx, self.Hy]
self.logger.info('Reading in TOA reflectance.')
toa = l8._get_toa()
self.toa = toa[:, self.Hx, self.Hy]
self.sen_time = l8.sen_time
self.logger.info('Getting elevation.')
ele_data = reproject_data(self.global_dem, self.example_file).data
mask = ~np.isfinite(ele_data)
ele_data = np.ma.array(ele_data, mask=mask) / 1000.
self.logger.info('Getting pripors from ECMWF forcasts.')
aot, tcwv, tco3 = np.array(self._read_cams(self.example_file))
self._get_ddv_aot(toa, l8, tcwv, tco3, ele_data)
aot, tcwv, tco3 = np.array(self._read_cams(self.example_file))
self.aot = aot[self.Hx,
self.Hy] #* (1-0.14) # validation of +14% biase
self.tco3 = tco3[self.Hx, self.Hy] #* (1 - 0.05)
self.tcwv = tcwv[self.Hx, self.Hy]
self.aot_unc = np.ones(self.aot.shape) * 0.5
self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2
self.tco3_unc = np.ones(self.tco3.shape) * 0.2
def _s2_aerosol(self, ):
self.s2_logger.propagate = False
self.s2_logger.info('Start to retrieve atmospheric parameters.')
self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
self.day, self.s2_u_bands)
self.s2_logger.info('Reading in TOA reflectance.')
selected_img = self.s2.get_s2_toa()
self.s2_file_dir = self.s2.s2_file_dir
self.s2.get_s2_cloud()
self.s2_logger.info('Loading emulators.')
self._load_xa_xb_xc_emus()
self.s2_logger.info(
'Find corresponding pixels between S2 and MODIS tiles')
tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2',
destination_res=500)
if len(tiles.keys()) > 1:
self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' %
len(tiles.keys()))
self.mcd43_files = []
boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], []
self.s2_logger.info(
'Getting the angles and simulated surface reflectance.')
for key in tiles.keys():
self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key)
mcd43_file = glob(self.mcd43_tmp %
(self.mcd43_dir, self.year, self.doy, key))[0]
self.mcd43_files.append(mcd43_file)
self.H_inds, self.L_inds = tiles[key]
Lx, Ly = self.L_inds
Hx, Hy = self.H_inds
Hxs.append(Hx)
Hys.append(Hy)
self.s2.get_s2_angles(self.reconstruct_s2_angle)
self.s2_angles = np.zeros((4, 6, len(Hx)))
for j, band in enumerate(self.s2_u_bands[:-2]):
self.s2_angles[[0,2],j,:] = (self.s2.angles['vza'][band])[Hx, Hy], \
(self.s2.angles['vaa'][band])[Hx, Hy]
self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \
self.s2.angles['saa'][Hx, Hy]
#use mean value to fill bad values
for i in range(4):
mask = ~np.isfinite(self.s2_angles[i])
if mask.sum() > 0:
self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \
np.flatnonzero(~mask), \
self.s2_angles[i][~mask]) # simple interpolation
vza, sza = self.s2_angles[:2]
vaa, saa = self.s2_angles[2:]
raa = vaa - saa
# get the simulated surface reflectance
s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\
bands=(3,4,1,2,6,7), Linds= [Lx, Ly])
boas.append(s2_boa)
boa_qas.append(s2_boa_qa)
brdf_stds.append(brdf_std)
self.s2_boa = np.hstack(boas)
self.s2_boa_qa = np.hstack(boa_qas)
self.brdf_stds = np.hstack(brdf_stds)
self.s2_logger.info('Applying spectral transform.')
self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
np.array(self.s2_spectral_transform)[1,:-1][...,None]
self.Hx = np.hstack(Hxs)
self.Hy = np.hstack(Hys)
del sza
del vza
del saa
del vaa
del raa
del mask
del boas
del boa_qas
del brdf_stds
del Hxs
del Hys
shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \
self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks)
self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1))
self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1))
self.vza = []
self.vaa = []
for band in self.s2_u_bands[:-2]:
self.vza.append(
self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1)))
self.vaa.append(
self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1)))
self.vza = np.array(self.vza)
self.vaa = np.array(self.vaa)
self.raa = self.saa[None, ...] - self.vaa
self.s2_logger.info('Getting elevation.')
example_file = self.s2.s2_file_dir + '/B04.jp2'
ele_data = reproject_data(self.global_dem,
example_file,
outputType=gdal.GDT_Float32).data
mask = ~np.isfinite(ele_data)
ele_data = np.ma.array(ele_data, mask=mask) / 1000.
self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \
self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1))
self.s2_logger.info('Getting pripors from ECMWF forcasts.')
sen_time_str = json.load(
open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
self.sen_time = datetime.datetime.strptime(sen_time_str,
u'%Y-%m-%dT%H:%M:%S.%fZ')
aot, tcwv, tco3 = np.array(self._read_cams(example_file)).reshape((3, self.num_blocks, \
self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2))
self.aot = aot #* (1-0.14) # validation of +14% biase
self.tco3 = tco3 * 46.698 #* (1 - 0.05)
tcwv = tcwv / 10.
self.tco3_unc = np.ones(self.tco3.shape) * 0.2
self.aot_unc = np.ones(self.aot.shape) * 0.5
self.s2_logger.info(
'Trying to get the tcwv from the emulation of sen2cor look up table.'
)
try:
self._get_tcwv(selected_img, self.s2.angles['vza'],
self.s2.angles['vaa'], self.s2.angles['sza'],
self.s2.angles['saa'], ele_data)
except:
self.s2_logger.warning(
'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.'
)
self.tcwv = tcwv
self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2
self.s2_logger.info('Trying to get the aot from ddv method.')
try:
solved = self._get_ddv_aot(selected_img)
if solved[0] < 0:
self.s2_logger.warning(
'DDV failed and only cams data used for the prior.')
else:
self.s2_logger.info(
'DDV solved aot is %.02f, and it will used as the mean value of cams prediction.'
% solved[0])
self.aot += (solved[0] - self.aot.mean())
except:
self.s2_logger.warning('Getting aot from ddv failed.')
self.s2_logger.info('Applying PSF model.')
if self.s2_psf is None:
xstd, ystd, ang, xs, ys = self._get_psf(selected_img)
else:
xstd, ystd, ang, xs, ys = self.s2_psf
# apply psf shifts without going out of the image extend
shifted_mask = np.logical_and.reduce(
((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]),
(self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0])))
self.Hx, self.Hy = self.Hx[shifted_mask] + int(
xs), self.Hy[shifted_mask] + int(ys)
#self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
self.s2_boa = self.s2_boa[:, shifted_mask]
self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]
self.brdf_stds = self.brdf_stds[:, shifted_mask]
self.s2_logger.info('Getting the convolved TOA reflectance.')
self.valid_pixs = sum(
shifted_mask) # count how many pixels is still within the s2 tile
ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
self.bad_pixs = np.zeros(self.valid_pixs).astype(bool)
imgs = []
for i, band in enumerate(self.s2_u_bands[:-2]):
if selected_img[band].shape != self.full_res:
imgs.append(
self.repeat_extend(selected_img[band],
shape=self.full_res))
else:
imgs.append(selected_img[band])
border_mask = np.zeros(self.full_res).astype(bool)
border_mask[[0, -1], :] = True
border_mask[:, [0, -1]] = True
self.bad_pixs = cloud_dilation(self.s2.cloud | border_mask,
iteration=ker_size / 2)[self.Hx,
self.Hy]
del selected_img
del self.s2.selected_img
del self.s2.angles['vza']
del self.s2.angles['vaa']
del self.s2.angles['sza']
del self.s2.angles['saa']
del self.s2.sza
del self.s2.saa
del self.s2
ker = self.gaussian(xstd, ystd, ang)
f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
.Hy] * 0.0001
half = parmap(f, imgs[:3])
self.s2_toa = np.array(half + parmap(f, imgs[3:]))
del imgs
# get the valid value masks
qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)
boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
np.all(self.s2_boa > 0, axis = 0) &\
np.all(self.s2_boa < 1, axis = 0)
toa_mask = (~self.bad_pixs) &\
np.all(self.s2_toa > 0, axis = 0) &\
np.all(self.s2_toa < 1, axis = 0)
self.s2_mask = boa_mask & toa_mask & qua_mask
self.Hx = self.Hx[self.s2_mask]
self.Hy = self.Hy[self.s2_mask]
self.s2_toa = self.s2_toa[:, self.s2_mask]
self.s2_boa = self.s2_boa[:, self.s2_mask]
self.s2_boa_qa = self.s2_boa_qa[:, self.s2_mask]
self.brdf_stds = self.brdf_stds[:, self.s2_mask]
self.s2_boa_unc = grab_uncertainty(self.s2_boa, self.boa_bands,
self.s2_boa_qa,
self.brdf_stds).get_boa_unc()
self.s2_logger.info('Solving...')
self.aero = solving_atmo_paras(
self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa,
self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc,
self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy,
self.full_res, self.aero_res, self.emus, self.band_indexs,
self.boa_bands)
solved = self.aero._optimization()
return solved
def get_s2_angles(self, reconstruct=True, slic=None):
tree = ET.parse(self.s2_file_dir + '/metadata.xml')
root = tree.getroot()
#Sun_Angles_Grid
saa = []
sza = []
msz = []
msa = []
#Viewing_Incidence_Angles_Grids
vza = {}
vaa = {}
mvz = {}
mva = {}
for child in root:
for j in child:
for k in j.findall('Sun_Angles_Grid'):
for l in k.findall('Zenith'):
for m in l.findall('Values_List'):
for x in m.findall('VALUES'):
sza.append(x.text.split())
for n in k.findall('Azimuth'):
for o in n.findall('Values_List'):
for p in o.findall('VALUES'):
saa.append(p.text.split())
for ms in j.findall('Mean_Sun_Angle'):
self.msz = float(ms.find('ZENITH_ANGLE').text)
self.msa = float(ms.find('AZIMUTH_ANGLE').text)
for k in j.findall('Viewing_Incidence_Angles_Grids'):
for l in k.findall('Zenith'):
for m in l.findall('Values_List'):
vza_sub = []
for x in m.findall('VALUES'):
vza_sub.append(x.text.split())
bi, di, angles = k.attrib['bandId'], \
k.attrib['detectorId'], np.array(vza_sub).astype(float)
vza[(int(bi), int(di))] = angles
for n in k.findall('Azimuth'):
for o in n.findall('Values_List'):
vaa_sub = []
for p in o.findall('VALUES'):
vaa_sub.append(p.text.split())
bi, di, angles = k.attrib['bandId'],\
k.attrib['detectorId'], np.array(vaa_sub).astype(float)
vaa[(int(bi), int(di))] = angles
for mvia in j.findall('Mean_Viewing_Incidence_Angle_List'):
for i in mvia.findall('Mean_Viewing_Incidence_Angle'):
mvz[int(i.attrib['bandId'])] = float(
i.find('ZENITH_ANGLE').text)
mva[int(i.attrib['bandId'])] = float(
i.find('AZIMUTH_ANGLE').text)
sza = np.array(sza).astype(float)
saa = np.array(saa).astype(float)
saa[saa > 180] = saa[saa > 180] - 360
mask = np.isnan(sza)
sza = griddata(np.array(np.where(~mask)).T, sza[~mask], \
(np.repeat(range(23), 23).reshape(23,23), \
np.tile (range(23), 23).reshape(23,23)), method='nearest')
mask = np.isnan(saa)
saa = griddata(np.array(np.where(~mask)).T, saa[~mask], \
(np.repeat(range(23), 23).reshape(23,23), \
np.tile (range(23), 23).reshape(23,23)), method='nearest')
self.saa, self.sza = np.repeat(np.repeat(np.array(saa), 500, axis = 0), 500, axis = 1)[:10980, :10980], \
np.repeat(np.repeat(np.array(sza), 500, axis = 0), 500, axis = 1)[:10980, :10980]
dete_id = np.unique([i[1] for i in vaa.keys()])
band_id = range(13)
bands_vaa = []
bands_vza = []
for i in band_id:
band_vaa = np.zeros((23, 23))
band_vza = np.zeros((23, 23))
band_vaa[:] = np.nan
band_vza[:] = np.nan
for j in dete_id:
try:
good = ~np.isnan(vaa[(i, j)])
band_vaa[good] = vaa[(i, j)][good]
good = ~np.isnan(vza[(i, j)])
band_vza[good] = vza[(i, j)][good]
except:
pass
bands_vaa.append(band_vaa)
bands_vza.append(band_vza)
bands_vaa, bands_vza = np.array(bands_vaa), np.array(bands_vza)
vaa = {}
vza = {}
mva_ = {}
mvz_ = {}
for i, band in enumerate(self.s2_bands):
vaa[band] = bands_vaa[i]
vza[band] = bands_vza[i]
try:
mva_[band] = mva[i]
mvz_[band] = mvz[i]
except:
mva_[band] = np.nan
mvz_[band] = np.nan
if self.bands is None:
bands = self.s2_bands
else:
bands = self.bands
self.vza = {}
self.vaa = {}
self.mvz = {}
self.mva = {}
for band in bands:
mask = np.isnan(vza[band])
g_vza = griddata(np.array(np.where(~mask)).T, vza[band][~mask], \
(np.repeat(range(23), 23).reshape(23,23), \
np.tile (range(23), 23).reshape(23,23)), method='nearest')
mask = np.isnan(vaa[band])
g_vaa = griddata(np.array(np.where(~mask)).T, vaa[band][~mask], \
(np.repeat(range(23), 23).reshape(23,23), \
np.tile (range(23), 23).reshape(23,23)), method='nearest')
self.vza[band] = np.repeat(np.repeat(g_vza, 500, axis=0),
500,
axis=1)[:10980, :10980]
g_vaa[g_vaa > 180] = g_vaa[g_vaa > 180] - 360
self.vaa[band] = np.repeat(np.repeat(g_vaa, 500, axis=0),
500,
axis=1)[:10980, :10980]
self.mvz[band] = mvz_[band]
self.mva[band] = mva_[band]
self.angles = {'sza':self.sza, 'saa':self.saa, 'msz':self.msz, 'msa':self.msa,\
'vza':self.vza, 'vaa': self.vaa, 'mvz':self.mvz, 'mva':self.mva}
if reconstruct:
try:
if len(glob(self.s2_file_dir + '/angles/VAA_VZA_*.img')) == 13:
pass
else:
#print 'Reconstructing Sentinel 2 angles...'
subprocess.call(['python', './python/s2a_angle_bands_mod.py', \
self.s2_file_dir+'/metadata.xml', '10'])
if self.bands is None:
bands = self.s2_bands
else:
bands = self.bands
self.vaa = {}
self.vza = {}
fname = [
self.s2_file_dir + '/angles/VAA_VZA_%s.img' % band
for band in bands
]
if len(glob(self.s2_file_dir + '/angles/VAA_VZA_*.img')) == 13:
f = lambda fn: reproject_data(
fn,
self.s2_file_dir + '/B04.jp2',
outputType=gdal.GDT_Float32).data
ret = parmap(f, fname)
for i, angs in enumerate(ret):
#angs[0][angs[0]<0] = (36000 + angs[0][angs[0]<0])
angs = angs.astype(float) / 100.
if slic is None:
self.vaa[bands[i]] = angs[0]
self.vza[bands[i]] = angs[1]
else:
x_ind, y_ind = np.array(slic)
self.vaa[bands[i]] = angs[0][x_ind, y_ind]
self.vza[bands[i]] = angs[1][x_ind, y_ind]
self.angles = {'sza':self.sza, 'saa':self.saa, 'msz':self.msz, 'msa':self.msa,\
'vza':self.vza, 'vaa': self.vaa, 'mvz':self.mvz, 'mva':self.mva}
else:
print 'Reconstruct failed and original angles are used.'
except:
print 'Reconstruct failed and original angles are used.'
def _mask_bad_pix(self):
#snow_mask = blue > 0.6
if os.path.exists(str(self.cloud_mask)):
cloud_g = gdal.Open(str(self.cloud_mask))
elif type(self.cloud_mask).__module__ == 'numpy':
cloud_g = array_to_raster(self.cloud_mask, self.example_file)
cloud = reproject_data(cloud_g, \
self.example_file, \
xRes=self.pixel_res, \
yRes=self.pixel_res, \
xSize=self.full_res[1], \
ySize=self.full_res[0], \
srcNodata = np.nan,\
outputType= gdal.GDT_Float32,\
resample = gdal.GRIORA_NearestNeighbour).data
cloud = cloud.astype(bool)
RED = None
BLUE = None
SWIR_1 = None
NIR = None
GREEN = None
if 3 in self.boa_bands:
BLUE = self._toa_bands[np.where(self.boa_bands == 3)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if BLUE is not None:
water_mask = BLUE < 0.05
snow_mask = BLUE > 0.6
#del BLUE
else:
self.logger.error(
'Blue band is needed for the retirval of aerosol.')
if 2 in self.boa_bands:
NIR = self._toa_bands[np.where(self.boa_bands == 2)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if 1 in self.boa_bands:
RED = self._toa_bands[np.where(self.boa_bands == 1)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if (RED is not None) & (NIR is not None):
NDVI = (NIR - RED) / (NIR + RED)
water_mask = ((NDVI < 0.01) & (NIR < 0.11)) | ((NDVI < 0.1) & (NIR < 0.05)) \
| (NIR <= 0.0001) | (RED <= 0.0001) | np.isnan(NIR) | np.isnan(RED)
del NIR
del RED
del NDVI
elif NIR is not None:
water_mask = NIR < 0.05
del NIR
elif RED is not None:
water_mask = RED < 0.05
del RED
if 6 in self.boa_bands:
SWIR_1 = self._toa_bands[np.where(self.boa_bands == 6)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if 1 in self.boa_bands:
GREEN = self._toa_bands[np.where(self.boa_bands == 4)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if (SWIR_1 is not None) & (GREEN is not None):
NDSI = (GREEN - SWIR_1) / (SWIR_1 + GREEN)
snow_mask = (NDSI > 0.42) | (SWIR_1 <= 0.0001) | (
GREEN <= 0.0001) | np.isnan(SWIR_1) | np.isnan(GREEN)
del SWIR_1
del GREEN
del NDSI
mask = water_mask | snow_mask | cloud | (BLUE > 1)
ker_size = int(2 * 1.96 * self.psf_ystd)
mask = binary_erosion(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=5).astype(bool)
#mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool)
mask = binary_dilation(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=30 + ker_size).astype(bool)
#mask[:30,:] = mask[:,:30] = mask[:,-30:] = mask[-30:,:] = True
self.bad_pix = mask
def MCD43_SurRef(MCD43_dir,
example_file,
year,
doy,
ang_files,
sun_view_ang_scale=[1, 1],
bands=(7, ),
tolz=0.001,
reproject=False):
f_temp = MCD43_dir + '/MCD43A1.A%s.%s.006*.hdf'
temp1 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Parameters_Band%d'
temp2 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Band_Mandatory_Quality_Band%d'
unique_tile = get_hv(example_file)
#print unique_tile
date = datetime.strptime('%d%03d' % (year, doy), '%Y%j')
days = [(date - timedelta(days = i)).strftime('%Y%j') for i in np.arange(16, 0, -1)] + \
[(date + timedelta(days = i)).strftime('%Y%j') for i in np.arange(0, 17, 1)]
#data_f = [[temp1%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days]
#qa_f = [[temp2%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days]
fnames = np.array([[[temp1%(glob.glob(f_temp%(day, tile))[0], band), \
temp2%(glob.glob(f_temp%(day, tile))[0], band)] for \
tile in unique_tile] for band in bands for day in days]).transpose(0,2,1)
g = gdal.Open(example_file)
mg = reproject_data(example_file,
gdal.BuildVRT('', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).g
temp_data = ~np.isnan(mg.ReadAsArray())
geotransform = mg.GetGeoTransform()
xgeo = geotransform[0] + np.arange(0.5, mg.RasterXSize,
1) * geotransform[1]
ygeo = geotransform[3] + np.arange(0.5, mg.RasterYSize,
1) * geotransform[5]
xgeo = np.repeat(xgeo[None, ...], mg.RasterYSize, axis=0)
ygeo = np.repeat(ygeo[..., None], mg.RasterXSize, axis=1)
m_proj = modis_sinu = osr.SpatialReference()
m_proj.ImportFromProj4(
"+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs"
)
h_proj = osr.SpatialReference()
h_proj.ImportFromWkt(gdal.Open(example_file).GetProjection())
h_xgeo, h_ygeo, _ = np.array(
osr.CoordinateTransformation(m_proj, h_proj).TransformPoints(
zip(xgeo[temp_data], ygeo[temp_data]))).T
geotransform = g.GetGeoTransform()
hy = ((h_xgeo - geotransform[0]) / geotransform[1]).astype(int)
hx = ((h_ygeo - geotransform[3]) / geotransform[5]).astype(int)
hmask = (hx >= 0) & (hx < g.RasterYSize) & (hy >= 0) & (hy < g.RasterXSize)
hy = hy[hmask]
hx = hx[hmask]
#print 'got vrt'
max_x, max_y = np.array(np.where(temp_data)).max(axis=1)
min_x, min_y = np.array(np.where(temp_data)).min(axis=1)
xoff, yoff = min_y, min_x
xsize, ysize = (max_y - min_y + 1), (max_x - min_x + 1)
#print 'read in data'
f = lambda fname: [gdal.BuildVRT('', list(fname[0])).ReadAsArray(xoff, yoff, xsize, ysize), \
gdal.BuildVRT('', list(fname[1])).ReadAsArray(xoff, yoff, xsize, ysize)]
#f = lambda fname: gdal.BuildVRT('', fname).ReadAsArray(xoff, yoff, xsize, ysize)
data, qa = np.array(parmap(f, fnames)).T
data = np.concatenate(data).reshape(
(len(bands), len(days), 3, ysize, xsize)).astype(float)
data = np.ma.array(data, mask=(data == 32767)).astype(float)
#test1, test2 = data.copy(), qa.copy()
#global test1; global test2
w = 0.618034**np.concatenate(qa).reshape(len(bands), len(days), ysize,
xsize).astype(float)
#w = 0.618034 ** qa.astype(float)
f = lambda band: np.array(smoothn(data[band[0],:,band[1],:,:], s=10., smoothOrder=1., \
axis=0, TolZ=tolz, verbose=False, isrobust=True, W = w[band[0]]))[[0,3],]
ba = np.array(
[np.tile(range(len(bands)), 3),
np.repeat(range(3), len(bands))]).T
#print 'smoothing....'
smed = np.array(parmap(f, ba))
#global smed
dat = np.concatenate(smed[:,0], axis=0).reshape(3, len(bands), len(days), ysize, \
xsize)[:,:,16, np.where(temp_data)[0]-min_x, np.where(temp_data)[1]-min_y]
wei = np.concatenate(smed[:,1], axis=0).reshape(3, len(bands), len(days), ysize, \
xsize)[:,:,16, np.where(temp_data)[0]-min_x, np.where(temp_data)[1]-min_y]
#std = data.std(axis = 1)[:, :, np.where(temp_data)[0]-min_x, np.where(temp_data)[1]-min_y]
#print 'get angles...'
sa_files, va_files = ang_files
if isinstance(va_files[0], str):
f = lambda ang_file: reproject_data(ang_file,
gdal.BuildVRT(
'', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data
vas = np.array(parmap(f, va_files))
elif isinstance(va_files[0], (np.ndarray, np.generic)):
f = lambda array: reproject_data(array_to_raster(array, example_file),
gdal.BuildVRT('', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data
vas = np.array(parmap(f, list(va_files)))
vas = vas * sun_view_ang_scale[1]
if isinstance(sa_files[0], str):
f = lambda ang_file: reproject_data(ang_file,
gdal.BuildVRT(
'', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data
sas = np.array(parmap(f, sa_files))
elif isinstance(sa_files[0], (np.ndarray, np.generic)):
f = lambda array: reproject_data(array_to_raster(array, example_file),
gdal.BuildVRT('', list(fnames[0, 0])),
outputType=gdal.GDT_Float32).data
sas = np.array(parmap(f, list(sa_files)))
if sas.shape[0] == 2:
sas = np.repeat((sas * sun_view_ang_scale[0])[None, ...],
len(bands),
axis=0)
elif sas.shape[0] == len(bands):
sas = sas * sun_view_ang_scale[0]
else:
raise IOError('Wrong shape of sun angles are given.')
raa = vas[:, 0, :, :] - sas[:, 0, :, :]
angles = vas[:, 1, temp_data], sas[:, 1, temp_data], raa[:, temp_data]
kk = get_kk(angles)
k_vol = kk.Ross
k_geo = kk.Li
sur_ref = (dat[0] + dat[1] * k_vol + dat[2] * k_geo) * 0.001
wei = 0.05 / wei
#print wei
unc = np.sqrt(wei[0, :, :]**2 + (wei[1, :, :]**2) * k_vol**2 +
(wei[2, :, :]**2) * k_geo**2)
#unc = np.sqrt((np.sqrt(std[:, 0, :]**2 + (std[:, 1, :]**2)*k_vol**2 + (std[:, 2, :]**2)*k_geo**2) * 0.001)**2 + \
# (np.sqrt(wei[0, :, :]**2 + (wei[1, :, :]**2)*k_vol**2 + (wei[2, :, :]**2)*k_geo**2))**2)
unc = np.minimum(unc, 0.1)
#print unc
if reproject:
f_dat = np.repeat(temp_data[None, ...], len(bands),
axis=0).astype(float)
f_dat[:] = np.nan
unc_dat = f_dat.copy()
f_dat[:, temp_data] = sur_ref
unc_dat[:, temp_data] = unc
f = lambda array: reproject_data(array_to_raster(
array, gdal.BuildVRT('', list(fnames[0, 0]))),
example_file,
outputType=gdal.GDT_Float32).data
f_dat = np.array(parmap(f, list(f_dat)))
unc_dat = np.array(parmap(f, list(unc_dat)))
mask = np.isnan(unc_dat) | (f_dat < 0.00001)
f_dat[mask] = np.nan
unc_dat[mask] = np.nan
unc_dat[unc_dat == 0] = 0.1
return f_dat, unc_dat
else:
lx, ly = np.where(temp_data)
sur_ref[sur_ref.mask] = np.nan
unc[unc.mask] = 0.1
return sur_ref.data[:, hmask], unc.data[:, hmask], hx, hy, lx[
hmask], ly[hmask], fnames[0, 0]
def _s2_aerosol(self, ):
self.s2_logger.propagate = False
self.s2_logger.info('Start to retrieve atmospheric parameters.')
self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month,
self.day, self.s2_u_bands)
self.s2_logger.info('Reading in TOA reflectance.')
selected_img = self.s2.get_s2_toa()
self.s2_file_dir = self.s2.s2_file_dir
self.s2.get_s2_cloud()
self.s2_logger.info(
'Find corresponding pixels between S2 and MODIS tiles')
tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2',
destination_res=500)
if len(tiles.keys()) > 1:
self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' %
len(tiles.keys()))
self.mcd43_files = []
szas, vzas, saas, vaas, raas = [], [], [], [], []
boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], []
for key in tiles.keys():
#h,v = int(key[1:3]), int(key[-2:])
self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key)
mcd43_file = glob(self.mcd43_tmp %
(self.mcd43_dir, self.year, self.doy, key))[0]
self.mcd43_files.append(mcd43_file)
self.H_inds, self.L_inds = tiles[key]
Lx, Ly = self.L_inds
Hx, Hy = self.H_inds
Hxs.append(Hx)
Hys.append(Hy)
self.s2_logger.info(
'Getting the angles and simulated surface reflectance.')
self.s2.get_s2_angles(self.reconstruct_s2_angle)
self.s2_angles = np.zeros((4, 6, len(Hx)))
for j, band in enumerate(self.s2_u_bands[:-2]):
self.s2_angles[[0,2],j,:] = self.s2.angles['vza'][band][Hx, Hy], \
self.s2.angles['vaa'][band][Hx, Hy]
self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \
self.s2.angles['saa'][Hx, Hy]
#use mean value to fill bad values
for i in range(4):
mask = ~np.isfinite(self.s2_angles[i])
if mask.sum() > 0:
self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \
np.flatnonzero(~mask), self.s2_angles[i][~mask]) # simple interpolation
vza, sza = self.s2_angles[:2]
vaa, saa = self.s2_angles[2:]
raa = vaa - saa
szas.append(sza)
vzas.append(vza)
raas.append(raa)
vaas.append(vaa)
saas.append(saa)
# get the simulated surface reflectance
s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\
bands=(3,4,1,2,6,7), Linds= [Lx, Ly])
boas.append(s2_boa)
boa_qas.append(s2_boa_qa)
brdf_stds.append(brdf_std)
self.s2_boa = np.hstack(boas)
self.s2_boa_qa = np.hstack(boa_qas)
self.brdf_stds = np.hstack(brdf_stds)
self.Hx = np.hstack(Hxs)
self.Hy = np.hstack(Hys)
vza = np.hstack(vzas)
sza = np.hstack(szas)
vaa = np.hstack(vaas)
saa = np.hstack(saas)
raa = np.hstack(raas)
self.s2_angles = np.array([vza, sza, vaa, saa])
#self.s2_boa, self.s2_boa_qa = self.s2_boa.flatten(), self.s2_boa_qa.flatten()
self.s2_logger.info('Applying spectral transform.')
self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \
np.array(self.s2_spectral_transform)[1,:-1][...,None]
self.s2_logger.info('Getting elevation.')
ele_data = reproject_data(self.global_dem,
self.s2.s2_file_dir + '/B04.jp2',
outputType=gdal.GDT_Float32).data
mask = ~np.isfinite(ele_data)
ele_data = np.ma.array(ele_data, mask=mask) / 1000.
self.elevation = ele_data[self.Hx, self.Hy]
self.s2_logger.info('Getting pripors from ECMWF forcasts.')
sen_time_str = json.load(
open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp']
self.sen_time = datetime.datetime.strptime(sen_time_str,
u'%Y-%m-%dT%H:%M:%S.%fZ')
example_file = self.s2.s2_file_dir + '/B04.jp2'
aod, tcwv, tco3 = np.array(self._read_cams(example_file))[:, self.Hx,
self.Hy]
self.s2_aod550 = aod #* (1-0.14) # validation of +14% biase
self.s2_tco3 = tco3 * 46.698 #* (1 - 0.05)
tcwv = tcwv / 10.
self.s2_tco3_unc = np.ones(self.s2_tco3.shape) * 0.2
self.s2_aod550_unc = np.ones(self.s2_aod550.shape) * 0.5
self.s2_logger.info(
'Trying to get the tcwv from the emulation of sen2cor look up table.'
)
#try:
self._get_tcwv(selected_img, vza, sza, raa, ele_data)
#except:
# self.s2_logger.warning('Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.')
# self.s2_tcwv = tcwv
# self.s2_tcwv_unc = np.ones(self.s2_tcwv.shape) * 0.2
self.s2_logger.info('Trying to get the aod from ddv method.')
try:
solved = self._get_ddv_aot(self.s2.angles, example_file,
self.s2_tcwv, ele_data, selected_img)
if solved[0] < 0:
self.s2_logger.warning(
'DDV failed and only cams data used for the prior.')
else:
self.s2_logger.info(
'DDV solved aod is %.02f, and it will used as the mean value of cams prediction.'
% solved[0])
self.s2_aod550 += (solved[0] - self.s2_aod550.mean())
except:
self.s2_logger.warning('Getting aod from ddv failed.')
self.s2_logger.info('Applying PSF model.')
if self.s2_psf is None:
self.s2_logger.info('No PSF parameters specified, start solving.')
high_img = np.repeat(
np.repeat(selected_img['B11'], 2, axis=0), 2, axis=1) * 0.0001
high_indexs = self.Hx, self.Hy
low_img = self.s2_boa[4]
qa, cloud = self.s2_boa_qa[4], self.s2.cloud
psf = psf_optimize(high_img, high_indexs, low_img, qa, cloud, 2)
xs, ys = psf.fire_shift_optimize()
xstd, ystd = 29.75, 39
ang = 0
self.s2_logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \
%(xstd, ystd, 0, xs, ys, 1-psf.costs.min()))
else:
xstd, ystd, ang, xs, ys = self.s2_psf
# apply psf shifts without going out of the image extend
shifted_mask = np.logical_and.reduce(
((self.Hx + int(xs) >= 0),
(self.Hx + int(xs) < self.s2_full_res[0]),
(self.Hy + int(ys) >= 0),
(self.Hy + int(ys) < self.s2_full_res[0])))
self.Hx, self.Hy = self.Hx[shifted_mask] + int(
xs), self.Hy[shifted_mask] + int(ys)
#self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask]
self.s2_boa = self.s2_boa[:, shifted_mask]
self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask]
self.s2_angles = self.s2_angles[:, :, shifted_mask]
self.elevation = self.elevation[shifted_mask]
self.s2_aod550 = self.s2_aod550[shifted_mask]
self.s2_tcwv = self.s2_tcwv[shifted_mask]
self.s2_tco3 = self.s2_tco3[shifted_mask]
self.s2_aod550_unc = self.s2_aod550_unc[shifted_mask]
self.s2_tcwv_unc = self.s2_tcwv_unc[shifted_mask]
self.s2_tco3_unc = self.s2_tco3_unc[shifted_mask]
self.brdf_stds = self.brdf_stds[:, shifted_mask]
self.s2_logger.info('Getting the convolved TOA reflectance.')
self.valid_pixs = sum(
shifted_mask) # count how many pixels is still within the s2 tile
ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd)))
self.bad_pixs = np.zeros(self.valid_pixs).astype(bool)
imgs = []
for i, band in enumerate(self.s2_u_bands[:-2]):
if selected_img[band].shape != self.s2_full_res:
selected_img[band] = self.repeat_extend(selected_img[band],
shape=self.s2_full_res)
else:
pass
selected_img[band][0, :] = -9999
selected_img[band][-1, :] = -9999
selected_img[band][:, 0] = -9999
selected_img[band][:, -1] = -9999
imgs.append(selected_img[band])
# filter out the bad pixels
self.bad_pixs |= cloud_dilation(self.s2.cloud |\
(selected_img[band] <= 0) | \
(selected_img[band] >= 10000),\
iteration= ker_size/2)[self.Hx, self.Hy]
del selected_img
del self.s2.selected_img
del high_img
del self.s2.angles
del self.s2.sza
del self.s2.saa
del self.s2
ker = self.gaussian(xstd, ystd, ang)
f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self
.Hy] * 0.0001
half = parmap(f, imgs[:3])
self.s2_toa = np.array(half + parmap(f, imgs[3:]))
#self.s2_toa = np.array(parmap(f,imgs))
del imgs
# get the valid value masks
qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0)
boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\
np.all(self.s2_boa > 0, axis = 0) &\
np.all(self.s2_boa < 1, axis = 0)
toa_mask = (~self.bad_pixs) &\
np.all(self.s2_toa > 0, axis = 0) &\
np.all(self.s2_toa < 1, axis = 0)
self.s2_mask = boa_mask & toa_mask & qua_mask & (~self.elevation.mask)
self.s2_AEE, self.s2_bounds = self._load_emus(self.s2_sensor)
def MCD43_SurRef(MCD43_dir,
example_file,
year,
doy,
ang_files,
sun_view_ang_scale=[1, 1],
bands=(7, ),
tolz=0.001):
f_temp = MCD43_dir + '/MCD43A1.A%s.%s.006*.hdf'
temp1 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Parameters_Band%d'
temp2 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Band_Mandatory_Quality_Band%d'
unique_tile = get_hv(example_file)
print unique_tile
date = datetime.strptime('%d%03d' % (year, doy), '%Y%j')
days = [(date - timedelta(days = i)).strftime('%Y%j') for i in np.arange(16, 0, -1)] + \
[(date + timedelta(days = i)).strftime('%Y%j') for i in np.arange(0, 17, 1)]
#data_f = [[temp1%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days]
#qa_f = [[temp2%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days]
fnames = np.array([[[temp1%(glob.glob(f_temp%(day, tile))[0], band), \
temp2%(glob.glob(f_temp%(day, tile))[0], band)] for \
tile in unique_tile] for band in bands for day in days]).transpose(0,2,1)
g = gdal.Open(example_file)
temp_data = ~np.isnan(
reproject_data(example_file,
gdal.BuildVRT('', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data)
print 'got vrt'
max_x, max_y = np.array(np.where(temp_data)).max(axis=1)
min_x, min_y = np.array(np.where(temp_data)).min(axis=1)
xoff, yoff = min_y, min_x
xsize, ysize = (max_y - min_y + 1), (max_x - min_x + 1)
print 'read in data'
f = lambda fname: [gdal.BuildVRT('', list(fname[0])).ReadAsArray(xoff, yoff, xsize, ysize), \
gdal.BuildVRT('', list(fname[1])).ReadAsArray(xoff, yoff, xsize, ysize)]
#f = lambda fname: gdal.BuildVRT('', fname).ReadAsArray(xoff, yoff, xsize, ysize)
data, qa = np.array(parmap(f, fnames)).T
data = np.concatenate(data).reshape(
(len(bands), len(days), 3, ysize, xsize)).astype(float)
data = np.ma.array(data, mask=(data == 32767)).astype(float)
w = np.repeat(0.618034**np.concatenate(qa).reshape(
len(bands), len(days), ysize, xsize).astype(float)[None, ...],
3,
axis=0).transpose(1, 2, 0, 3, 4)
w[w < 0.3] = 0.
print 'smoothing....'
# instead of doing smoothn, we actually only need a laplace distribution filter for the middle date
pdf = np.exp(-abs(np.arange(-16, 17, 1) - 0.) / 40.) / (2 * 40.)
pdf = pdf / pdf.sum()
w = w * pdf[None, ..., None, None, None]
dat = ((w * data).sum(axis=1) /
w.sum(axis=1))[:, :,
np.where(temp_data)[0] - min_x,
np.where(temp_data)[1] - min_y]
wei = w[:, 16, 0,
np.where(temp_data)[0] - min_x,
np.where(temp_data)[1] - min_y] / pdf[16]
std = data.std(axis=1)[:, :,
np.where(temp_data)[0] - min_x,
np.where(temp_data)[1] - min_y]
print 'get angles...'
va_files, sa_files = ang_files
if isinstance(va_files[0], str):
f = lambda ang_file: reproject_data(ang_file,
gdal.BuildVRT(
'', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data
vas = np.array(parmap(f, va_files))
elif isinstance(sa_files[0], (np.ndarray, np.generic)):
f = lambda array: reproject_data(array_to_raster(array, example_file),
gdal.BuildVRT('', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data
vas = np.array(parmap(f, list(va_files)))
vas = vas * sun_view_ang_scale[1]
if isinstance(sa_files[0], str):
f = lambda ang_file: reproject_data(ang_file,
gdal.BuildVRT(
'', list(fnames[0, 0])),
outputType=gdal.GDT_Float64).data
sas = np.array(parmap(f, sa_files))
elif isinstance(sa_files[0], (np.ndarray, np.generic)):
f = lambda array: reproject_data(array_to_raster(array, example_file),
gdal.BuildVRT('', list(fnames[0, 0])),
outputType=gdal.GDT_Float32).data
sas = np.array(parmap(f, list(sa_files)))
if sas.shape[0] == 2:
sas = np.repeat((sas * sun_view_ang_scale[0])[None, ...],
len(bands),
axis=0)
elif sas.shape[0] == len(bands):
sas = sas * sun_view_ang_scale[0]
else:
raise IOError('Wrong shape of sun angles are given.')
raa = vas[:, 0, :, :] - sas[:, 0, :, :]
angles = vas[:, 1, temp_data], sas[:, 1, temp_data], raa[:, temp_data]
kk = get_kk(angles)
k_vol = kk.Ross
k_geo = kk.Li
sur_ref = (dat[:, 0] + dat[:, 1] * k_vol + dat[:, 2] * k_geo) * 0.001
wei = 0.05 / wei
print wei.T
unc = np.sqrt((np.sqrt(std[:, 0, :]**2 + (std[:, 1, :]**2) * k_vol**2 +
(std[:, 2, :]**2) * k_geo**2) * 0.001)**2 + wei**2)
unc = np.minimum(unc, 0.5)
print unc
f_dat = np.repeat(temp_data[None, ...], len(bands), axis=0).astype(float)
f_dat[:] = np.nan
unc_dat = f_dat.copy()
f_dat[:, temp_data] = sur_ref
unc_dat[:, temp_data] = unc
f = lambda array: reproject_data(array_to_raster(
array, gdal.BuildVRT('', list(fnames[0, 0]))),
example_file,
outputType=gdal.GDT_Float32).data
f_dat = np.array(parmap(f, list(f_dat)))
unc_dat = np.array(parmap(f, list(unc_dat)))
mask = np.isnan(unc_dat) | (f_dat < 0.00001)
f_dat[mask] = np.nan
unc_dat[mask] = np.nan
return f_dat, unc_dat
bandid.append(int(n.attrib['bandId']))
solar_irradiance = sorted(solar_irradiance, key = lambda i: bandid[solar_irradiance.index(i)])
bandid = sorted(bandid)
return alpha, beta, gains, u_diff_temp, quant, u_sun, solar_irradiance, bandid, spacecraft
def cal_unc(toa, sza, toa_band_id, alpha, beta, gains, u_diff_temp, quant, u_sun, solar_irradiance, bandid, spacecraft):
rut_algo = s2_rut_algo.S2RutAlgo()
rut_algo.a = gains[toa_band_id]
rut_algo.beta = beta[toa_band_id]
rut_algo.alpha = alpha[toa_band_id]
rut_algo.tecta = sza
rut_algo.e_sun = solar_irradiance[toa_band_id]
rut_algo.u_sun = u_sun
rut_algo.quant = quant
rut_algo.u_diff_temp = u_diff_temp[toa_band_id]
unc = rut_algo.unc_calculation(toa, toa_band_id, spacecraft)
return unc
b2 = '/data/nemesis/S2_data/S2A_MSIL1C_20181209T031111_N0207_R075_T50SMG_20181209T045755.SAFE/GRANULE/L1C_T50SMG_A018090_20181209T031113/IMG_DATA/T50SMG_20181209T031111_B01.jp2'
sza = '/data/nemesis/S2_data/S2A_MSIL1C_20181209T031111_N0207_R075_T50SMG_20181209T045755.SAFE/GRANULE/L1C_T50SMG_A018090_20181209T031113/ANG_DATA/SAA_SZA.tif'
#b2 = '/data/nemesis/S2_data/S2B_MSIL1C_20180611T110619_N0206_R137_T30UYD_20180611T170311.SAFE/GRANULE/L1C_T30UYD_A006598_20180611T110704/IMG_DATA/T30UYD_20180611T110619_B12.jp2'
#sza = '/data/nemesis/S2_data/S2B_MSIL1C_20180611T110619_N0206_R137_T30UYD_20180611T170311.SAFE/GRANULE/L1C_T30UYD_A006598_20180611T110704/ANG_DATA/SAA_SZA.tif'
sza = reproject_data(sza, b2, outputType= gdal.GDT_Float32).data[1]/100.
b2 = gdal.Open(b2).ReadAsArray()
coefs = get_coefs(product_meta_file, datasctrip_meta_file)
unc = cal_unc(b2, sza, 12, *coefs)
def _l8_aerosol(self,):
self.logger.propagate = False
self.logger.info('Start to retrieve atmospheric parameters.')
l8 = read_l8(self.l8_toa_dir, self.l8_tile, self.year, self.month, self.day, bands = self.bands)
self.l8_header = l8.header
self.logger.info('Loading emulators.')
self._load_xa_xb_xc_emus()
self.logger.info('Find corresponding pixels between L8 and MODIS tiles')
self.example_file = self.l8_toa_dir + '/%s_b%d.tif'%(l8.header, 1)
if len(glob(self.l8_toa_dir + '/MCD43_%s.npz'%(l8.header))) == 0:
boa, unc, hx, hy, lx, ly, flist = MCD43_SurRef(self.mcd43_dir, self.example_file, \
self.year, self.doy, [l8.saa_sza, l8.vaa_vza],
sun_view_ang_scale=[0.01, 0.01], bands = [3,4,1,2,6,7], tolz=0.003)
np.savez(self.l8_toa_dir + 'MCD43_%s.npz'%l8.header, boa=boa, unc=unc, hx=hx, hy=hy, lx=lx, ly=ly, flist=flist)
else:
f = np.load(self.l8_toa_dir + 'MCD43_%s.npz'%l8.header)
boa, unc, hx, hy, lx, ly, flist = f['boa'], f['unc'], f['hx'], f['hy'], f['lx'], f['ly'], f['flist']
self.Hx, self.Hy = hx, hy
self.logger.info('Applying spectral transform.')
self.boa_qa = np.ma.array(unc)
self.boa = np.ma.array(boa)*np.array(self.spectral_transform)[0][...,None] + \
np.array(self.spectral_transform)[1][...,None]
self.logger.info('Reading in TOA reflectance.')
self.sen_time = l8.sen_time
self.cloud = l8._get_qa()
self.full_res = self.cloud.shape
self.ecloud = binary_erosion(self.cloud, structure=np.ones((3,3)).astype(bool), iterations=10).astype(bool)
border_mask = np.zeros(self.full_res).astype(bool)
border_mask[[0, -1], :] = True
border_mask[:, [0, -1]] = True
xstd, ystd = 12., 20.
ker_size = 2*int(round(max(1.96*xstd, 1.96*ystd)))
self.dcloud = binary_dilation(self.ecloud | border_mask, structure=np.ones((3,3)).astype(bool), iterations=ker_size/2+10).astype(bool)
self.logger.info('Getting elevation.')
ele_data = reproject_data(self.global_dem, self.example_file, outputType = gdal.GDT_Float32).data/1000.
mask = ~np.isfinite(ele_data)
self.ele = np.ma.array(ele_data, mask = mask)
self.ele[mask] = np.nan
self.logger.info('Getting pripors from ECMWF forcasts.')
self.aot, self.tcwv, self.tco3 = np.array(self._read_cams(self.example_file))
self.logger.info('Mean values of priors are: %.02f, %.02f, %.02f'%\
(np.nanmean(self.aot), np.nanmean(self.tcwv), np.nanmean(self.tco3)))
self.toa = l8._get_toa()
self.saa, self.sza, self.vaa, self.vza = l8._get_angles()
self.saa[self.saa.mask] = self.sza[self.sza.mask] = \
self.vaa[self.vaa.mask] = self.vza[self.vza.mask] = np.nan
self.logger.info('Getting DDV aot prior')
self._get_ddv_aot(self.toa, l8, self.tcwv, self.tco3, ele_data)
self.logger.info('Sorting data.')
self.block_size = int(np.ceil(1. * self.aero_res / 30.))
self.num_blocks = int(np.ceil(max(self.full_res) / (1. * self.block_size)))
self.efull_res = self.block_size * self.num_blocks
shape1 = (self.num_blocks, self.block_size, self.num_blocks, self.block_size)
shape2 = (self.vza.shape[0], self.num_blocks, self.block_size, self.num_blocks, self.block_size)
self.ele = np.nanmean(self._extend_vals(self.ele ).reshape(shape1), axis=(3,1))
self.aot = np.nanmean(self._extend_vals(self.aot ).reshape(shape1), axis=(3,1))
self.tcwv = np.nanmean(self._extend_vals(self.tcwv).reshape(shape1), axis=(3,1))
self.tco3 = np.nanmean(self._extend_vals(self.tco3).reshape(shape1), axis=(3,1))
self.saa = np.nanmean(self._extend_vals(self.saa ).reshape(shape2), axis=(4,2))
self.sza = np.nanmean(self._extend_vals(self.sza ).reshape(shape2), axis=(4,2))
self.vaa = np.nanmean(self._extend_vals(self.vaa ).reshape(shape2), axis=(4,2))
self.vza = np.nanmean(self._extend_vals(self.vza ).reshape(shape2), axis=(4,2))
self.aot_unc = np.ones(self.aot.shape) * 0.8
self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2
self.tco3_unc = np.ones(self.tco3.shape) * 0.2
#mod08_aot, myd08_aot = self._mcd08_aot()
#self.logger.info('Mean values for priors are: %.02f, %.02f, %.02f and mod08 and myd08 aot are: %.02f, %.02f'%\
# (np.nanmean(self.aot), np.nanmean(self.tcwv), np.nanmean(self.tco3), mod08_aot, myd08_aot))
#if np.isnan(mod08_aot):
self.aot[:] = np.nanmean(self.aot)
#else:
# temp = np.zeros_like(self.aot)
# temp[:] = mod08_aot
# self.aot = temp
self.logger.info('Applying PSF model.')
if self.l8_psf is None:
xstd, ystd, ang, xs, ys = self._get_psf()
else:
xstd, ystd, ang, xs, ys = self.l8_psf
shifted_mask = np.logical_and.reduce(((self.Hx+int(xs)>=0),
(self.Hx+int(xs)<self.full_res[0]),
(self.Hy+int(ys)>=0),
(self.Hy+int(ys)<self.full_res[1])))
self.Hx, self.Hy = self.Hx[shifted_mask]+int(xs), self.Hy[shifted_mask]+int(ys)
self.boa = self.boa [:, shifted_mask]
self.boa_qa = self.boa_qa[:, shifted_mask]
self.logger.info('Getting the convolved TOA reflectance.')
self.bad_pixs = self.dcloud[self.Hx, self.Hy]
ker = self.gaussian(xstd, ystd, ang)
f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self.Hy]
self.toa = np.array(parmap(f, list(self.toa)))
qua_mask = np.all(self.boa_qa <= self.qa_thresh, axis = 0)
boa_mask = np.all(~self.boa.mask,axis = 0 ) &\
np.all(self.boa >= 0.001, axis = 0) &\
np.all(self.boa < 1, axis = 0)
toa_mask = (~self.bad_pixs) &\
np.all(self.toa >= 0.0001, axis = 0) &\
np.all(self.toa < 1., axis = 0)
self.l8_mask = boa_mask & toa_mask & qua_mask
self.Hx = self.Hx [self.l8_mask]
self.Hy = self.Hy [self.l8_mask]
self.toa = self.toa [:, self.l8_mask]
self.boa = self.boa [:, self.l8_mask]
self.boa_unc = self.boa_qa[:, self.l8_mask]
self.logger.info('Solving...')
tempm = np.zeros((self.efull_res, self.efull_res))
tempm[self.Hx, self.Hy] = 1
tempm = tempm.reshape(self.num_blocks, self.block_size, \
self.num_blocks, self.block_size).astype(int).sum(axis=(3,1))
self.mask = np.nansum(self._extend_vals((~self.dcloud).astype(int)).reshape(shape1), axis=(3,1))
self.mask = ((self.mask/((1.*self.block_size)**2)) > 0.) & ((tempm/((self.aero_res/500.)**2)) > 0.) & \
(np.any(~np.isnan([self.aot, self.tcwv, self.tco3, self.sza[0]]), axis = 0))
self.mask = binary_erosion(self.mask, structure=np.ones((5, 5)).astype(bool))
self.tcwv[~self.mask] = np.nanmean(self.tcwv)
self.tco3[~self.mask] = np.nanmean(self.tco3)
self.aero = solving_atmo_paras(self.boa,
self.toa,
self.sza,
self.vza,
self.saa,
self.vaa,
self.aot,
self.tcwv,
self.tco3,
self.ele,
self.aot_unc,
self.tcwv_unc,
self.tco3_unc,
self.boa_unc,
self.Hx, self.Hy,
self.mask,
(self.efull_res, self.efull_res),
self.aero_res,
self.emus,
self.band_indexs,
self.boa_bands,
gamma = 2.,
alpha = -1.2,
pix_res = 30)
solved = self.aero._optimization()
return solved
