def _parse_atmo(self,):
atmos = [self.aot, self.tcwv, self.tco3]
atmo_uncs = [self.aot_unc, self.tcwv_unc, self.tco3_unc]
atmo_scale = self.atmo_scale
cams_names = ['aod550', 'tcwv', 'gtco3']
defalt_uncs = [0.4, 0.1, 0.05]
if self.obs_time is not None:
time_ind = np.abs((self.obs_time.hour + self.obs_time.minute/60. + \
self.obs_time.second/3600.) - np.arange(0,25,3)).argmin()
for i in range(3):
if atmos[i] is None:
atmos[i] = self.cams_dir + '/'.join([datetime.strftime(self.obs_time, '%Y_%m_%d'),\
datetime.strftime(self.obs_time, '%Y_%m_%d')+'_%s.tif'%cams_names[i]])
var_g = self._var_parser(atmos[i])
_g = reproject_data(var_g, self.mg, srcNodata = None, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).g
offset = var_g.GetOffset()
scale = var_g.GetScale()
data = _g.GetRasterBand(int(time_ind+1)).ReadAsArray() * scale + offset
atmos[i] = data * self.cams_scale[i]
if atmo_uncs[i] is None:
atmo_uncs[i] = defalt_uncs[i]
else:
for i in range(3):
var_g = self._var_parser(atmos[i])
data = reproject_data(var_g, self.mg, srcNodata = np.nan, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
atmos[i] = data * self.atmo_scale[i]
unc_g = self._var_parser(atmo_uncs[i])
ug = reproject_data(unc_g, self.mg, srcNodata = np.nan, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
atmo_uncs[i] = ug
self._aot, self._tcwv, self._tco3 = atmos
self._aot_unc, self._tcwv_unc, self._tco3_unc = atmo_uncs
def _annoying_angles(self, ):
_sun_angles = []
_view_angles = []
for fname in self._sun_angles:
#nodatas = [float(i.split("=")[1]) for i in gdal.Info(fname).split('\n') if' NoData' in i]
try:
nodatas = ' '.join([
i.split("=")[1] for i in gdal.Info(fname).split('\n')
if ' NoData' in i
])
except:
nodatas = None
ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_sun_angles.append(ang)
for fname in self._view_angles:
try:
nodatas = ' '.join([
i.split("=")[1] for i in gdal.Info(fname).split('\n')
if ' NoData' in i
])
except:
nodatas = None
ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_view_angles.append(ang)
_view_angles = np.array(_view_angles)
_sun_angles = np.squeeze(np.array(_sun_angles))
if len(self._view_angles) == len(self.toa_bands):
if self.a_z_order == 1:
self._vaa = _view_angles[:, 0]
self._vza = _view_angles[:, 1]
else:
self._vaa = _view_angles[:, 1]
self._vza = _view_angles[:, 0]
elif len(self._view_angles) == 2 * len(self.toa_bands):
self._vaa = _view_angles[:len(self.toa_bands)]
self._vza = _view_angles[len(self.toa_bands):]
if os.path.exists(str(self.sun_angles)):
if self.a_z_order == 1:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
else:
self._saa = _sun_angles[1]
self._sza = _sun_angles[0]
elif len(self.sun_angles) == 2:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
self._sza = self._sza * self.ang_scale
self._saa = self._saa * self.ang_scale
self._vza = self._vza * self.ang_scale
self._vaa = self._vaa * self.ang_scale
self._vaa, self._vza, self._saa, self._sza = map(
np.deg2rad, [self._vaa, self._vza, self._saa, self._sza])
def do_cloud(cloud_bands, cloud_name=None):
toas = [
reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data
for band in cloud_bands
]
toas = np.array(toas) / 10000.
mask = np.all(toas >= 0.0001, axis=0)
valid_pixel = mask.sum()
cloud_proba = cl.predict_proba(toas[:, mask].T)
cloud_mask = np.zeros_like(toas[0])
cloud_mask[mask] = cloud_proba[:, 1]
cloud_mask[~mask] = -256
if cloud_name is None:
return cloud_mask
else:
g = gdal.Open(cloud_bands[0])
dst = gdal.GetDriverByName('GTiff').Create(
cloud_name,
g.RasterXSize,
g.RasterYSize,
1,
gdal.GDT_Byte,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((cloud_mask * 100).astype(int))
dst.GetRasterBand(1).SetNoDataValue(-256)
dst = None
g = None
return cloud_mask
def _resamplers(self, ):
'''
Define the sresamplers used for resampling different gdal files or objects to required resolution and size.
'''
self.nearest_resampler = lambda fname: reproject_data(fname, \
self.example_file, \
xRes=self.aero_res*0.5, \
yRes=self.aero_res*0.5, \
srcNodata = None,\
outputType= gdal.GDT_Float32, \
resample = 0 ).g # GRIORA_NearestNeighbour due to version changes...
self.bilinear_resampler = lambda fname: reproject_data(fname, \
self.example_file, \
xRes=self.aero_res*0.5, \
yRes=self.aero_res*0.5, \
srcNodata = 0,\
outputType= gdal.GDT_Float32,\
resample = 1 ).g # GRIORA_Bilinear
def _parse_aux(self,):
auxs = [self.global_dem, self.mask.astype(float)]
scales = [self.ele_scale, 1]
for i in range(len(auxs)):
var_g = self._var_parser(auxs[i])
dat = reproject_data(var_g, self.mg, srcNodata = 0, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
auxs[i] = dat * scales[i]
self._ele, self._cmask = auxs
self._ele[np.isnan(self._ele)] = 0
self._cmask[np.isnan(self._cmask)] = 0
self._cmask = binary_dilation(self._cmask, structure = np.ones((3,3)).astype(bool)).astype(bool)
def _annoying_angles(self, ):
_sun_angles = []
_view_angles = []
for fname in self._sun_angles:
ang = reproject_data(fname, self.mg, srcNodata = None, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_sun_angles.append(ang)
for fname in self._view_angles:
ang = reproject_data(fname, self.mg, srcNodata = None, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_view_angles.append(ang)
_view_angles = np.array(_view_angles)
_sun_angles = np.squeeze(np.array(_sun_angles))
if len(self._view_angles) == len(self.toa_bands):
if self.a_z_order == 1:
self._vaa = _view_angles[:, 0]
self._vza = _view_angles[:, 1]
else:
self._vaa = _view_angles[:, 1]
self._vza = _view_angles[:, 0]
elif len(self._view_angles) == 2 * len(self.toa_bands):
self._vaa = _view_angles[:len(self.toa_bands)]
self._vza = _view_angles[len(self.toa_bands):]
if os.path.exists(str(self.sun_angles)):
if self.a_z_order == 1:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
else:
self._saa = _sun_angles[1]
self._sza = _sun_angles[0]
elif len(self.sun_angles) == 2:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
self._sza = self._sza * self.ang_scale
self._saa = self._saa * self.ang_scale
self._vza = self._vza * self.ang_scale
self._vaa = self._vaa * self.ang_scale
self._vaa, self._vza, self._saa, self._sza = map(
np.deg2rad, [self._vaa, self._vza, self._saa, self._sza])
def _do_chunk(self, ind):
toa_g = self._toa_bands[ind]
geo = toa_g.GetGeoTransform()
x_size, y_size = toa_g.RasterXSize, toa_g.RasterYSize
boas = []
uncs = []
for i in range(self._chunks):
r = i * 1. / self._chunks
x_start = int(r * x_size)
xmin = geo[0] + r * x_size * geo[1]
r = (i+1.)/ self._chunks
x_end = int(r * x_size)
xmax = geo[0] + r * x_size * geo[1]
x_off = x_end - x_start
toa = toa_g.ReadAsArray(x_start, 0, x_off, int(y_size))
if isinstance(self.ref_scale, np.ndarray):
chunk_ref_scale = self.ref_scale_g.ReadAsArray(x_start, 0, x_off, int(y_size))
else:
chunk_ref_scale = self.ref_scale
if isinstance(self.ref_off, np.ndarray):
chunk_ref_off = self.ref_off_g.ReadAsArray(x_start, 0, x_off, int(y_size))
else:
chunk_ref_off = self.ref_off
_g = gdal.Warp('', toa_g, format = 'MEM', outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = 0)
xRes = yRes = int(geo[1])
xps = [self.xap_H[ind], self.xbp_H[ind], self.xcp_H[ind]]
for j in range(3):
xps[j] = reproject_data(xps[j], _g,xRes=xRes, yRes = yRes,xSize=x_off, \
ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data
xap_H, xbp_H, xcp_H = xps
toa = toa * chunk_ref_scale + chunk_ref_off
y = xap_H * toa - xbp_H
boa = y / (1 + xcp_H * y)
xhs = [self.xap_dH[ind], self.xbp_dH[ind], self.xcp_dH[ind]]
for j in range(3):
#var_g = xhs[j]
xhs[j] = reproject_data(xhs[j], _g, xRes=xRes, yRes = yRes, xSize=x_off, \
ySize=y_size, resample = 0, \
outputType= gdal.GDT_Float32).data.transpose(1,2,0)
xap_dH, xbp_dH, xcp_dH = xhs
del xps; del xhs
dH = -1 * (-toa[...,None] * xap_dH - \
2 * toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \
toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \
xbp_dH + \
xbp_H[...,None]**2 * xcp_dH) / \
(toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \
xbp_H[...,None] * xcp_H[...,None] + 1)**2
toa_dH = xap_H / (xcp_H*(toa * xap_H - xbp_H) + 1)**2
del xap_H; del xbp_H; del xcp_H; del xap_dH; del xbp_dH; del xcp_dH
aot_dH, tcwv_dH, tco3_dH = [dH[:, :,i] for i in range(3)]
del dH
tmp = []
for unc_g in [self._aot_unc, self._tcwv_unc, self._tco3_unc]:
unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data
tmp.append(unc)
aot_unc, tcwv_unc, tco3_unc = tmp; del tmp
unc = np.sqrt(aot_dH ** 2 * aot_unc**2 + tcwv_dH ** 2 * tcwv_unc**2 + tco3_dH ** 2 * tco3_unc**2 + toa_dH**2 * 0.015**2)
del aot_unc; del tcwv_unc; del tco3_unc
boas.append(boa)
uncs.append(unc)
boas, uncs = np.hstack(boas), np.hstack(uncs)
boa_name = self.toa_dir + '/' + '.'.join(self.toa_bands[ind].split('/')[-1].split('.')[0:-1]) + '_sur.tif'
unc_name = boa_name.replace('_sur.tif', '_sur_unc.tif')
projectionRef = toa_g.GetProjectionRef()
geotransform = toa_g.GetGeoTransform()
self._save_band(boas, boa_name, projectionRef, geotransform)
self._save_band(uncs, unc_name, projectionRef, geotransform)
if ind in [self.ri, self.gi, self.bi]:
return boas
else:
return None
def do_aot_tcwv(aot_bands,
tcwv_bands,
cams_dir,
obs_time,
sun_ang_name,
view_ang_name,
dem,
tcwv_name=None,
aot_name=None):
toas = [
reproject_data(str(band),
tcwv_bands[0],
dstNodata=0,
resample=5,
xRes=120,
yRes=120).data for band in aot_bands
]
toas = np.array(toas) / 10000.
mask = np.all(toas >= 0.0001, axis=0)
time_ind = np.abs((obs_time.hour + obs_time.minute / 60. +
obs_time.second / 3600.) -
np.arange(0, 25, 3)).argmin()
prior_uncs = 0.1
prior_scale = 46.698
prior_f = cams_dir + '/'.join([
datetime.datetime.strftime(obs_time, '%Y_%m_%d'),
datetime.datetime.strftime(obs_time, '%Y_%m_%d') + '_gtco3.tif'
])
var_g = gdal.Open(prior_f)
prior_g = reproject_data(prior_f,
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).g
g = var_g.GetRasterBand(int(time_ind + 1))
offset = g.GetOffset()
scale = g.GetScale()
tco3 = prior_g.GetRasterBand(
int(time_ind + 1)).ReadAsArray() * scale + offset
tco3[:] = np.nanmean(tco3) * prior_scale
saa, sza = reproject_data(str(sun_ang_name),
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).data / 100.
vaa, vza = reproject_data(str(view_ang_name),
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).data / 100.
raa = vaa - saa
sza = np.cos(np.deg2rad(sza))
vza = np.cos(np.deg2rad(vza))
raa = np.cos(np.deg2rad(raa))
ele = reproject_data(
dem, tcwv_bands[0], dstNodata=0, resample=1, xRes=120,
yRes=120).data / 10000.
X = np.vstack([toas[[9, 8]],
np.array([sza, vza, raa, tco3, ele])]).reshape(7, -1).T
iso_tcwv = Two_NN(np_model_file=file_path + '/emus/S2_TCWV.npz')
tcwv = iso_tcwv.predict(X)[0].reshape(toas[0].shape).astype(float)
bad = (tcwv < 0) | (tcwv > 8) | (~mask) | np.isnan(tcwv)
tcwv[bad] = np.nanmedian(tcwv[~bad])
bands_min = [
0.11572497, 0.08986528, 0.07280412, 0.05007033, 0.06228712, 0.06849915,
0.07015634, 0.06554198, 0.06809415, 0.02089167, 0.00080242, 0.04392302,
0.03012728
]
bands_max = [
0.32072931, 0.30801709, 0.32420026, 0.38214899, 0.3951169, 0.41631785,
0.44391895, 0.42444658, 0.46161492, 0.19927658, 0.00870671, 0.39533241,
0.32505772
]
for _, toa in enumerate(toas):
mask = mask & (toa >= bands_min[_]) & (toa <= bands_max[_])
X = np.vstack([toas, np.array([sza, vza, raa, tco3,
ele])]).reshape(18, -1).T
gbm = joblib.load(file_path + '/emus/lgb.pkl')
aot = gbm.predict(X).reshape(toas[0].shape).astype(float)
aot = np.exp(-1 * aot)
shape = toas[0].shape
if mask.sum() > 3:
aot_min, aot_median, aot_max = np.nanpercentile(aot[mask], [5, 50, 95])
good_aot = (aot >= aot_min) & (aot <= aot_max)
indx, indy = np.where(good_aot.reshape(shape))
myInterpolator = NearestNDInterpolator((indx, indy), aot[good_aot])
grid_x, grid_y = np.mgrid[0:shape[0]:1, 0:shape[1]:1, ]
aot = myInterpolator(grid_x, grid_y)
aot = smoothn(aot,
isrobust=True,
TolZ=1e-6,
W=100 * ((aot >= aot_min) & (aot <= aot_max)),
s=10,
MaxIter=1000)[0]
else:
aot = np.nanmedian(aot) * np.ones(shape)
if tcwv_name is not None:
g = gdal.Open(tcwv_bands[0])
ySize, xSize = tcwv.shape
dst = gdal.GetDriverByName('GTiff').Create(
tcwv_name,
xSize,
ySize,
1,
gdal.GDT_UInt16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((tcwv * 1000).astype(int))
dst.GetRasterBand(1).SetNoDataValue(65535)
dst = None
g = None
if aot_name is not None:
g = gdal.Open(aot_bands[0])
ySize, xSize = aot.shape
dst = gdal.GetDriverByName('GTiff').Create(
aot_name,
xSize,
ySize,
1,
gdal.GDT_UInt16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((aot * 1000).astype(int))
dst.GetRasterBand(1).SetNoDataValue(65535)
dst = None
g = None
return aot, tcwv
def _mask_bad_pix(self):
#snow_mask = blue > 0.6
if os.path.exists(str(self.cloud_mask)):
cloud_g = gdal.Open(str(self.cloud_mask))
elif type(self.cloud_mask).__module__ == 'numpy':
cloud_g = array_to_raster(self.cloud_mask, self.example_file)
cloud = reproject_data(cloud_g, \
self.example_file, \
xRes=self.pixel_res, \
yRes=self.pixel_res, \
xSize=self.full_res[1], \
ySize=self.full_res[0], \
srcNodata = np.nan,\
outputType= gdal.GDT_Float32,\
resample = 0).data
cloud = cloud.astype(bool)
RED = None
BLUE = None
SWIR_1 = None
NIR = None
GREEN = None
if 3 in self.boa_bands:
BLUE = self._toa_bands[np.where(self.boa_bands == 3)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if BLUE is not None:
water_mask = BLUE < 0.05
snow_mask = BLUE > 0.6
#del BLUE
else:
self.logger.error(
'Blue band is needed for the retirval of aerosol.')
if 2 in self.boa_bands:
NIR = self._toa_bands[np.where(self.boa_bands == 2)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if 1 in self.boa_bands:
RED = self._toa_bands[np.where(self.boa_bands == 1)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if (RED is not None) & (NIR is not None):
NDVI = (NIR - RED) / (NIR + RED)
water_mask = ((NDVI < 0.01) & (NIR < 0.11)) | ((NDVI < 0.1) & (NIR < 0.05)) \
| (NIR <= 0.0001) | (RED <= 0.0001) | np.isnan(NIR) | np.isnan(RED)
del NIR
del RED
del NDVI
elif NIR is not None:
water_mask = NIR < 0.05
del NIR
elif RED is not None:
water_mask = RED < 0.05
del RED
if 6 in self.boa_bands:
SWIR_1 = self._toa_bands[np.where(self.boa_bands == 6)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if 1 in self.boa_bands:
GREEN = self._toa_bands[np.where(self.boa_bands == 4)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if (SWIR_1 is not None) & (GREEN is not None):
NDSI = (GREEN - SWIR_1) / (SWIR_1 + GREEN)
snow_mask = (NDSI > 0.42) | (SWIR_1 <= 0.0001) | (
GREEN <= 0.0001) | np.isnan(SWIR_1) | np.isnan(GREEN)
del SWIR_1
del GREEN
del NDSI
mask = water_mask | snow_mask | cloud | (BLUE > 1)
ker_size = int(2 * 1.96 * self.psf_ystd)
mask = binary_erosion(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=5).astype(bool)
#mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool)
mask = binary_dilation(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=30 + ker_size).astype(bool)
mask[:30, :] = mask[:, :30] = mask[:, -30:] = mask[-30:, :] = True
self.bad_pix = mask
