def _annoying_angles(self,):
_sun_angles = []
_view_angles = []
for fname in self._sun_angles:
#nodatas = [float(i.split("=")[1]) for i in gdal.Info(fname).split('\n') if' NoData' in i]
try:
nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i])
except:
nodatas = None
ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_sun_angles.append(ang)
for fname in self._view_angles:
try:
nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i])
except:
nodatas = None
ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
_view_angles.append(ang)
_view_angles = np.array(_view_angles)
_sun_angles = np.squeeze(np.array(_sun_angles))
if len(self._view_angles) == len(self.toa_bands):
if self.a_z_order == 1:
self._vaa = _view_angles[:,0]
self._vza = _view_angles[:,1]
else:
self._vaa = _view_angles[:,1]
self._vza = _view_angles[:,0]
elif len(self._view_angles) == 2*len(self.toa_bands):
self._vaa = _view_angles[:len(self.toa_bands)]
self._vza = _view_angles[len(self.toa_bands):]
if os.path.exists(str(self.sun_angles)):
if self.a_z_order == 1:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
else:
self._saa = _sun_angles[1]
self._sza = _sun_angles[0]
elif len(self.sun_angles) == 2:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
self._sza = self._sza * self.ang_scale
self._saa = self._saa * self.ang_scale
self._vza = self._vza * self.ang_scale
self._vaa = self._vaa * self.ang_scale
self._vaa, self._vza, self._saa, self._sza = map(np.deg2rad, [self._vaa, self._vza, self._saa, self._sza])
def do_cloud(cloud_bands, cloud_name=None):
toas = [
reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data
for band in cloud_bands
]
toas = np.array(toas) / 10000.
mask = np.all(toas >= 0.0001, axis=0)
valid_pixel = mask.sum()
cloud_proba = cl.predict_proba(toas[:, mask].T)
cloud_mask = np.zeros_like(toas[0])
cloud_mask[mask] = cloud_proba[:, 1]
cloud_mask[~mask] = -256
if cloud_name is None:
return cloud_mask
else:
g = gdal.Open(cloud_bands[0])
dst = gdal.GetDriverByName('GTiff').Create(
cloud_name,
g.RasterXSize,
g.RasterYSize,
1,
gdal.GDT_Byte,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((cloud_mask * 100).astype(int))
dst.GetRasterBand(1).SetNoDataValue(-256)
dst = None
g = None
return cloud_mask
def _parse_aux(self,):
auxs = [self.global_dem, self.mask.astype(float)]
scales = [self.ele_scale, 1]
for i in range(len(auxs)):
var_g = self._var_parser(auxs[i])
dat = reproject_data(var_g, self.mg, srcNodata = 0, resample = \
0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
auxs[i] = dat * scales[i]
self._ele, self._cmask = auxs
self._ele[np.isnan(self._ele)] = 0
self._cmask[np.isnan(self._cmask)] = 0
self._cmask = binary_dilation(self._cmask, structure = np.ones((3,3)).astype(bool)).astype(bool)
def _resamplers(self, ):
"""
Define the sresamplers used for resampling different gdal files or objects to required resolution and size.
"""
self.nearest_resampler = lambda fname: reproject_data(
fname,
self.example_file,
xRes=self.aero_res * 0.5,
yRes=self.aero_res * 0.5,
srcNodata=None,
outputType=gdal.GDT_Float32,
resample=0,
).g # GRIORA_NearestNeighbour due to version changes...
self.bilinear_resampler = lambda fname: reproject_data(
fname,
self.example_file,
xRes=self.aero_res * 0.5,
yRes=self.aero_res * 0.5,
srcNodata=0,
outputType=gdal.GDT_Float32,
resample=1,
).g # GRIORA_Bilinear
def _annoying_angles(self, destination):
mg = destination
_sun_angles = []
_view_angles = []
for fname in self._sun_angles:
try:
nodatas = " ".join([
i.split("=")[1] for i in gdal.Info(fname).split("\n")
if " NoData" in i
])
except:
nodatas = None
ang = reproject_data(
fname,
mg,
srcNodata=None,
resample=0,
dstNodata=np.nan,
outputType=gdal.GDT_Float32,
).data
_sun_angles.append(ang)
for fname in self._view_angles:
try:
nodatas = " ".join([
i.split("=")[1] for i in gdal.Info(fname).split("\n")
if " NoData" in i
])
except:
nodatas = None
ang = reproject_data(
fname,
mg,
srcNodata=None,
resample=0,
dstNodata=np.nan,
outputType=gdal.GDT_Float32,
).data
_view_angles.append(ang)
_view_angles = np.squeeze(np.array(_view_angles))
_sun_angles = np.squeeze(np.array(_sun_angles))
if len(self.view_angles) == len(self.toa_bands):
if self.a_z_order == 1:
self._vaa = _view_angles[:, 0]
self._vza = _view_angles[:, 1]
else:
self._vaa = _view_angles[:, 1]
self._vza = _view_angles[:, 0]
elif len(self.view_angles) == 2 * len(self.toa_bands):
self._vaa = _view_angles[:len(self.toa_bands)]
self._vza = _view_angles[len(self.toa_bands):]
if os.path.exists(str(self.sun_angles)):
if self.a_z_order == 1:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
else:
self._saa = _sun_angles[1]
self._sza = _sun_angles[0]
elif len(self.sun_angles) == 2:
self._saa = _sun_angles[0]
self._sza = _sun_angles[1]
self._sza = self._sza * self.ang_scale
self._saa = self._saa * self.ang_scale
self._vza = self._vza * self.ang_scale
self._vaa = self._vaa * self.ang_scale
def _mask_bad_pix(self):
# snow_mask = blue > 0.6
if os.path.exists(str(self.cloud_mask)):
cloud_g = gdal.Open(str(self.cloud_mask))
elif type(self.cloud_mask).__module__ == "numpy":
cloud_g = array_to_raster(self.cloud_mask, self.example_file)
cloud = reproject_data(
cloud_g,
self.example_file,
xRes=self.pixel_res,
yRes=self.pixel_res,
xSize=self.full_res[1],
ySize=self.full_res[0],
srcNodata=np.nan,
outputType=gdal.GDT_Float32,
resample=0,
).data
cloud = cloud.astype(bool)
RED = None
BLUE = None
SWIR_1 = None
NIR = None
GREEN = None
if 3 in self.boa_bands:
BLUE = (self._toa_bands[np.where(
self.boa_bands == 3)[0][0]].ReadAsArray() * self.ref_scale +
self.ref_off)
if BLUE is not None:
water_mask = BLUE < 0.05
snow_mask = BLUE > 0.6
# del BLUE
else:
self.logger.error(
"Blue band is needed for the retirval of aerosol.")
if 2 in self.boa_bands:
NIR = (self._toa_bands[np.where(
self.boa_bands == 2)[0][0]].ReadAsArray() * self.ref_scale +
self.ref_off)
if 1 in self.boa_bands:
RED = (self._toa_bands[np.where(
self.boa_bands == 1)[0][0]].ReadAsArray() * self.ref_scale +
self.ref_off)
if (RED is not None) & (NIR is not None):
NDVI = (NIR - RED) / (NIR + RED)
water_mask = (((NDVI < 0.01) & (NIR < 0.11))
| ((NDVI < 0.1) & (NIR < 0.05))
| (NIR <= 0.0001)
| (RED <= 0.0001)
| np.isnan(NIR)
| np.isnan(RED))
del NIR
del RED
del NDVI
elif NIR is not None:
water_mask = NIR < 0.05
del NIR
elif RED is not None:
water_mask = RED < 0.05
del RED
if 6 in self.boa_bands:
SWIR_1 = (self._toa_bands[np.where(
self.boa_bands == 6)[0][0]].ReadAsArray() * self.ref_scale +
self.ref_off)
if 1 in self.boa_bands:
GREEN = (self._toa_bands[np.where(
self.boa_bands == 4)[0][0]].ReadAsArray() * self.ref_scale +
self.ref_off)
if (SWIR_1 is not None) & (GREEN is not None):
NDSI = (GREEN - SWIR_1) / (SWIR_1 + GREEN)
snow_mask = ((NDSI > 0.42)
| (SWIR_1 <= 0.0001)
| (GREEN <= 0.0001)
| np.isnan(SWIR_1)
| np.isnan(GREEN))
del SWIR_1
del GREEN
del NDSI
mask = water_mask | snow_mask | cloud | (BLUE > 1)
ker_size = int(2 * 1.96 * self.psf_ystd)
mask = binary_erosion(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=5).astype(bool)
# mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool)
mask = binary_dilation(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=30 + ker_size).astype(bool)
mask[:30, :] = mask[:, :30] = mask[:, -30:] = mask[-30:, :] = True
self.bad_pix = mask
def do_aot_tcwv(aot_bands,
tcwv_bands,
cams_dir,
obs_time,
sun_ang_name,
view_ang_name,
dem,
tcwv_name=None,
aot_name=None):
toas = [
reproject_data(str(band),
tcwv_bands[0],
dstNodata=0,
resample=5,
xRes=120,
yRes=120).data for band in aot_bands
]
toas = np.array(toas) / 10000.
mask = np.all(toas >= 0.0001, axis=0)
time_ind = np.abs((obs_time.hour + obs_time.minute / 60. +
obs_time.second / 3600.) -
np.arange(0, 25, 3)).argmin()
prior_uncs = 0.1
prior_scale = 46.698
prior_f = cams_dir + '/'.join([
datetime.datetime.strftime(obs_time, '%Y_%m_%d'),
datetime.datetime.strftime(obs_time, '%Y_%m_%d') + '_gtco3.tif'
])
var_g = gdal.Open(prior_f)
prior_g = reproject_data(prior_f,
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).g
g = var_g.GetRasterBand(int(time_ind + 1))
offset = g.GetOffset()
scale = g.GetScale()
tco3 = prior_g.GetRasterBand(
int(time_ind + 1)).ReadAsArray() * scale + offset
tco3[:] = np.nanmean(tco3) * prior_scale
saa, sza = reproject_data(str(sun_ang_name),
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).data / 100.
vaa, vza = reproject_data(str(view_ang_name),
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).data / 100.
raa = vaa - saa
sza = np.cos(np.deg2rad(sza))
vza = np.cos(np.deg2rad(vza))
raa = np.cos(np.deg2rad(raa))
ele = reproject_data(
dem, tcwv_bands[0], dstNodata=0, resample=1, xRes=120,
yRes=120).data / 10000.
X = np.vstack([toas[[9, 8]],
np.array([sza, vza, raa, tco3, ele])]).reshape(7, -1).T
iso_tcwv = Two_NN(np_model_file=file_path + '/emus/S2_TCWV.npz')
tcwv = iso_tcwv.predict(X)[0].reshape(toas[0].shape).astype(float)
bad = (tcwv < 0) | (tcwv > 8) | (~mask) | np.isnan(tcwv)
tcwv[bad] = np.nanmedian(tcwv[~bad])
bands_min = [
0.11572497, 0.08986528, 0.07280412, 0.05007033, 0.06228712, 0.06849915,
0.07015634, 0.06554198, 0.06809415, 0.02089167, 0.00080242, 0.04392302,
0.03012728
]
bands_max = [
0.32072931, 0.30801709, 0.32420026, 0.38214899, 0.3951169, 0.41631785,
0.44391895, 0.42444658, 0.46161492, 0.19927658, 0.00870671, 0.39533241,
0.32505772
]
for _, toa in enumerate(toas):
mask = mask & (toa >= bands_min[_]) & (toa <= bands_max[_])
X = np.vstack([toas, np.array([sza, vza, raa, tco3,
ele])]).reshape(18, -1).T
gbm = joblib.load(file_path + '/emus/lgb.pkl')
aot = gbm.predict(X).reshape(toas[0].shape).astype(float)
aot = np.exp(-1 * aot)
shape = toas[0].shape
if mask.sum() > 3:
aot_min, aot_median, aot_max = np.nanpercentile(aot[mask], [5, 50, 95])
good_aot = (aot >= aot_min) & (aot <= aot_max)
indx, indy = np.where(good_aot.reshape(shape))
myInterpolator = NearestNDInterpolator((indx, indy), aot[good_aot])
grid_x, grid_y = np.mgrid[0:shape[0]:1, 0:shape[1]:1, ]
aot = myInterpolator(grid_x, grid_y)
aot = smoothn(aot,
isrobust=True,
TolZ=1e-6,
W=100 * ((aot >= aot_min) & (aot <= aot_max)),
s=10,
MaxIter=1000)[0]
else:
aot = np.nanmedian(aot) * np.ones(shape)
if tcwv_name is not None:
g = gdal.Open(tcwv_bands[0])
ySize, xSize = tcwv.shape
dst = gdal.GetDriverByName('GTiff').Create(
tcwv_name,
xSize,
ySize,
1,
gdal.GDT_UInt16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((tcwv * 1000).astype(int))
dst.GetRasterBand(1).SetNoDataValue(65535)
dst = None
g = None
if aot_name is not None:
g = gdal.Open(aot_bands[0])
ySize, xSize = aot.shape
dst = gdal.GetDriverByName('GTiff').Create(
aot_name,
xSize,
ySize,
1,
gdal.GDT_UInt16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((aot * 1000).astype(int))
dst.GetRasterBand(1).SetNoDataValue(65535)
dst = None
g = None
return aot, tcwv
def _do_chunk(self, ind):
toa_g = self._toa_bands[ind]
geo = toa_g.GetGeoTransform()
x_size, y_size = toa_g.RasterXSize, toa_g.RasterYSize
boas = []
uncs = []
for i in range(self._chunks):
r = i * 1. / self._chunks
x_start = int(r * x_size)
xmin = geo[0] + r * x_size * geo[1]
r = (i+1.)/ self._chunks
x_end = int(r * x_size)
xmax = geo[0] + r * x_size * geo[1]
x_off = x_end - x_start
toa = toa_g.ReadAsArray(x_start, 0, x_off, int(y_size))
_g = gdal.Warp('', toa_g, format = 'MEM', outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = 0)
xRes = yRes = int(geo[1])
xps = [self.xap_H[ind], self.xbp_H[ind], self.xcp_H[ind]]
for j in range(3):
xps[j] = reproject_data(xps[j], _g,xRes=xRes, yRes = yRes,xSize=x_off, \
ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data
xap_H, xbp_H, xcp_H = xps
toa = toa * self.ref_scale + self.ref_off
y = xap_H * toa - xbp_H
boa = y / (1 + xcp_H * y)
xhs = [self.xap_dH[ind], self.xbp_dH[ind], self.xcp_dH[ind]]
for j in range(3):
#var_g = xhs[j]
xhs[j] = reproject_data(xhs[j], _g, xRes=xRes, yRes = yRes, xSize=x_off, \
ySize=y_size, resample = 0, \
outputType= gdal.GDT_Float32).data.transpose(1,2,0)
xap_dH, xbp_dH, xcp_dH = xhs
del xps; del xhs
dH = -1 * (-toa[...,None] * xap_dH - \
2 * toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \
toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \
xbp_dH + \
xbp_H[...,None]**2 * xcp_dH) / \
(toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \
xbp_H[...,None] * xcp_H[...,None] + 1)**2
toa_dH = xap_H / (xcp_H*(toa * xap_H - xbp_H) + 1)**2
del xap_H; del xbp_H; del xcp_H; del xap_dH; del xbp_dH; del xcp_dH
aot_dH, tcwv_dH, tco3_dH = [dH[:, :,i] for i in range(3)]
del dH
tmp = []
for unc_g in [self._aot_unc, self._tcwv_unc, self._tco3_unc]:
unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data
tmp.append(unc)
aot_unc, tcwv_unc, tco3_unc = tmp; del tmp
unc = np.sqrt(aot_dH ** 2 * aot_unc**2 + tcwv_dH ** 2 * tcwv_unc**2 + tco3_dH ** 2 * tco3_unc**2 + toa_dH**2 * 0.015**2)
del aot_unc; del tcwv_unc; del tco3_unc
boas.append(boa)
uncs.append(unc)
boas, uncs = np.hstack(boas), np.hstack(uncs)
boa_name = self.toa_dir + '/' + '.'.join(self.toa_bands[ind].split('/')[-1].split('.')[0:-1]) + '_sur.tif'
unc_name = boa_name.replace('_sur.tif', '_sur_unc.tif')
projectionRef = toa_g.GetProjectionRef()
geotransform = toa_g.GetGeoTransform()
self._save_band(boas, boa_name, projectionRef, geotransform)
self._save_band(uncs, unc_name, projectionRef, geotransform)
if ind in [self.ri, self.gi, self.bi]:
return boas
else:
return None