def _load_xa_xb_xc_emus(self,):
xaps = []
xbps = []
xcps = []
for band in self.toa_bands:
band_name = 'B' + band.upper().split('/')[-1].split('B')[-1].split('.')[0]
xap_emu = glob(self.emus_dir + '/isotropic_%s_%s_%s_xap.npz'%(self.sensor, self.satellite, band_name))[0]
xbp_emu = glob(self.emus_dir + '/isotropic_%s_%s_%s_xbp.npz'%(self.sensor, self.satellite, band_name))[0]
xcp_emu = glob(self.emus_dir + '/isotropic_%s_%s_%s_xcp.npz'%(self.sensor, self.satellite, band_name))[0]
xap = Two_NN(np_model_file=xap_emu)
xbp = Two_NN(np_model_file=xbp_emu)
xcp = Two_NN(np_model_file=xcp_emu)
xaps.append(xap)
xbps.append(xbp)
xcps.append(xcp)
self.xap_emus, self.xbp_emus, self.xcp_emus = xaps, xbps, xcps
def do_aot_tcwv(aot_bands,
tcwv_bands,
cams_dir,
obs_time,
sun_ang_name,
view_ang_name,
dem,
tcwv_name=None,
aot_name=None):
toas = [
reproject_data(str(band),
tcwv_bands[0],
dstNodata=0,
resample=5,
xRes=120,
yRes=120).data for band in aot_bands
]
toas = np.array(toas) / 10000.
mask = np.all(toas >= 0.0001, axis=0)
time_ind = np.abs((obs_time.hour + obs_time.minute / 60. +
obs_time.second / 3600.) -
np.arange(0, 25, 3)).argmin()
prior_uncs = 0.1
prior_scale = 46.698
prior_f = cams_dir + '/'.join([
datetime.datetime.strftime(obs_time, '%Y_%m_%d'),
datetime.datetime.strftime(obs_time, '%Y_%m_%d') + '_gtco3.tif'
])
var_g = gdal.Open(prior_f)
prior_g = reproject_data(prior_f,
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).g
g = var_g.GetRasterBand(int(time_ind + 1))
offset = g.GetOffset()
scale = g.GetScale()
tco3 = prior_g.GetRasterBand(
int(time_ind + 1)).ReadAsArray() * scale + offset
tco3[:] = np.nanmean(tco3) * prior_scale
saa, sza = reproject_data(str(sun_ang_name),
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).data / 100.
vaa, vza = reproject_data(str(view_ang_name),
tcwv_bands[0],
dstNodata=0,
resample=1,
xRes=120,
yRes=120).data / 100.
raa = vaa - saa
sza = np.cos(np.deg2rad(sza))
vza = np.cos(np.deg2rad(vza))
raa = np.cos(np.deg2rad(raa))
ele = reproject_data(
dem, tcwv_bands[0], dstNodata=0, resample=1, xRes=120,
yRes=120).data / 10000.
X = np.vstack([toas[[9, 8]],
np.array([sza, vza, raa, tco3, ele])]).reshape(7, -1).T
iso_tcwv = Two_NN(np_model_file=file_path + '/emus/S2_TCWV.npz')
tcwv = iso_tcwv.predict(X)[0].reshape(toas[0].shape).astype(float)
bad = (tcwv < 0) | (tcwv > 8) | (~mask) | np.isnan(tcwv)
tcwv[bad] = np.nanmedian(tcwv[~bad])
bands_min = [
0.11572497, 0.08986528, 0.07280412, 0.05007033, 0.06228712, 0.06849915,
0.07015634, 0.06554198, 0.06809415, 0.02089167, 0.00080242, 0.04392302,
0.03012728
]
bands_max = [
0.32072931, 0.30801709, 0.32420026, 0.38214899, 0.3951169, 0.41631785,
0.44391895, 0.42444658, 0.46161492, 0.19927658, 0.00870671, 0.39533241,
0.32505772
]
for _, toa in enumerate(toas):
mask = mask & (toa >= bands_min[_]) & (toa <= bands_max[_])
X = np.vstack([toas, np.array([sza, vza, raa, tco3,
ele])]).reshape(18, -1).T
gbm = joblib.load(file_path + '/emus/lgb.pkl')
aot = gbm.predict(X).reshape(toas[0].shape).astype(float)
aot = np.exp(-1 * aot)
shape = toas[0].shape
if mask.sum() > 3:
aot_min, aot_median, aot_max = np.nanpercentile(aot[mask], [5, 50, 95])
good_aot = (aot >= aot_min) & (aot <= aot_max)
indx, indy = np.where(good_aot.reshape(shape))
myInterpolator = NearestNDInterpolator((indx, indy), aot[good_aot])
grid_x, grid_y = np.mgrid[0:shape[0]:1, 0:shape[1]:1, ]
aot = myInterpolator(grid_x, grid_y)
aot = smoothn(aot,
isrobust=True,
TolZ=1e-6,
W=100 * ((aot >= aot_min) & (aot <= aot_max)),
s=10,
MaxIter=1000)[0]
else:
aot = np.nanmedian(aot) * np.ones(shape)
if tcwv_name is not None:
g = gdal.Open(tcwv_bands[0])
ySize, xSize = tcwv.shape
dst = gdal.GetDriverByName('GTiff').Create(
tcwv_name,
xSize,
ySize,
1,
gdal.GDT_UInt16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((tcwv * 1000).astype(int))
dst.GetRasterBand(1).SetNoDataValue(65535)
dst = None
g = None
if aot_name is not None:
g = gdal.Open(aot_bands[0])
ySize, xSize = aot.shape
dst = gdal.GetDriverByName('GTiff').Create(
aot_name,
xSize,
ySize,
1,
gdal.GDT_UInt16,
options=["TILED=YES", "COMPRESS=DEFLATE"])
dst.SetGeoTransform(g.GetGeoTransform())
dst.SetProjection(g.GetProjection())
dst.GetRasterBand(1).WriteArray((aot * 1000).astype(int))
dst.GetRasterBand(1).SetNoDataValue(65535)
dst = None
g = None
return aot, tcwv
def __init__(
self,
sensor_sat,
toa_bands,
band_wv,
band_index,
view_angles,
sun_angles,
obs_time,
cloud_mask,
gamma=10.,
a_z_order=1,
pixel_res=None,
aoi=None,
aot_prior=None,
wv_prior=None,
o3_prior=None,
aot_unc=None,
wv_unc=None,
o3_unc=None,
log_file=None,
ref_scale=0.0001,
ref_off=0,
ang_scale=0.01,
ele_scale=0.001,
prior_scale=[1., 0.1, 46.698, 1., 1., 1.],
emus_dir='SIAC/emus/',
mcd43_dir='~/DATA/Multiply/MCD43/',
global_dem='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt',
cams_dir='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/',
spec_m_dir='SIAC/spectral_mapping',
aero_res=1000):
self.sensor = sensor_sat[0]
self.satellite = sensor_sat[1]
self.toa_bands = toa_bands
self.band_wv = band_wv
self.band_index = band_index
self.view_angles = view_angles
self.sun_angles = sun_angles
self.obs_time = obs_time
self.cloud_mask = cloud_mask
self.gamma = gamma
self.a_z_order = a_z_order
self.pixel_res = pixel_res
self.aoi = aoi
self.aot_prior = aot_prior
self.wv_prior = wv_prior
self.o3_prior = o3_prior
self.aot_unc = aot_unc
self.wv_unc = wv_unc
self.o3_unc = o3_unc
self.log_file = log_file
self.ref_scale = ref_scale
self.ref_off = ref_off
self.ang_scale = ang_scale
self.ele_scale = ele_scale
self.prior_scale = prior_scale
self.mcd43_dir = mcd43_dir
self.emus_dir = emus_dir
self.global_dem = global_dem
self.cams_dir = cams_dir
self.boa_wv = [645, 859, 469, 555, 1640, 2130]
self.aero_res = aero_res
self.mcd43_tmp = '%s/MCD43A1.A%d%03d.%s.006.*.hdf'
self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1]))
try:
#spec_map = np.loadtxt(spec_m_dir + '/Aqua_%s_spectral_mapping.txt'%self.satellite).T
self.spec_map = Two_NN(
np_model_file=spec_m_dir +
'/Aqua_%s_spectral_mapping.npz' % self.satellite)
except:
#spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T
self.spec_map = Two_NN(
np_model_file=spec_m_dir +
'/Aqua_%s_spectral_mapping.npz' % self.sensor)
#self.spec_slope = spec_map[0]
#self.spec_off = spec_map[1]
self.logger = create_logger(self.log_file)