def _load_xa_xb_xc_emus(self,): xaps = [] xbps = [] xcps = [] for band in self.toa_bands: band_name = 'B' + band.upper().split('/')[-1].split('B')[-1].split('.')[0] xap_emu = glob(self.emus_dir + '/isotropic_%s_%s_%s_xap.npz'%(self.sensor, self.satellite, band_name))[0] xbp_emu = glob(self.emus_dir + '/isotropic_%s_%s_%s_xbp.npz'%(self.sensor, self.satellite, band_name))[0] xcp_emu = glob(self.emus_dir + '/isotropic_%s_%s_%s_xcp.npz'%(self.sensor, self.satellite, band_name))[0] xap = Two_NN(np_model_file=xap_emu) xbp = Two_NN(np_model_file=xbp_emu) xcp = Two_NN(np_model_file=xcp_emu) xaps.append(xap) xbps.append(xbp) xcps.append(xcp) self.xap_emus, self.xbp_emus, self.xcp_emus = xaps, xbps, xcps
def do_aot_tcwv(aot_bands, tcwv_bands, cams_dir, obs_time, sun_ang_name, view_ang_name, dem, tcwv_name=None, aot_name=None): toas = [ reproject_data(str(band), tcwv_bands[0], dstNodata=0, resample=5, xRes=120, yRes=120).data for band in aot_bands ] toas = np.array(toas) / 10000. mask = np.all(toas >= 0.0001, axis=0) time_ind = np.abs((obs_time.hour + obs_time.minute / 60. + obs_time.second / 3600.) - np.arange(0, 25, 3)).argmin() prior_uncs = 0.1 prior_scale = 46.698 prior_f = cams_dir + '/'.join([ datetime.datetime.strftime(obs_time, '%Y_%m_%d'), datetime.datetime.strftime(obs_time, '%Y_%m_%d') + '_gtco3.tif' ]) var_g = gdal.Open(prior_f) prior_g = reproject_data(prior_f, tcwv_bands[0], dstNodata=0, resample=1, xRes=120, yRes=120).g g = var_g.GetRasterBand(int(time_ind + 1)) offset = g.GetOffset() scale = g.GetScale() tco3 = prior_g.GetRasterBand( int(time_ind + 1)).ReadAsArray() * scale + offset tco3[:] = np.nanmean(tco3) * prior_scale saa, sza = reproject_data(str(sun_ang_name), tcwv_bands[0], dstNodata=0, resample=1, xRes=120, yRes=120).data / 100. vaa, vza = reproject_data(str(view_ang_name), tcwv_bands[0], dstNodata=0, resample=1, xRes=120, yRes=120).data / 100. raa = vaa - saa sza = np.cos(np.deg2rad(sza)) vza = np.cos(np.deg2rad(vza)) raa = np.cos(np.deg2rad(raa)) ele = reproject_data( dem, tcwv_bands[0], dstNodata=0, resample=1, xRes=120, yRes=120).data / 10000. X = np.vstack([toas[[9, 8]], np.array([sza, vza, raa, tco3, ele])]).reshape(7, -1).T iso_tcwv = Two_NN(np_model_file=file_path + '/emus/S2_TCWV.npz') tcwv = iso_tcwv.predict(X)[0].reshape(toas[0].shape).astype(float) bad = (tcwv < 0) | (tcwv > 8) | (~mask) | np.isnan(tcwv) tcwv[bad] = np.nanmedian(tcwv[~bad]) bands_min = [ 0.11572497, 0.08986528, 0.07280412, 0.05007033, 0.06228712, 0.06849915, 0.07015634, 0.06554198, 0.06809415, 0.02089167, 0.00080242, 0.04392302, 0.03012728 ] bands_max = [ 0.32072931, 0.30801709, 0.32420026, 0.38214899, 0.3951169, 0.41631785, 0.44391895, 0.42444658, 0.46161492, 0.19927658, 0.00870671, 0.39533241, 0.32505772 ] for _, toa in enumerate(toas): mask = mask & (toa >= bands_min[_]) & (toa <= bands_max[_]) X = np.vstack([toas, np.array([sza, vza, raa, tco3, ele])]).reshape(18, -1).T gbm = joblib.load(file_path + '/emus/lgb.pkl') aot = gbm.predict(X).reshape(toas[0].shape).astype(float) aot = np.exp(-1 * aot) shape = toas[0].shape if mask.sum() > 3: aot_min, aot_median, aot_max = np.nanpercentile(aot[mask], [5, 50, 95]) good_aot = (aot >= aot_min) & (aot <= aot_max) indx, indy = np.where(good_aot.reshape(shape)) myInterpolator = NearestNDInterpolator((indx, indy), aot[good_aot]) grid_x, grid_y = np.mgrid[0:shape[0]:1, 0:shape[1]:1, ] aot = myInterpolator(grid_x, grid_y) aot = smoothn(aot, isrobust=True, TolZ=1e-6, W=100 * ((aot >= aot_min) & (aot <= aot_max)), s=10, MaxIter=1000)[0] else: aot = np.nanmedian(aot) * np.ones(shape) if tcwv_name is not None: g = gdal.Open(tcwv_bands[0]) ySize, xSize = tcwv.shape dst = gdal.GetDriverByName('GTiff').Create( tcwv_name, xSize, ySize, 1, gdal.GDT_UInt16, options=["TILED=YES", "COMPRESS=DEFLATE"]) dst.SetGeoTransform(g.GetGeoTransform()) dst.SetProjection(g.GetProjection()) dst.GetRasterBand(1).WriteArray((tcwv * 1000).astype(int)) dst.GetRasterBand(1).SetNoDataValue(65535) dst = None g = None if aot_name is not None: g = gdal.Open(aot_bands[0]) ySize, xSize = aot.shape dst = gdal.GetDriverByName('GTiff').Create( aot_name, xSize, ySize, 1, gdal.GDT_UInt16, options=["TILED=YES", "COMPRESS=DEFLATE"]) dst.SetGeoTransform(g.GetGeoTransform()) dst.SetProjection(g.GetProjection()) dst.GetRasterBand(1).WriteArray((aot * 1000).astype(int)) dst.GetRasterBand(1).SetNoDataValue(65535) dst = None g = None return aot, tcwv
def __init__( self, sensor_sat, toa_bands, band_wv, band_index, view_angles, sun_angles, obs_time, cloud_mask, gamma=10., a_z_order=1, pixel_res=None, aoi=None, aot_prior=None, wv_prior=None, o3_prior=None, aot_unc=None, wv_unc=None, o3_unc=None, log_file=None, ref_scale=0.0001, ref_off=0, ang_scale=0.01, ele_scale=0.001, prior_scale=[1., 0.1, 46.698, 1., 1., 1.], emus_dir='SIAC/emus/', mcd43_dir='~/DATA/Multiply/MCD43/', global_dem='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', cams_dir='/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', spec_m_dir='SIAC/spectral_mapping', aero_res=1000): self.sensor = sensor_sat[0] self.satellite = sensor_sat[1] self.toa_bands = toa_bands self.band_wv = band_wv self.band_index = band_index self.view_angles = view_angles self.sun_angles = sun_angles self.obs_time = obs_time self.cloud_mask = cloud_mask self.gamma = gamma self.a_z_order = a_z_order self.pixel_res = pixel_res self.aoi = aoi self.aot_prior = aot_prior self.wv_prior = wv_prior self.o3_prior = o3_prior self.aot_unc = aot_unc self.wv_unc = wv_unc self.o3_unc = o3_unc self.log_file = log_file self.ref_scale = ref_scale self.ref_off = ref_off self.ang_scale = ang_scale self.ele_scale = ele_scale self.prior_scale = prior_scale self.mcd43_dir = mcd43_dir self.emus_dir = emus_dir self.global_dem = global_dem self.cams_dir = cams_dir self.boa_wv = [645, 859, 469, 555, 1640, 2130] self.aero_res = aero_res self.mcd43_tmp = '%s/MCD43A1.A%d%03d.%s.006.*.hdf' self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1])) try: #spec_map = np.loadtxt(spec_m_dir + '/Aqua_%s_spectral_mapping.txt'%self.satellite).T self.spec_map = Two_NN( np_model_file=spec_m_dir + '/Aqua_%s_spectral_mapping.npz' % self.satellite) except: #spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T self.spec_map = Two_NN( np_model_file=spec_m_dir + '/Aqua_%s_spectral_mapping.npz' % self.sensor) #self.spec_slope = spec_map[0] #self.spec_off = spec_map[1] self.logger = create_logger(self.log_file)