def _parse_atmo(self,): atmos = [self.aot, self.tcwv, self.tco3] atmo_uncs = [self.aot_unc, self.tcwv_unc, self.tco3_unc] atmo_scale = self.atmo_scale cams_names = ['aod550', 'tcwv', 'gtco3'] defalt_uncs = [0.4, 0.1, 0.05] if self.obs_time is not None: time_ind = np.abs((self.obs_time.hour + self.obs_time.minute/60. + \ self.obs_time.second/3600.) - np.arange(0,25,3)).argmin() for i in range(3): if atmos[i] is None: atmos[i] = self.cams_dir + '/'.join([datetime.strftime(self.obs_time, '%Y_%m_%d'),\ datetime.strftime(self.obs_time, '%Y_%m_%d')+'_%s.tif'%cams_names[i]]) var_g = self._var_parser(atmos[i]) _g = reproject_data(var_g, self.mg, srcNodata = None, resample = \ gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).g offset = var_g.GetOffset() scale = var_g.GetScale() data = _g.GetRasterBand(int(time_ind+1)).ReadAsArray() * scale + offset atmos[i] = data * self.cams_scale[i] if atmo_uncs[i] is None: atmo_uncs[i] = defalt_uncs[i] else: for i in range(3): var_g = self._var_parser(atmos[i]) data = reproject_data(var_g, self.mg, srcNodata = np.nan, resample = \ gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data atmos[i] = data * self.atmo_scale[i] unc_g = self._var_parser(atmo_uncs[i]) ug = reproject_data(unc_g, self.mg, srcNodata = np.nan, resample = \ gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data atmo_uncs[i] = ug self._aot, self._tcwv, self._tco3 = atmos self._aot_unc, self._tcwv_unc, self._tco3_unc = atmo_uncs
def get_control_variables(self, ): aod = reproject_data(self.mod_l1b_dir+'/atmo_paras/' + \ self.example_file.split('/')[-1].split('_EV_')[0]+'_EV_aod550.tif', self.example_file).data tcwv = reproject_data(self.mod_l1b_dir+'/atmo_paras/' + \ self.example_file.split('/')[-1].split('_EV_')[0]+'_EV_tcwv.tif', self.example_file).data tco3 = reproject_data(self.mod_l1b_dir+'/atmo_paras/' +\ self.example_file.split('/')[-1].split('_EV_')[0]+'_EV_tco3.tif', self.example_file).data ele = reproject_data(self.global_dem, self.example_file).data mask = ~np.isfinite(ele) if mask.sum() > 0: ele[mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), ele[~mask]) # simple interpolation return aod, tcwv, tco3, ele
def do_cloud(cloud_bands, cloud_name=None): toas = [ reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data for band in cloud_bands ] toas = np.array(toas) / 10000. mask = np.all(toas >= 0.0001, axis=0) valid_pixel = mask.sum() if valid_pixel > 0: cloud_proba = cl.predict_proba(toas[:, mask].T) cloud_mask = np.zeros_like(toas[0]) cloud_mask[mask] = cloud_proba[:, 1] else: cloud_mask = np.zeros_like(toas[0]) cloud_mask[~mask] = 2.56 if cloud_name is None: return cloud_mask else: g = gdal.Open(cloud_bands[0]) dst = gdal.GetDriverByName('GTiff').Create( cloud_name, g.RasterXSize, g.RasterYSize, 1, gdal.GDT_Byte, options=["TILED=YES", "COMPRESS=DEFLATE"]) dst.SetGeoTransform(g.GetGeoTransform()) dst.SetProjection(g.GetProjection()) dst.GetRasterBand(1).WriteArray((cloud_mask * 100).astype(int)) dst.GetRasterBand(1).SetNoDataValue(256) dst = None g = None return cloud_mask
def _read_cams(self, example_file, parameters=['aod550', 'tcwv', 'gtco3']): netcdf_file = datetime.datetime(self.sen_time.year, self.sen_time.month, \ self.sen_time.day).strftime("%Y-%m-%d.nc") template = 'NETCDF:"%s":%s' ind = np.abs((self.sen_time.hour + self.sen_time.minute/60. + \ self.sen_time.second/3600.) - np.arange(0,25,3)).argmin() sr = osr.SpatialReference() sr.ImportFromEPSG(4326) proj = sr.ExportToWkt() results = [] for para in parameters: fname = template % (self.cams_dir + '/' + netcdf_file, para) g = gdal.Open(fname) g.SetProjection(proj) sub = g.GetRasterBand(ind + 1) offset = sub.GetOffset() scale = sub.GetScale() bad_pix = int(sub.GetNoDataValue()) rep_g = reproject_data(g, example_file, outputType=gdal.GDT_Float32).g data = rep_g.GetRasterBand(ind + 1).ReadAsArray() data = data * scale + offset mask = (data == (bad_pix * scale + offset)) if mask.sum() >= 1: data[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), data[~mask]) results.append(data) return results
def _get_control_variables(self, ): aot = reproject_data(self.l8_toa_dir + '/%s_%s.tif'%(self.l8_header, 'aot'), \ self.example_file, outputType= gdal.GDT_Float32).data tcwv = reproject_data(self.l8_toa_dir + '/%s_%s.tif'%(self.l8_header, 'tcwv'), \ self.example_file, outputType= gdal.GDT_Float32).data tco3 = reproject_data(self.l8_toa_dir + '/%s_%s.tif'%(self.l8_header, 'tco3'), \ self.example_file, outputType= gdal.GDT_Float32).data ele = reproject_data(self.global_dem, self.example_file, outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele) ele[mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), ele[~mask]) # simple interpolation return aot, tcwv, tco3, ele
def get_control_variables(self, target_band): aod = reproject_data(self.s2.s2_file_dir+'/aot.tif', \ self.s2.s2_file_dir+'/%s.jp2'%target_band, outputType= gdal.GDT_Float32).data tcwv = reproject_data(self.s2.s2_file_dir+'/tcwv.tif', \ self.s2.s2_file_dir+'/%s.jp2'%target_band, outputType= gdal.GDT_Float32).data tco3 = reproject_data(self.s2.s2_file_dir+'/tco3.tif', \ self.s2.s2_file_dir+'/%s.jp2'%target_band, outputType= gdal.GDT_Float32).data ele = reproject_data(self.global_dem, self.s2.s2_file_dir + '/%s.jp2' % target_band, outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele) ele[mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), ele[~mask]) # simple interpolation return aod, tcwv, tco3, ele.astype(float)
def _resamplers(self, ): ''' Define the sresamplers used for resampling different gdal files or objects to required resolution and size. ''' self.nearest_resampler = lambda fname: reproject_data(fname, \ self.example_file, \ xRes=self.aero_res*0.5, \ yRes=self.aero_res*0.5, \ srcNodata = None,\ outputType= gdal.GDT_Float32, \ resample = gdal.GRIORA_NearestNeighbour).g self.bilinear_resampler = lambda fname: reproject_data(fname, \ self.example_file, \ xRes=self.aero_res*0.5, \ yRes=self.aero_res*0.5, \ srcNodata = 0,\ outputType= gdal.GDT_Float32,\ resample = gdal.GRIORA_Bilinear).g
def _mcd08_aot(self,): temp = 'HDF4_EOS:EOS_GRID:"%s":mod08:Aerosol_Optical_Depth_Land_Ocean_Mean' try: g = gdal.Open(temp%glob('%s/MOD08_D3.A2016%03d.006.*.hdf'%(self.mod08_dir, self.doy))[0]) dat = reproject_data(g, self.example_file, outputType= gdal.GDT_Float32).data * g.GetRasterBand(1).GetScale() + g.GetRasterBand(1).GetOffset() dat[dat<=0] = np.nan dat[dat>1.7] = np.nan mod08_aot = np.nanmean(dat) except: mod08_aot = np.nan try: g1 = gdal.Open(temp%glob('%s/MYD08_D3.A2016%03d.006.*.hdf'%(self.myd08_dir, self.doy))[0]) dat1 = reproject_data(g1, self.example_file, outputType= gdal.GDT_Float32).data * g1.GetRasterBand(1).GetScale() + g1.GetRasterBand(1).GetOffset() dat1[dat1<=0] = np.nan dat1[dat1>1.7] = np.nan myd08_aot = np.nanmean(dat1) except: myd08_aot = np.nan return mod08_aot, myd08_aot
def _parse_aux(self,): auxs = [self.global_dem, self.mask.astype(float)] scales = [self.ele_scale, 1] for i in range(len(auxs)): var_g = self._var_parser(auxs[i]) dat = reproject_data(var_g, self.mg, srcNodata = 0, resample = \ gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data auxs[i] = dat * scales[i] self._ele, self._cmask = auxs self._ele[np.isnan(self._ele)] = 0 self._cmask[np.isnan(self._cmask)] = 0 self._cmask = binary_dilation(self._cmask, structure = np.ones((3,3)).astype(bool)).astype(bool)
def _annoying_angles(self,): _sun_angles = [] _view_angles = [] for fname in self._sun_angles: ang = reproject_data(fname, self.mg, srcNodata = None, resample = \ gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _sun_angles.append(ang) for fname in self._view_angles: ang = reproject_data(fname, self.mg, srcNodata = None, resample = \ gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _view_angles.append(ang) _view_angles = np.array(_view_angles) _sun_angles = np.squeeze(np.array(_sun_angles)) if len(self._view_angles) == len(self.toa_bands): if self.a_z_order == 1: self._vaa = _view_angles[:,0] self._vza = _view_angles[:,1] else: self._vaa = _view_angles[:,1] self._vza = _view_angles[:,0] elif len(self._view_angles) == 2*len(self.toa_bands): self._vaa = _view_angles[:len(self.toa_bands)] self._vza = _view_angles[len(self.toa_bands):] if os.path.exists(str(self.sun_angles)): if self.a_z_order == 1: self._saa = _sun_angles[0] self._sza = _sun_angles[1] else: self._saa = _sun_angles[1] self._sza = _sun_angles[0] elif len(self.sun_angles) == 2: self._saa = _sun_angles[0] self._sza = _sun_angles[1] self._sza = self._sza * self.ang_scale self._saa = self._saa * self.ang_scale self._vza = self._vza * self.ang_scale self._vaa = self._vaa * self.ang_scale self._vaa, self._vza, self._saa, self._sza = map(np.deg2rad, [self._vaa, self._vza, self._saa, self._sza])
def _mod08_aot(self, ): try: temp = 'HDF4_EOS:EOS_GRID:"%s":mod08:Aerosol_Optical_Depth_Land_Ocean_Mean' g = gdal.Open(temp % glob('%s/MOD08_D3.A2016%03d.006.*.hdf' % (self.mod08_dir, self.doy))[0]) dat = reproject_data( g, self.s2_file_dir + '/B01.jp2', outputType=gdal.GDT_Float32).data * g.GetRasterBand( 1).GetScale() + g.GetRasterBand(1).GetOffset() dat[dat <= 0] = np.nan dat[dat > 1.5] = np.nan mod08_aot = np.nanmean(dat) except: mod08_aot = np.nan return mod08_aot
def _mcd43_cloud(self, flist, lx, ly, example_file, boa, b12): g = gdal.BuildVRT('', list(flist)) temp_data = np.zeros((g.RasterYSize, g.RasterXSize)) temp_data[lx, ly] = boa[5, :] self.boa_b12 = reproject_data(array_to_raster(temp_data, g), example_file, outputType=gdal.GDT_Float32).data toa_b12 = np.repeat(np.repeat(b12 / 10000., 2, axis=0), 2, axis=1) mask = abs(self.boa_b12 - toa_b12) > 0.1 emask = binary_erosion(mask, structure=np.ones((3, 3)).astype(bool), iterations=15) dmask = binary_dilation(emask, structure=np.ones((3, 3)).astype(bool), iterations=100) self.cloud = self.s2.cloud | dmask
def _modis_aerosol(self, ): self.modis_logger.info('Getting emualated surface reflectance.') vza, sza, vaa, saa = self.modis_angle self.modis_boa, self.modis_boa_qa, self.brdf_stds = get_brdf_six( self.mcd43_file, angles=[vza, sza, vaa - saa], bands=(1, 2, 3, 4, 5, 6, 7), Linds=None) if self.mod_cloud is None: self.modis_cloud = np.zeros_like(self.modis_toa[0]).astype(bool) self.modis_logger.info('Getting elevation.') ele = reproject_data(self.global_dem, self.example_file) ele.get_it() mask = ~np.isfinite(ele.data) ele.data = np.ma.array(ele.data, mask=mask) self.elevation = ele.data / 1000. self.modis_logger.info('Getting pripors from ECMWF forcasts.') aod, tcwv, tco3 = self._read_cams(self.example_file) self.aod550 = aod * (1 - 0.14) # validation of +14% biase self.tco3 = tco3 * 46.698 * (1 - 0.05) self.tcwv = tcwv / 10. self.tco3_unc = np.ones(self.tco3.shape) * 0.2 self.aod550_unc = np.ones(self.aod550.shape) * 0.5 self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 qua_mask = np.all(self.modis_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.modis_boa.mask, axis = 0) &\ np.all(self.modis_boa>0, axis=0) &\ np.all(self.modis_boa<1, axis=0) toa_mask = np.all(np.isfinite(self.modis_toa), axis=0) &\ np.all(self.modis_toa>0, axis=0) & \ np.all(self.modis_toa<1, axis=0) self.modis_mask = qua_mask & boa_mask & toa_mask & (~self.modis_cloud) self.modis_AEE, self.modis_bounds = self._load_emus(self.modis_sensor) self.modis_solved = []
def _do_chunk(self, ind): toa_g = self._toa_bands[ind] geo = toa_g.GetGeoTransform() x_size, y_size = toa_g.RasterXSize, toa_g.RasterYSize boas = [] uncs = [] for i in range(self._chunks): r = i * 1. / self._chunks x_start = int(r * x_size) xmin = geo[0] + r * x_size * geo[1] r = (i+1.)/ self._chunks x_end = int(r * x_size) xmax = geo[0] + r * x_size * geo[1] x_off = x_end - x_start toa = toa_g.ReadAsArray(x_start, 0, x_off, int(y_size)) _g = gdal.Warp('', toa_g, format = 'MEM', outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = gdal.GRIORA_NearestNeighbour) xRes = yRes = int(geo[1]) xps = [self.xap_H[ind], self.xbp_H[ind], self.xcp_H[ind]] for j in range(3): xps[j] = reproject_data(xps[j], _g,xRes=xRes, yRes = yRes,xSize=x_off, \ ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, outputType= gdal.GDT_Float32).data xap_H, xbp_H, xcp_H = xps toa = toa * self.ref_scale + self.ref_off y = xap_H * toa - xbp_H boa = y / (1 + xcp_H * y) xhs = [self.xap_dH[ind], self.xbp_dH[ind], self.xcp_dH[ind]] for j in range(3): #var_g = xhs[j] xhs[j] = reproject_data(xhs[j], _g, xRes=xRes, yRes = yRes, xSize=x_off, \ ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, \ outputType= gdal.GDT_Float32).data.transpose(1,2,0) xap_dH, xbp_dH, xcp_dH = xhs del xps; del xhs dH = -1 * (-toa[...,None] * xap_dH - \ 2 * toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \ toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \ xbp_dH + \ xbp_H[...,None]**2 * xcp_dH) / \ (toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \ xbp_H[...,None] * xcp_H[...,None] + 1)**2 toa_dH = xap_H / (xcp_H*(toa * xap_H - xbp_H) + 1)**2 del xap_H; del xbp_H; del xcp_H; del xap_dH; del xbp_dH; del xcp_dH aot_dH, tcwv_dH, tco3_dH = [dH[:, :,i] for i in range(3)] del dH tmp = [] for unc_g in [self._aot_unc, self._tcwv_unc, self._tco3_unc]: unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, outputType= gdal.GDT_Float32).data tmp.append(unc) aot_unc, tcwv_unc, tco3_unc = tmp; del tmp unc = np.sqrt(aot_dH ** 2 * aot_unc**2 + tcwv_dH ** 2 * tcwv_unc**2 + tco3_dH ** 2 * tco3_unc**2 + toa_dH**2 * 0.015**2) del aot_unc; del tcwv_unc; del tco3_unc boas.append(boa) uncs.append(unc) boas, uncs = np.hstack(boas), np.hstack(uncs) boa_name = self.toa_dir + '/' + '.'.join(self.toa_bands[ind].split('/')[-1].split('.')[0:-1]) + '_sur.tif' unc_name = boa_name.replace('_sur.tif', '_sur_unc.tif') projectionRef = toa_g.GetProjectionRef() geotransform = toa_g.GetGeoTransform() self._save_band(boas, boa_name, projectionRef, geotransform) self._save_band(uncs, unc_name, projectionRef, geotransform) if ind in [self.ri, self.gi, self.bi]: return boas else: return None
def _s2_aerosol(self, ): self.s2_logger.propagate = False self.s2_logger.info('Start to retrieve atmospheric parameters.') self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month, self.day, self.s2_u_bands) self.s2_logger.info('Reading in TOA reflectance.') selected_img = self.s2.get_s2_toa() self.s2_file_dir = self.s2.s2_file_dir self.example_file = self.s2.s2_file_dir + '/B04.jp2' self.s2.get_s2_cloud() self.s2_logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.s2_logger.info( 'Getting the angles and simulated surface reflectance.') self.s2.get_s2_angles() sa_files = [self.s2.angles['saa'], self.s2.angles['sza']] if os.path.exists(self.s2_file_dir + '/angles/'): va_files = [ self.s2_file_dir + '/angles/VAA_VZA_B%02d.img' % i for i in [2, 3, 4, 8, 11, 12] ] else: va_files = [ np.array( [self.s2.angles['vaa'][band], self.s2.angles['vza'][band]]) for band in self.s2_u_bands[:-2] ] if len(glob(self.s2_file_dir + '/MCD43.npz')) == 0: boa, unc, hx, hy, lx, ly, flist = MCD43_SurRef(self.mcd43_dir, self.example_file, \ self.year, self.doy, [sa_files, va_files], sun_view_ang_scale=[1, 0.01], bands = [3,4,1,2,6,7], tolz=0.001, reproject=False) np.savez(self.s2_file_dir + '/MCD43.npz', boa=boa, unc=unc, hx=hx, hy=hy, lx=lx, ly=ly, flist=flist) else: f = np.load(self.s2_file_dir + '/MCD43.npz') boa, unc, hx, hy, lx, ly, flist = f['boa'], f['unc'], f['hx'], f[ 'hy'], f['lx'], f['ly'], f['flist'] self.Hx, self.Hy = hx, hy self.s2_logger.info('Update cloud mask.') flist = [ f.split('/')[0] + self.mcd43_dir + '/' + f.split('/')[-1] for f in flist ] self._mcd43_cloud(flist, lx, ly, self.example_file, boa, selected_img['B12']) self.s2_logger.info('Applying spectral transform.') self.s2_boa_qa = np.ma.array(unc) self.s2_boa = np.ma.array(boa)*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \ np.array(self.s2_spectral_transform)[1,:-1][...,None] shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \ self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks) self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1)) self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1)) self.vza = [] self.vaa = [] for band in self.s2_u_bands[:-2]: self.vza.append( self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1))) self.vaa.append( self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1))) self.vza = np.array(self.vza) self.vaa = np.array(self.vaa) self.raa = self.saa[None, ...] - self.vaa self.s2_logger.info('Getting elevation.') ele_data = reproject_data(self.global_dem, self.example_file, outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \ self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1)) self.s2_logger.info('Getting pripors from ECMWF forcasts.') sen_time_str = json.load( open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp'] self.sen_time = datetime.datetime.strptime(sen_time_str, u'%Y-%m-%dT%H:%M:%S.%fZ') aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)).reshape((3, self.num_blocks, \ self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2)) mod08_aot = self._mod08_aot() #if np.isnan(mod08_aot): self.aot = aot - (aot.mean() - np.nanmean([mod08_aot, aot.mean()])) #else: # temp = np.zeros_like(aot) # temp[:] = mod08_aot # self.aot = temp self.tco3 = tco3 * 46.698 tcwv = tcwv / 10. self.tco3_unc = np.ones(self.tco3.shape) * 0.2 self.aot_unc = np.ones(self.aot.shape) * 2. self.s2_logger.info( 'Trying to get the tcwv from the emulation of sen2cor look up table.' ) try: self._get_tcwv(selected_img, self.s2.angles['vza'], self.s2.angles['vaa'], self.s2.angles['sza'], self.s2.angles['saa'], ele_data) except: self.s2_logger.warning( 'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.' ) self.tcwv = tcwv self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 ''' self.s2_logger.info('Trying to get the aot from ddv method.') try: solved = self._get_ddv_aot(selected_img) if solved[0] < 0: self.s2_logger.warning('DDV failed and only cams data used for the prior.') else: self.s2_logger.info('DDV solved aot is %.02f, and it will used as the mean value of cams prediction.'%solved[0]) self.aot[:] = solved[0] #self.aot + (solved[0]-self.aot.mean()) except: self.s2_logger.warning('Getting aot from ddv failed.') ''' self.s2_logger.info( 'Mean values for priors are: %.02f, %.02f, %.02f and mod08 aot is %.02f' % (aot.mean(), self.tcwv.mean(), self.tco3.mean(), mod08_aot)) self.s2_logger.info('Applying PSF model.') if self.s2_psf is None: xstd, ystd, ang, xs, ys = self._get_psf(selected_img) else: xstd, ystd, ang, xs, ys = self.s2_psf # apply psf shifts without going out of the image extend shifted_mask = np.logical_and.reduce( ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]), (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0]))) self.Hx, self.Hy = self.Hx[shifted_mask] + int( xs), self.Hy[shifted_mask] + int(ys) #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask] self.s2_boa = self.s2_boa[:, shifted_mask] self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask] self.s2_logger.info('Getting the convolved TOA reflectance.') #self.valid_pixs = sum(shifted_mask) # count how many pixels is still within the s2 tile ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd))) #self.bad_pixs = np.zeros(self.valid_pixs).astype(bool) imgs = [] for i, band in enumerate(self.s2_u_bands[:-2]): if selected_img[band].shape != self.full_res: imgs.append( self.repeat_extend(selected_img[band], shape=self.full_res)) else: imgs.append(selected_img[band]) border_mask = np.zeros(self.full_res).astype(bool) border_mask[[0, -1], :] = True border_mask[:, [0, -1]] = True self.dcloud = binary_dilation(self.cloud | border_mask, structure=np.ones((3, 3)).astype(bool), iterations=ker_size / 2).astype(bool) self.bad_pixs = self.dcloud[self.Hx, self.Hy] del selected_img del self.s2.selected_img del self.s2.angles['vza'] del self.s2.angles['vaa'] del self.s2.angles['sza'] del self.s2.angles['saa'] del self.s2.sza del self.s2.saa del self.s2 ker = self.gaussian(xstd, ystd, ang) f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self .Hy] * 0.0001 half = parmap(f, imgs[:3]) self.s2_toa = np.array(half + parmap(f, imgs[3:])) del imgs # get the valid value masks qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\ np.all(self.s2_boa >= 0.001, axis = 0) &\ np.all(self.s2_boa < 1, axis = 0) toa_mask = (~self.bad_pixs) &\ np.all(self.s2_toa >= 0.0001, axis = 0) &\ np.all(self.s2_toa < 1., axis = 0) self.s2_mask = boa_mask & toa_mask & qua_mask self.Hx = self.Hx[self.s2_mask] self.Hy = self.Hy[self.s2_mask] self.s2_toa = self.s2_toa[:, self.s2_mask] self.s2_boa = self.s2_boa[:, self.s2_mask] self.s2_boa_unc = self.s2_boa_qa[:, self.s2_mask] self.s2_logger.info('Solving...') tempm = np.zeros(self.full_res) tempm[self.Hx, self.Hy] = 1 tempm = tempm.reshape(self.num_blocks, self.block_size, \ self.num_blocks, self.block_size).astype(int).sum(axis=(3,1)) mask = ~self.dcloud self.mask = mask.reshape(self.num_blocks, self.block_size, \ self.num_blocks, self.block_size).astype(int).sum(axis=(3,1)) self.mask = ((self.mask / ((1. * self.block_size)**2)) >= 0.5) & ( (tempm / ((self.aero_res / 500.)**2)) >= 0.5) self.mask = binary_erosion(self.mask, structure=np.ones((3, 3)).astype(bool)) self.aero = solving_atmo_paras( self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa, self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc, self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy, self.mask, self.full_res, self.aero_res, self.emus, self.band_indexs, self.boa_bands) solved = self.aero._optimization() return solved
def _l8_aerosol(self, ): self.logger.propagate = False self.logger.info('Start to retrieve atmospheric parameters.') l8 = read_l8(self.l8_toa_dir, self.l8_tile, self.year, self.month, self.day, bands=self.bands) l8._get_angles() self.logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.logger.info( 'Find corresponding pixels between L8 and MODIS tiles') self.example_file = self.l8_toa_dir + '/%s_b%d.tif' % (l8.header, 1) tiles = Find_corresponding_pixels(self.example_file, destination_res=500) if len(tiles.keys()) > 1: self.logger.info('This Landsat 8 tile covers %d MODIS tile.' % len(tiles.keys())) self.mcd43_files = [] boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], [] for key in tiles.keys()[1:]: self.logger.info('Getting BOA from MODIS tile: %s.' % key) mcd43_file = glob(self.mcd43_tmp % (self.mcd43_dir, self.year, self.doy, key))[0] self.mcd43_files.append(mcd43_file) self.H_inds, self.L_inds = tiles[key] Lx, Ly = self.L_inds Hx, Hy = self.H_inds Hxs.append(Hx) Hys.append(Hy) vza, sza = l8.vza[:, Hx, Hy], l8.sza[:, Hx, Hy] vaa, saa = l8.vaa[:, Hx, Hy], l8.saa[:, Hx, Hy] raa = vaa - saa boa, boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\ bands=(3,4,1,2,6,7), Linds= [Lx, Ly]) boas.append(boa) boa_qas.append(boa_qa) brdf_stds.append(brdf_std) self.boa = np.hstack(boas) self.boa_qa = np.hstack(boa_qas) self.brdf_stds = np.hstack(brdf_stds) self.logger.info('Applying spectral transform.') self.boa = self.boa*np.array(self.spectral_transform)[0][...,None] + \ np.array(self.spectral_transform)[1][...,None] self.Hx = np.hstack(Hxs) self.Hy = np.hstack(Hys) self.sza = l8.sza[:, self.Hx, self.Hy] self.vza = l8.vza[:, self.Hx, self.Hy] self.saa = l8.saa[:, self.Hx, self.Hy] self.vaa = l8.vaa[:, self.Hx, self.Hy] self.logger.info('Reading in TOA reflectance.') toa = l8._get_toa() self.toa = toa[:, self.Hx, self.Hy] self.sen_time = l8.sen_time self.logger.info('Getting elevation.') ele_data = reproject_data(self.global_dem, self.example_file).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.logger.info('Getting pripors from ECMWF forcasts.') aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)) self._get_ddv_aot(toa, l8, tcwv, tco3, ele_data) aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)) self.aot = aot[self.Hx, self.Hy] #* (1-0.14) # validation of +14% biase self.tco3 = tco3[self.Hx, self.Hy] #* (1 - 0.05) self.tcwv = tcwv[self.Hx, self.Hy] self.aot_unc = np.ones(self.aot.shape) * 0.5 self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 self.tco3_unc = np.ones(self.tco3.shape) * 0.2
def _s2_aerosol(self, ): self.s2_logger.propagate = False self.s2_logger.info('Start to retrieve atmospheric parameters.') self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month, self.day, self.s2_u_bands) self.s2_logger.info('Reading in TOA reflectance.') selected_img = self.s2.get_s2_toa() self.s2_file_dir = self.s2.s2_file_dir self.s2.get_s2_cloud() self.s2_logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.s2_logger.info( 'Find corresponding pixels between S2 and MODIS tiles') tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2', destination_res=500) if len(tiles.keys()) > 1: self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' % len(tiles.keys())) self.mcd43_files = [] boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], [] self.s2_logger.info( 'Getting the angles and simulated surface reflectance.') for key in tiles.keys(): self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key) mcd43_file = glob(self.mcd43_tmp % (self.mcd43_dir, self.year, self.doy, key))[0] self.mcd43_files.append(mcd43_file) self.H_inds, self.L_inds = tiles[key] Lx, Ly = self.L_inds Hx, Hy = self.H_inds Hxs.append(Hx) Hys.append(Hy) self.s2.get_s2_angles(self.reconstruct_s2_angle) self.s2_angles = np.zeros((4, 6, len(Hx))) for j, band in enumerate(self.s2_u_bands[:-2]): self.s2_angles[[0,2],j,:] = (self.s2.angles['vza'][band])[Hx, Hy], \ (self.s2.angles['vaa'][band])[Hx, Hy] self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \ self.s2.angles['saa'][Hx, Hy] #use mean value to fill bad values for i in range(4): mask = ~np.isfinite(self.s2_angles[i]) if mask.sum() > 0: self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), \ self.s2_angles[i][~mask]) # simple interpolation vza, sza = self.s2_angles[:2] vaa, saa = self.s2_angles[2:] raa = vaa - saa # get the simulated surface reflectance s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\ bands=(3,4,1,2,6,7), Linds= [Lx, Ly]) boas.append(s2_boa) boa_qas.append(s2_boa_qa) brdf_stds.append(brdf_std) self.s2_boa = np.hstack(boas) self.s2_boa_qa = np.hstack(boa_qas) self.brdf_stds = np.hstack(brdf_stds) self.s2_logger.info('Applying spectral transform.') self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \ np.array(self.s2_spectral_transform)[1,:-1][...,None] self.Hx = np.hstack(Hxs) self.Hy = np.hstack(Hys) del sza del vza del saa del vaa del raa del mask del boas del boa_qas del brdf_stds del Hxs del Hys shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \ self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks) self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1)) self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1)) self.vza = [] self.vaa = [] for band in self.s2_u_bands[:-2]: self.vza.append( self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1))) self.vaa.append( self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1))) self.vza = np.array(self.vza) self.vaa = np.array(self.vaa) self.raa = self.saa[None, ...] - self.vaa self.s2_logger.info('Getting elevation.') example_file = self.s2.s2_file_dir + '/B04.jp2' ele_data = reproject_data(self.global_dem, example_file, outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \ self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1)) self.s2_logger.info('Getting pripors from ECMWF forcasts.') sen_time_str = json.load( open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp'] self.sen_time = datetime.datetime.strptime(sen_time_str, u'%Y-%m-%dT%H:%M:%S.%fZ') aot, tcwv, tco3 = np.array(self._read_cams(example_file)).reshape((3, self.num_blocks, \ self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2)) self.aot = aot #* (1-0.14) # validation of +14% biase self.tco3 = tco3 * 46.698 #* (1 - 0.05) tcwv = tcwv / 10. self.tco3_unc = np.ones(self.tco3.shape) * 0.2 self.aot_unc = np.ones(self.aot.shape) * 0.5 self.s2_logger.info( 'Trying to get the tcwv from the emulation of sen2cor look up table.' ) try: self._get_tcwv(selected_img, self.s2.angles['vza'], self.s2.angles['vaa'], self.s2.angles['sza'], self.s2.angles['saa'], ele_data) except: self.s2_logger.warning( 'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.' ) self.tcwv = tcwv self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 self.s2_logger.info('Trying to get the aot from ddv method.') try: solved = self._get_ddv_aot(selected_img) if solved[0] < 0: self.s2_logger.warning( 'DDV failed and only cams data used for the prior.') else: self.s2_logger.info( 'DDV solved aot is %.02f, and it will used as the mean value of cams prediction.' % solved[0]) self.aot += (solved[0] - self.aot.mean()) except: self.s2_logger.warning('Getting aot from ddv failed.') self.s2_logger.info('Applying PSF model.') if self.s2_psf is None: xstd, ystd, ang, xs, ys = self._get_psf(selected_img) else: xstd, ystd, ang, xs, ys = self.s2_psf # apply psf shifts without going out of the image extend shifted_mask = np.logical_and.reduce( ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]), (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0]))) self.Hx, self.Hy = self.Hx[shifted_mask] + int( xs), self.Hy[shifted_mask] + int(ys) #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask] self.s2_boa = self.s2_boa[:, shifted_mask] self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask] self.brdf_stds = self.brdf_stds[:, shifted_mask] self.s2_logger.info('Getting the convolved TOA reflectance.') self.valid_pixs = sum( shifted_mask) # count how many pixels is still within the s2 tile ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd))) self.bad_pixs = np.zeros(self.valid_pixs).astype(bool) imgs = [] for i, band in enumerate(self.s2_u_bands[:-2]): if selected_img[band].shape != self.full_res: imgs.append( self.repeat_extend(selected_img[band], shape=self.full_res)) else: imgs.append(selected_img[band]) border_mask = np.zeros(self.full_res).astype(bool) border_mask[[0, -1], :] = True border_mask[:, [0, -1]] = True self.bad_pixs = cloud_dilation(self.s2.cloud | border_mask, iteration=ker_size / 2)[self.Hx, self.Hy] del selected_img del self.s2.selected_img del self.s2.angles['vza'] del self.s2.angles['vaa'] del self.s2.angles['sza'] del self.s2.angles['saa'] del self.s2.sza del self.s2.saa del self.s2 ker = self.gaussian(xstd, ystd, ang) f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self .Hy] * 0.0001 half = parmap(f, imgs[:3]) self.s2_toa = np.array(half + parmap(f, imgs[3:])) del imgs # get the valid value masks qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\ np.all(self.s2_boa > 0, axis = 0) &\ np.all(self.s2_boa < 1, axis = 0) toa_mask = (~self.bad_pixs) &\ np.all(self.s2_toa > 0, axis = 0) &\ np.all(self.s2_toa < 1, axis = 0) self.s2_mask = boa_mask & toa_mask & qua_mask self.Hx = self.Hx[self.s2_mask] self.Hy = self.Hy[self.s2_mask] self.s2_toa = self.s2_toa[:, self.s2_mask] self.s2_boa = self.s2_boa[:, self.s2_mask] self.s2_boa_qa = self.s2_boa_qa[:, self.s2_mask] self.brdf_stds = self.brdf_stds[:, self.s2_mask] self.s2_boa_unc = grab_uncertainty(self.s2_boa, self.boa_bands, self.s2_boa_qa, self.brdf_stds).get_boa_unc() self.s2_logger.info('Solving...') self.aero = solving_atmo_paras( self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa, self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc, self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy, self.full_res, self.aero_res, self.emus, self.band_indexs, self.boa_bands) solved = self.aero._optimization() return solved
def get_s2_angles(self, reconstruct=True, slic=None): tree = ET.parse(self.s2_file_dir + '/metadata.xml') root = tree.getroot() #Sun_Angles_Grid saa = [] sza = [] msz = [] msa = [] #Viewing_Incidence_Angles_Grids vza = {} vaa = {} mvz = {} mva = {} for child in root: for j in child: for k in j.findall('Sun_Angles_Grid'): for l in k.findall('Zenith'): for m in l.findall('Values_List'): for x in m.findall('VALUES'): sza.append(x.text.split()) for n in k.findall('Azimuth'): for o in n.findall('Values_List'): for p in o.findall('VALUES'): saa.append(p.text.split()) for ms in j.findall('Mean_Sun_Angle'): self.msz = float(ms.find('ZENITH_ANGLE').text) self.msa = float(ms.find('AZIMUTH_ANGLE').text) for k in j.findall('Viewing_Incidence_Angles_Grids'): for l in k.findall('Zenith'): for m in l.findall('Values_List'): vza_sub = [] for x in m.findall('VALUES'): vza_sub.append(x.text.split()) bi, di, angles = k.attrib['bandId'], \ k.attrib['detectorId'], np.array(vza_sub).astype(float) vza[(int(bi), int(di))] = angles for n in k.findall('Azimuth'): for o in n.findall('Values_List'): vaa_sub = [] for p in o.findall('VALUES'): vaa_sub.append(p.text.split()) bi, di, angles = k.attrib['bandId'],\ k.attrib['detectorId'], np.array(vaa_sub).astype(float) vaa[(int(bi), int(di))] = angles for mvia in j.findall('Mean_Viewing_Incidence_Angle_List'): for i in mvia.findall('Mean_Viewing_Incidence_Angle'): mvz[int(i.attrib['bandId'])] = float( i.find('ZENITH_ANGLE').text) mva[int(i.attrib['bandId'])] = float( i.find('AZIMUTH_ANGLE').text) sza = np.array(sza).astype(float) saa = np.array(saa).astype(float) saa[saa > 180] = saa[saa > 180] - 360 mask = np.isnan(sza) sza = griddata(np.array(np.where(~mask)).T, sza[~mask], \ (np.repeat(range(23), 23).reshape(23,23), \ np.tile (range(23), 23).reshape(23,23)), method='nearest') mask = np.isnan(saa) saa = griddata(np.array(np.where(~mask)).T, saa[~mask], \ (np.repeat(range(23), 23).reshape(23,23), \ np.tile (range(23), 23).reshape(23,23)), method='nearest') self.saa, self.sza = np.repeat(np.repeat(np.array(saa), 500, axis = 0), 500, axis = 1)[:10980, :10980], \ np.repeat(np.repeat(np.array(sza), 500, axis = 0), 500, axis = 1)[:10980, :10980] dete_id = np.unique([i[1] for i in vaa.keys()]) band_id = range(13) bands_vaa = [] bands_vza = [] for i in band_id: band_vaa = np.zeros((23, 23)) band_vza = np.zeros((23, 23)) band_vaa[:] = np.nan band_vza[:] = np.nan for j in dete_id: try: good = ~np.isnan(vaa[(i, j)]) band_vaa[good] = vaa[(i, j)][good] good = ~np.isnan(vza[(i, j)]) band_vza[good] = vza[(i, j)][good] except: pass bands_vaa.append(band_vaa) bands_vza.append(band_vza) bands_vaa, bands_vza = np.array(bands_vaa), np.array(bands_vza) vaa = {} vza = {} mva_ = {} mvz_ = {} for i, band in enumerate(self.s2_bands): vaa[band] = bands_vaa[i] vza[band] = bands_vza[i] try: mva_[band] = mva[i] mvz_[band] = mvz[i] except: mva_[band] = np.nan mvz_[band] = np.nan if self.bands is None: bands = self.s2_bands else: bands = self.bands self.vza = {} self.vaa = {} self.mvz = {} self.mva = {} for band in bands: mask = np.isnan(vza[band]) g_vza = griddata(np.array(np.where(~mask)).T, vza[band][~mask], \ (np.repeat(range(23), 23).reshape(23,23), \ np.tile (range(23), 23).reshape(23,23)), method='nearest') mask = np.isnan(vaa[band]) g_vaa = griddata(np.array(np.where(~mask)).T, vaa[band][~mask], \ (np.repeat(range(23), 23).reshape(23,23), \ np.tile (range(23), 23).reshape(23,23)), method='nearest') self.vza[band] = np.repeat(np.repeat(g_vza, 500, axis=0), 500, axis=1)[:10980, :10980] g_vaa[g_vaa > 180] = g_vaa[g_vaa > 180] - 360 self.vaa[band] = np.repeat(np.repeat(g_vaa, 500, axis=0), 500, axis=1)[:10980, :10980] self.mvz[band] = mvz_[band] self.mva[band] = mva_[band] self.angles = {'sza':self.sza, 'saa':self.saa, 'msz':self.msz, 'msa':self.msa,\ 'vza':self.vza, 'vaa': self.vaa, 'mvz':self.mvz, 'mva':self.mva} if reconstruct: try: if len(glob(self.s2_file_dir + '/angles/VAA_VZA_*.img')) == 13: pass else: #print 'Reconstructing Sentinel 2 angles...' subprocess.call(['python', './python/s2a_angle_bands_mod.py', \ self.s2_file_dir+'/metadata.xml', '10']) if self.bands is None: bands = self.s2_bands else: bands = self.bands self.vaa = {} self.vza = {} fname = [ self.s2_file_dir + '/angles/VAA_VZA_%s.img' % band for band in bands ] if len(glob(self.s2_file_dir + '/angles/VAA_VZA_*.img')) == 13: f = lambda fn: reproject_data( fn, self.s2_file_dir + '/B04.jp2', outputType=gdal.GDT_Float32).data ret = parmap(f, fname) for i, angs in enumerate(ret): #angs[0][angs[0]<0] = (36000 + angs[0][angs[0]<0]) angs = angs.astype(float) / 100. if slic is None: self.vaa[bands[i]] = angs[0] self.vza[bands[i]] = angs[1] else: x_ind, y_ind = np.array(slic) self.vaa[bands[i]] = angs[0][x_ind, y_ind] self.vza[bands[i]] = angs[1][x_ind, y_ind] self.angles = {'sza':self.sza, 'saa':self.saa, 'msz':self.msz, 'msa':self.msa,\ 'vza':self.vza, 'vaa': self.vaa, 'mvz':self.mvz, 'mva':self.mva} else: print 'Reconstruct failed and original angles are used.' except: print 'Reconstruct failed and original angles are used.'
def _mask_bad_pix(self): #snow_mask = blue > 0.6 if os.path.exists(str(self.cloud_mask)): cloud_g = gdal.Open(str(self.cloud_mask)) elif type(self.cloud_mask).__module__ == 'numpy': cloud_g = array_to_raster(self.cloud_mask, self.example_file) cloud = reproject_data(cloud_g, \ self.example_file, \ xRes=self.pixel_res, \ yRes=self.pixel_res, \ xSize=self.full_res[1], \ ySize=self.full_res[0], \ srcNodata = np.nan,\ outputType= gdal.GDT_Float32,\ resample = gdal.GRIORA_NearestNeighbour).data cloud = cloud.astype(bool) RED = None BLUE = None SWIR_1 = None NIR = None GREEN = None if 3 in self.boa_bands: BLUE = self._toa_bands[np.where(self.boa_bands == 3)[0][ 0]].ReadAsArray() * self.ref_scale + self.ref_off if BLUE is not None: water_mask = BLUE < 0.05 snow_mask = BLUE > 0.6 #del BLUE else: self.logger.error( 'Blue band is needed for the retirval of aerosol.') if 2 in self.boa_bands: NIR = self._toa_bands[np.where(self.boa_bands == 2)[0][ 0]].ReadAsArray() * self.ref_scale + self.ref_off if 1 in self.boa_bands: RED = self._toa_bands[np.where(self.boa_bands == 1)[0][ 0]].ReadAsArray() * self.ref_scale + self.ref_off if (RED is not None) & (NIR is not None): NDVI = (NIR - RED) / (NIR + RED) water_mask = ((NDVI < 0.01) & (NIR < 0.11)) | ((NDVI < 0.1) & (NIR < 0.05)) \ | (NIR <= 0.0001) | (RED <= 0.0001) | np.isnan(NIR) | np.isnan(RED) del NIR del RED del NDVI elif NIR is not None: water_mask = NIR < 0.05 del NIR elif RED is not None: water_mask = RED < 0.05 del RED if 6 in self.boa_bands: SWIR_1 = self._toa_bands[np.where(self.boa_bands == 6)[0][ 0]].ReadAsArray() * self.ref_scale + self.ref_off if 1 in self.boa_bands: GREEN = self._toa_bands[np.where(self.boa_bands == 4)[0][ 0]].ReadAsArray() * self.ref_scale + self.ref_off if (SWIR_1 is not None) & (GREEN is not None): NDSI = (GREEN - SWIR_1) / (SWIR_1 + GREEN) snow_mask = (NDSI > 0.42) | (SWIR_1 <= 0.0001) | ( GREEN <= 0.0001) | np.isnan(SWIR_1) | np.isnan(GREEN) del SWIR_1 del GREEN del NDSI mask = water_mask | snow_mask | cloud | (BLUE > 1) ker_size = int(2 * 1.96 * self.psf_ystd) mask = binary_erosion(mask, structure=np.ones((3, 3)).astype(bool), iterations=5).astype(bool) #mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool) mask = binary_dilation(mask, structure=np.ones((3, 3)).astype(bool), iterations=30 + ker_size).astype(bool) #mask[:30,:] = mask[:,:30] = mask[:,-30:] = mask[-30:,:] = True self.bad_pix = mask
def MCD43_SurRef(MCD43_dir, example_file, year, doy, ang_files, sun_view_ang_scale=[1, 1], bands=(7, ), tolz=0.001, reproject=False): f_temp = MCD43_dir + '/MCD43A1.A%s.%s.006*.hdf' temp1 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Parameters_Band%d' temp2 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Band_Mandatory_Quality_Band%d' unique_tile = get_hv(example_file) #print unique_tile date = datetime.strptime('%d%03d' % (year, doy), '%Y%j') days = [(date - timedelta(days = i)).strftime('%Y%j') for i in np.arange(16, 0, -1)] + \ [(date + timedelta(days = i)).strftime('%Y%j') for i in np.arange(0, 17, 1)] #data_f = [[temp1%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days] #qa_f = [[temp2%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days] fnames = np.array([[[temp1%(glob.glob(f_temp%(day, tile))[0], band), \ temp2%(glob.glob(f_temp%(day, tile))[0], band)] for \ tile in unique_tile] for band in bands for day in days]).transpose(0,2,1) g = gdal.Open(example_file) mg = reproject_data(example_file, gdal.BuildVRT('', list(fnames[0, 0])), outputType=gdal.GDT_Float64).g temp_data = ~np.isnan(mg.ReadAsArray()) geotransform = mg.GetGeoTransform() xgeo = geotransform[0] + np.arange(0.5, mg.RasterXSize, 1) * geotransform[1] ygeo = geotransform[3] + np.arange(0.5, mg.RasterYSize, 1) * geotransform[5] xgeo = np.repeat(xgeo[None, ...], mg.RasterYSize, axis=0) ygeo = np.repeat(ygeo[..., None], mg.RasterXSize, axis=1) m_proj = modis_sinu = osr.SpatialReference() m_proj.ImportFromProj4( "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs" ) h_proj = osr.SpatialReference() h_proj.ImportFromWkt(gdal.Open(example_file).GetProjection()) h_xgeo, h_ygeo, _ = np.array( osr.CoordinateTransformation(m_proj, h_proj).TransformPoints( zip(xgeo[temp_data], ygeo[temp_data]))).T geotransform = g.GetGeoTransform() hy = ((h_xgeo - geotransform[0]) / geotransform[1]).astype(int) hx = ((h_ygeo - geotransform[3]) / geotransform[5]).astype(int) hmask = (hx >= 0) & (hx < g.RasterYSize) & (hy >= 0) & (hy < g.RasterXSize) hy = hy[hmask] hx = hx[hmask] #print 'got vrt' max_x, max_y = np.array(np.where(temp_data)).max(axis=1) min_x, min_y = np.array(np.where(temp_data)).min(axis=1) xoff, yoff = min_y, min_x xsize, ysize = (max_y - min_y + 1), (max_x - min_x + 1) #print 'read in data' f = lambda fname: [gdal.BuildVRT('', list(fname[0])).ReadAsArray(xoff, yoff, xsize, ysize), \ gdal.BuildVRT('', list(fname[1])).ReadAsArray(xoff, yoff, xsize, ysize)] #f = lambda fname: gdal.BuildVRT('', fname).ReadAsArray(xoff, yoff, xsize, ysize) data, qa = np.array(parmap(f, fnames)).T data = np.concatenate(data).reshape( (len(bands), len(days), 3, ysize, xsize)).astype(float) data = np.ma.array(data, mask=(data == 32767)).astype(float) #test1, test2 = data.copy(), qa.copy() #global test1; global test2 w = 0.618034**np.concatenate(qa).reshape(len(bands), len(days), ysize, xsize).astype(float) #w = 0.618034 ** qa.astype(float) f = lambda band: np.array(smoothn(data[band[0],:,band[1],:,:], s=10., smoothOrder=1., \ axis=0, TolZ=tolz, verbose=False, isrobust=True, W = w[band[0]]))[[0,3],] ba = np.array( [np.tile(range(len(bands)), 3), np.repeat(range(3), len(bands))]).T #print 'smoothing....' smed = np.array(parmap(f, ba)) #global smed dat = np.concatenate(smed[:,0], axis=0).reshape(3, len(bands), len(days), ysize, \ xsize)[:,:,16, np.where(temp_data)[0]-min_x, np.where(temp_data)[1]-min_y] wei = np.concatenate(smed[:,1], axis=0).reshape(3, len(bands), len(days), ysize, \ xsize)[:,:,16, np.where(temp_data)[0]-min_x, np.where(temp_data)[1]-min_y] #std = data.std(axis = 1)[:, :, np.where(temp_data)[0]-min_x, np.where(temp_data)[1]-min_y] #print 'get angles...' sa_files, va_files = ang_files if isinstance(va_files[0], str): f = lambda ang_file: reproject_data(ang_file, gdal.BuildVRT( '', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data vas = np.array(parmap(f, va_files)) elif isinstance(va_files[0], (np.ndarray, np.generic)): f = lambda array: reproject_data(array_to_raster(array, example_file), gdal.BuildVRT('', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data vas = np.array(parmap(f, list(va_files))) vas = vas * sun_view_ang_scale[1] if isinstance(sa_files[0], str): f = lambda ang_file: reproject_data(ang_file, gdal.BuildVRT( '', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data sas = np.array(parmap(f, sa_files)) elif isinstance(sa_files[0], (np.ndarray, np.generic)): f = lambda array: reproject_data(array_to_raster(array, example_file), gdal.BuildVRT('', list(fnames[0, 0])), outputType=gdal.GDT_Float32).data sas = np.array(parmap(f, list(sa_files))) if sas.shape[0] == 2: sas = np.repeat((sas * sun_view_ang_scale[0])[None, ...], len(bands), axis=0) elif sas.shape[0] == len(bands): sas = sas * sun_view_ang_scale[0] else: raise IOError('Wrong shape of sun angles are given.') raa = vas[:, 0, :, :] - sas[:, 0, :, :] angles = vas[:, 1, temp_data], sas[:, 1, temp_data], raa[:, temp_data] kk = get_kk(angles) k_vol = kk.Ross k_geo = kk.Li sur_ref = (dat[0] + dat[1] * k_vol + dat[2] * k_geo) * 0.001 wei = 0.05 / wei #print wei unc = np.sqrt(wei[0, :, :]**2 + (wei[1, :, :]**2) * k_vol**2 + (wei[2, :, :]**2) * k_geo**2) #unc = np.sqrt((np.sqrt(std[:, 0, :]**2 + (std[:, 1, :]**2)*k_vol**2 + (std[:, 2, :]**2)*k_geo**2) * 0.001)**2 + \ # (np.sqrt(wei[0, :, :]**2 + (wei[1, :, :]**2)*k_vol**2 + (wei[2, :, :]**2)*k_geo**2))**2) unc = np.minimum(unc, 0.1) #print unc if reproject: f_dat = np.repeat(temp_data[None, ...], len(bands), axis=0).astype(float) f_dat[:] = np.nan unc_dat = f_dat.copy() f_dat[:, temp_data] = sur_ref unc_dat[:, temp_data] = unc f = lambda array: reproject_data(array_to_raster( array, gdal.BuildVRT('', list(fnames[0, 0]))), example_file, outputType=gdal.GDT_Float32).data f_dat = np.array(parmap(f, list(f_dat))) unc_dat = np.array(parmap(f, list(unc_dat))) mask = np.isnan(unc_dat) | (f_dat < 0.00001) f_dat[mask] = np.nan unc_dat[mask] = np.nan unc_dat[unc_dat == 0] = 0.1 return f_dat, unc_dat else: lx, ly = np.where(temp_data) sur_ref[sur_ref.mask] = np.nan unc[unc.mask] = 0.1 return sur_ref.data[:, hmask], unc.data[:, hmask], hx, hy, lx[ hmask], ly[hmask], fnames[0, 0]
def _s2_aerosol(self, ): self.s2_logger.propagate = False self.s2_logger.info('Start to retrieve atmospheric parameters.') self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month, self.day, self.s2_u_bands) self.s2_logger.info('Reading in TOA reflectance.') selected_img = self.s2.get_s2_toa() self.s2_file_dir = self.s2.s2_file_dir self.s2.get_s2_cloud() self.s2_logger.info( 'Find corresponding pixels between S2 and MODIS tiles') tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2', destination_res=500) if len(tiles.keys()) > 1: self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' % len(tiles.keys())) self.mcd43_files = [] szas, vzas, saas, vaas, raas = [], [], [], [], [] boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], [] for key in tiles.keys(): #h,v = int(key[1:3]), int(key[-2:]) self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key) mcd43_file = glob(self.mcd43_tmp % (self.mcd43_dir, self.year, self.doy, key))[0] self.mcd43_files.append(mcd43_file) self.H_inds, self.L_inds = tiles[key] Lx, Ly = self.L_inds Hx, Hy = self.H_inds Hxs.append(Hx) Hys.append(Hy) self.s2_logger.info( 'Getting the angles and simulated surface reflectance.') self.s2.get_s2_angles(self.reconstruct_s2_angle) self.s2_angles = np.zeros((4, 6, len(Hx))) for j, band in enumerate(self.s2_u_bands[:-2]): self.s2_angles[[0,2],j,:] = self.s2.angles['vza'][band][Hx, Hy], \ self.s2.angles['vaa'][band][Hx, Hy] self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \ self.s2.angles['saa'][Hx, Hy] #use mean value to fill bad values for i in range(4): mask = ~np.isfinite(self.s2_angles[i]) if mask.sum() > 0: self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), self.s2_angles[i][~mask]) # simple interpolation vza, sza = self.s2_angles[:2] vaa, saa = self.s2_angles[2:] raa = vaa - saa szas.append(sza) vzas.append(vza) raas.append(raa) vaas.append(vaa) saas.append(saa) # get the simulated surface reflectance s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\ bands=(3,4,1,2,6,7), Linds= [Lx, Ly]) boas.append(s2_boa) boa_qas.append(s2_boa_qa) brdf_stds.append(brdf_std) self.s2_boa = np.hstack(boas) self.s2_boa_qa = np.hstack(boa_qas) self.brdf_stds = np.hstack(brdf_stds) self.Hx = np.hstack(Hxs) self.Hy = np.hstack(Hys) vza = np.hstack(vzas) sza = np.hstack(szas) vaa = np.hstack(vaas) saa = np.hstack(saas) raa = np.hstack(raas) self.s2_angles = np.array([vza, sza, vaa, saa]) #self.s2_boa, self.s2_boa_qa = self.s2_boa.flatten(), self.s2_boa_qa.flatten() self.s2_logger.info('Applying spectral transform.') self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \ np.array(self.s2_spectral_transform)[1,:-1][...,None] self.s2_logger.info('Getting elevation.') ele_data = reproject_data(self.global_dem, self.s2.s2_file_dir + '/B04.jp2', outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.elevation = ele_data[self.Hx, self.Hy] self.s2_logger.info('Getting pripors from ECMWF forcasts.') sen_time_str = json.load( open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp'] self.sen_time = datetime.datetime.strptime(sen_time_str, u'%Y-%m-%dT%H:%M:%S.%fZ') example_file = self.s2.s2_file_dir + '/B04.jp2' aod, tcwv, tco3 = np.array(self._read_cams(example_file))[:, self.Hx, self.Hy] self.s2_aod550 = aod #* (1-0.14) # validation of +14% biase self.s2_tco3 = tco3 * 46.698 #* (1 - 0.05) tcwv = tcwv / 10. self.s2_tco3_unc = np.ones(self.s2_tco3.shape) * 0.2 self.s2_aod550_unc = np.ones(self.s2_aod550.shape) * 0.5 self.s2_logger.info( 'Trying to get the tcwv from the emulation of sen2cor look up table.' ) #try: self._get_tcwv(selected_img, vza, sza, raa, ele_data) #except: # self.s2_logger.warning('Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.') # self.s2_tcwv = tcwv # self.s2_tcwv_unc = np.ones(self.s2_tcwv.shape) * 0.2 self.s2_logger.info('Trying to get the aod from ddv method.') try: solved = self._get_ddv_aot(self.s2.angles, example_file, self.s2_tcwv, ele_data, selected_img) if solved[0] < 0: self.s2_logger.warning( 'DDV failed and only cams data used for the prior.') else: self.s2_logger.info( 'DDV solved aod is %.02f, and it will used as the mean value of cams prediction.' % solved[0]) self.s2_aod550 += (solved[0] - self.s2_aod550.mean()) except: self.s2_logger.warning('Getting aod from ddv failed.') self.s2_logger.info('Applying PSF model.') if self.s2_psf is None: self.s2_logger.info('No PSF parameters specified, start solving.') high_img = np.repeat( np.repeat(selected_img['B11'], 2, axis=0), 2, axis=1) * 0.0001 high_indexs = self.Hx, self.Hy low_img = self.s2_boa[4] qa, cloud = self.s2_boa_qa[4], self.s2.cloud psf = psf_optimize(high_img, high_indexs, low_img, qa, cloud, 2) xs, ys = psf.fire_shift_optimize() xstd, ystd = 29.75, 39 ang = 0 self.s2_logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \ %(xstd, ystd, 0, xs, ys, 1-psf.costs.min())) else: xstd, ystd, ang, xs, ys = self.s2_psf # apply psf shifts without going out of the image extend shifted_mask = np.logical_and.reduce( ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.s2_full_res[0]), (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.s2_full_res[0]))) self.Hx, self.Hy = self.Hx[shifted_mask] + int( xs), self.Hy[shifted_mask] + int(ys) #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask] self.s2_boa = self.s2_boa[:, shifted_mask] self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask] self.s2_angles = self.s2_angles[:, :, shifted_mask] self.elevation = self.elevation[shifted_mask] self.s2_aod550 = self.s2_aod550[shifted_mask] self.s2_tcwv = self.s2_tcwv[shifted_mask] self.s2_tco3 = self.s2_tco3[shifted_mask] self.s2_aod550_unc = self.s2_aod550_unc[shifted_mask] self.s2_tcwv_unc = self.s2_tcwv_unc[shifted_mask] self.s2_tco3_unc = self.s2_tco3_unc[shifted_mask] self.brdf_stds = self.brdf_stds[:, shifted_mask] self.s2_logger.info('Getting the convolved TOA reflectance.') self.valid_pixs = sum( shifted_mask) # count how many pixels is still within the s2 tile ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd))) self.bad_pixs = np.zeros(self.valid_pixs).astype(bool) imgs = [] for i, band in enumerate(self.s2_u_bands[:-2]): if selected_img[band].shape != self.s2_full_res: selected_img[band] = self.repeat_extend(selected_img[band], shape=self.s2_full_res) else: pass selected_img[band][0, :] = -9999 selected_img[band][-1, :] = -9999 selected_img[band][:, 0] = -9999 selected_img[band][:, -1] = -9999 imgs.append(selected_img[band]) # filter out the bad pixels self.bad_pixs |= cloud_dilation(self.s2.cloud |\ (selected_img[band] <= 0) | \ (selected_img[band] >= 10000),\ iteration= ker_size/2)[self.Hx, self.Hy] del selected_img del self.s2.selected_img del high_img del self.s2.angles del self.s2.sza del self.s2.saa del self.s2 ker = self.gaussian(xstd, ystd, ang) f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self .Hy] * 0.0001 half = parmap(f, imgs[:3]) self.s2_toa = np.array(half + parmap(f, imgs[3:])) #self.s2_toa = np.array(parmap(f,imgs)) del imgs # get the valid value masks qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\ np.all(self.s2_boa > 0, axis = 0) &\ np.all(self.s2_boa < 1, axis = 0) toa_mask = (~self.bad_pixs) &\ np.all(self.s2_toa > 0, axis = 0) &\ np.all(self.s2_toa < 1, axis = 0) self.s2_mask = boa_mask & toa_mask & qua_mask & (~self.elevation.mask) self.s2_AEE, self.s2_bounds = self._load_emus(self.s2_sensor)
def MCD43_SurRef(MCD43_dir, example_file, year, doy, ang_files, sun_view_ang_scale=[1, 1], bands=(7, ), tolz=0.001): f_temp = MCD43_dir + '/MCD43A1.A%s.%s.006*.hdf' temp1 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Parameters_Band%d' temp2 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Band_Mandatory_Quality_Band%d' unique_tile = get_hv(example_file) print unique_tile date = datetime.strptime('%d%03d' % (year, doy), '%Y%j') days = [(date - timedelta(days = i)).strftime('%Y%j') for i in np.arange(16, 0, -1)] + \ [(date + timedelta(days = i)).strftime('%Y%j') for i in np.arange(0, 17, 1)] #data_f = [[temp1%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days] #qa_f = [[temp2%(glob.glob(f_temp%(day, tile))[0], band) for tile in unique_tile] for band in bands for day in days] fnames = np.array([[[temp1%(glob.glob(f_temp%(day, tile))[0], band), \ temp2%(glob.glob(f_temp%(day, tile))[0], band)] for \ tile in unique_tile] for band in bands for day in days]).transpose(0,2,1) g = gdal.Open(example_file) temp_data = ~np.isnan( reproject_data(example_file, gdal.BuildVRT('', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data) print 'got vrt' max_x, max_y = np.array(np.where(temp_data)).max(axis=1) min_x, min_y = np.array(np.where(temp_data)).min(axis=1) xoff, yoff = min_y, min_x xsize, ysize = (max_y - min_y + 1), (max_x - min_x + 1) print 'read in data' f = lambda fname: [gdal.BuildVRT('', list(fname[0])).ReadAsArray(xoff, yoff, xsize, ysize), \ gdal.BuildVRT('', list(fname[1])).ReadAsArray(xoff, yoff, xsize, ysize)] #f = lambda fname: gdal.BuildVRT('', fname).ReadAsArray(xoff, yoff, xsize, ysize) data, qa = np.array(parmap(f, fnames)).T data = np.concatenate(data).reshape( (len(bands), len(days), 3, ysize, xsize)).astype(float) data = np.ma.array(data, mask=(data == 32767)).astype(float) w = np.repeat(0.618034**np.concatenate(qa).reshape( len(bands), len(days), ysize, xsize).astype(float)[None, ...], 3, axis=0).transpose(1, 2, 0, 3, 4) w[w < 0.3] = 0. print 'smoothing....' # instead of doing smoothn, we actually only need a laplace distribution filter for the middle date pdf = np.exp(-abs(np.arange(-16, 17, 1) - 0.) / 40.) / (2 * 40.) pdf = pdf / pdf.sum() w = w * pdf[None, ..., None, None, None] dat = ((w * data).sum(axis=1) / w.sum(axis=1))[:, :, np.where(temp_data)[0] - min_x, np.where(temp_data)[1] - min_y] wei = w[:, 16, 0, np.where(temp_data)[0] - min_x, np.where(temp_data)[1] - min_y] / pdf[16] std = data.std(axis=1)[:, :, np.where(temp_data)[0] - min_x, np.where(temp_data)[1] - min_y] print 'get angles...' va_files, sa_files = ang_files if isinstance(va_files[0], str): f = lambda ang_file: reproject_data(ang_file, gdal.BuildVRT( '', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data vas = np.array(parmap(f, va_files)) elif isinstance(sa_files[0], (np.ndarray, np.generic)): f = lambda array: reproject_data(array_to_raster(array, example_file), gdal.BuildVRT('', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data vas = np.array(parmap(f, list(va_files))) vas = vas * sun_view_ang_scale[1] if isinstance(sa_files[0], str): f = lambda ang_file: reproject_data(ang_file, gdal.BuildVRT( '', list(fnames[0, 0])), outputType=gdal.GDT_Float64).data sas = np.array(parmap(f, sa_files)) elif isinstance(sa_files[0], (np.ndarray, np.generic)): f = lambda array: reproject_data(array_to_raster(array, example_file), gdal.BuildVRT('', list(fnames[0, 0])), outputType=gdal.GDT_Float32).data sas = np.array(parmap(f, list(sa_files))) if sas.shape[0] == 2: sas = np.repeat((sas * sun_view_ang_scale[0])[None, ...], len(bands), axis=0) elif sas.shape[0] == len(bands): sas = sas * sun_view_ang_scale[0] else: raise IOError('Wrong shape of sun angles are given.') raa = vas[:, 0, :, :] - sas[:, 0, :, :] angles = vas[:, 1, temp_data], sas[:, 1, temp_data], raa[:, temp_data] kk = get_kk(angles) k_vol = kk.Ross k_geo = kk.Li sur_ref = (dat[:, 0] + dat[:, 1] * k_vol + dat[:, 2] * k_geo) * 0.001 wei = 0.05 / wei print wei.T unc = np.sqrt((np.sqrt(std[:, 0, :]**2 + (std[:, 1, :]**2) * k_vol**2 + (std[:, 2, :]**2) * k_geo**2) * 0.001)**2 + wei**2) unc = np.minimum(unc, 0.5) print unc f_dat = np.repeat(temp_data[None, ...], len(bands), axis=0).astype(float) f_dat[:] = np.nan unc_dat = f_dat.copy() f_dat[:, temp_data] = sur_ref unc_dat[:, temp_data] = unc f = lambda array: reproject_data(array_to_raster( array, gdal.BuildVRT('', list(fnames[0, 0]))), example_file, outputType=gdal.GDT_Float32).data f_dat = np.array(parmap(f, list(f_dat))) unc_dat = np.array(parmap(f, list(unc_dat))) mask = np.isnan(unc_dat) | (f_dat < 0.00001) f_dat[mask] = np.nan unc_dat[mask] = np.nan return f_dat, unc_dat
bandid.append(int(n.attrib['bandId'])) solar_irradiance = sorted(solar_irradiance, key = lambda i: bandid[solar_irradiance.index(i)]) bandid = sorted(bandid) return alpha, beta, gains, u_diff_temp, quant, u_sun, solar_irradiance, bandid, spacecraft def cal_unc(toa, sza, toa_band_id, alpha, beta, gains, u_diff_temp, quant, u_sun, solar_irradiance, bandid, spacecraft): rut_algo = s2_rut_algo.S2RutAlgo() rut_algo.a = gains[toa_band_id] rut_algo.beta = beta[toa_band_id] rut_algo.alpha = alpha[toa_band_id] rut_algo.tecta = sza rut_algo.e_sun = solar_irradiance[toa_band_id] rut_algo.u_sun = u_sun rut_algo.quant = quant rut_algo.u_diff_temp = u_diff_temp[toa_band_id] unc = rut_algo.unc_calculation(toa, toa_band_id, spacecraft) return unc b2 = '/data/nemesis/S2_data/S2A_MSIL1C_20181209T031111_N0207_R075_T50SMG_20181209T045755.SAFE/GRANULE/L1C_T50SMG_A018090_20181209T031113/IMG_DATA/T50SMG_20181209T031111_B01.jp2' sza = '/data/nemesis/S2_data/S2A_MSIL1C_20181209T031111_N0207_R075_T50SMG_20181209T045755.SAFE/GRANULE/L1C_T50SMG_A018090_20181209T031113/ANG_DATA/SAA_SZA.tif' #b2 = '/data/nemesis/S2_data/S2B_MSIL1C_20180611T110619_N0206_R137_T30UYD_20180611T170311.SAFE/GRANULE/L1C_T30UYD_A006598_20180611T110704/IMG_DATA/T30UYD_20180611T110619_B12.jp2' #sza = '/data/nemesis/S2_data/S2B_MSIL1C_20180611T110619_N0206_R137_T30UYD_20180611T170311.SAFE/GRANULE/L1C_T30UYD_A006598_20180611T110704/ANG_DATA/SAA_SZA.tif' sza = reproject_data(sza, b2, outputType= gdal.GDT_Float32).data[1]/100. b2 = gdal.Open(b2).ReadAsArray() coefs = get_coefs(product_meta_file, datasctrip_meta_file) unc = cal_unc(b2, sza, 12, *coefs)
def _l8_aerosol(self,): self.logger.propagate = False self.logger.info('Start to retrieve atmospheric parameters.') l8 = read_l8(self.l8_toa_dir, self.l8_tile, self.year, self.month, self.day, bands = self.bands) self.l8_header = l8.header self.logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.logger.info('Find corresponding pixels between L8 and MODIS tiles') self.example_file = self.l8_toa_dir + '/%s_b%d.tif'%(l8.header, 1) if len(glob(self.l8_toa_dir + '/MCD43_%s.npz'%(l8.header))) == 0: boa, unc, hx, hy, lx, ly, flist = MCD43_SurRef(self.mcd43_dir, self.example_file, \ self.year, self.doy, [l8.saa_sza, l8.vaa_vza], sun_view_ang_scale=[0.01, 0.01], bands = [3,4,1,2,6,7], tolz=0.003) np.savez(self.l8_toa_dir + 'MCD43_%s.npz'%l8.header, boa=boa, unc=unc, hx=hx, hy=hy, lx=lx, ly=ly, flist=flist) else: f = np.load(self.l8_toa_dir + 'MCD43_%s.npz'%l8.header) boa, unc, hx, hy, lx, ly, flist = f['boa'], f['unc'], f['hx'], f['hy'], f['lx'], f['ly'], f['flist'] self.Hx, self.Hy = hx, hy self.logger.info('Applying spectral transform.') self.boa_qa = np.ma.array(unc) self.boa = np.ma.array(boa)*np.array(self.spectral_transform)[0][...,None] + \ np.array(self.spectral_transform)[1][...,None] self.logger.info('Reading in TOA reflectance.') self.sen_time = l8.sen_time self.cloud = l8._get_qa() self.full_res = self.cloud.shape self.ecloud = binary_erosion(self.cloud, structure=np.ones((3,3)).astype(bool), iterations=10).astype(bool) border_mask = np.zeros(self.full_res).astype(bool) border_mask[[0, -1], :] = True border_mask[:, [0, -1]] = True xstd, ystd = 12., 20. ker_size = 2*int(round(max(1.96*xstd, 1.96*ystd))) self.dcloud = binary_dilation(self.ecloud | border_mask, structure=np.ones((3,3)).astype(bool), iterations=ker_size/2+10).astype(bool) self.logger.info('Getting elevation.') ele_data = reproject_data(self.global_dem, self.example_file, outputType = gdal.GDT_Float32).data/1000. mask = ~np.isfinite(ele_data) self.ele = np.ma.array(ele_data, mask = mask) self.ele[mask] = np.nan self.logger.info('Getting pripors from ECMWF forcasts.') self.aot, self.tcwv, self.tco3 = np.array(self._read_cams(self.example_file)) self.logger.info('Mean values of priors are: %.02f, %.02f, %.02f'%\ (np.nanmean(self.aot), np.nanmean(self.tcwv), np.nanmean(self.tco3))) self.toa = l8._get_toa() self.saa, self.sza, self.vaa, self.vza = l8._get_angles() self.saa[self.saa.mask] = self.sza[self.sza.mask] = \ self.vaa[self.vaa.mask] = self.vza[self.vza.mask] = np.nan self.logger.info('Getting DDV aot prior') self._get_ddv_aot(self.toa, l8, self.tcwv, self.tco3, ele_data) self.logger.info('Sorting data.') self.block_size = int(np.ceil(1. * self.aero_res / 30.)) self.num_blocks = int(np.ceil(max(self.full_res) / (1. * self.block_size))) self.efull_res = self.block_size * self.num_blocks shape1 = (self.num_blocks, self.block_size, self.num_blocks, self.block_size) shape2 = (self.vza.shape[0], self.num_blocks, self.block_size, self.num_blocks, self.block_size) self.ele = np.nanmean(self._extend_vals(self.ele ).reshape(shape1), axis=(3,1)) self.aot = np.nanmean(self._extend_vals(self.aot ).reshape(shape1), axis=(3,1)) self.tcwv = np.nanmean(self._extend_vals(self.tcwv).reshape(shape1), axis=(3,1)) self.tco3 = np.nanmean(self._extend_vals(self.tco3).reshape(shape1), axis=(3,1)) self.saa = np.nanmean(self._extend_vals(self.saa ).reshape(shape2), axis=(4,2)) self.sza = np.nanmean(self._extend_vals(self.sza ).reshape(shape2), axis=(4,2)) self.vaa = np.nanmean(self._extend_vals(self.vaa ).reshape(shape2), axis=(4,2)) self.vza = np.nanmean(self._extend_vals(self.vza ).reshape(shape2), axis=(4,2)) self.aot_unc = np.ones(self.aot.shape) * 0.8 self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 self.tco3_unc = np.ones(self.tco3.shape) * 0.2 #mod08_aot, myd08_aot = self._mcd08_aot() #self.logger.info('Mean values for priors are: %.02f, %.02f, %.02f and mod08 and myd08 aot are: %.02f, %.02f'%\ # (np.nanmean(self.aot), np.nanmean(self.tcwv), np.nanmean(self.tco3), mod08_aot, myd08_aot)) #if np.isnan(mod08_aot): self.aot[:] = np.nanmean(self.aot) #else: # temp = np.zeros_like(self.aot) # temp[:] = mod08_aot # self.aot = temp self.logger.info('Applying PSF model.') if self.l8_psf is None: xstd, ystd, ang, xs, ys = self._get_psf() else: xstd, ystd, ang, xs, ys = self.l8_psf shifted_mask = np.logical_and.reduce(((self.Hx+int(xs)>=0), (self.Hx+int(xs)<self.full_res[0]), (self.Hy+int(ys)>=0), (self.Hy+int(ys)<self.full_res[1]))) self.Hx, self.Hy = self.Hx[shifted_mask]+int(xs), self.Hy[shifted_mask]+int(ys) self.boa = self.boa [:, shifted_mask] self.boa_qa = self.boa_qa[:, shifted_mask] self.logger.info('Getting the convolved TOA reflectance.') self.bad_pixs = self.dcloud[self.Hx, self.Hy] ker = self.gaussian(xstd, ystd, ang) f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self.Hy] self.toa = np.array(parmap(f, list(self.toa))) qua_mask = np.all(self.boa_qa <= self.qa_thresh, axis = 0) boa_mask = np.all(~self.boa.mask,axis = 0 ) &\ np.all(self.boa >= 0.001, axis = 0) &\ np.all(self.boa < 1, axis = 0) toa_mask = (~self.bad_pixs) &\ np.all(self.toa >= 0.0001, axis = 0) &\ np.all(self.toa < 1., axis = 0) self.l8_mask = boa_mask & toa_mask & qua_mask self.Hx = self.Hx [self.l8_mask] self.Hy = self.Hy [self.l8_mask] self.toa = self.toa [:, self.l8_mask] self.boa = self.boa [:, self.l8_mask] self.boa_unc = self.boa_qa[:, self.l8_mask] self.logger.info('Solving...') tempm = np.zeros((self.efull_res, self.efull_res)) tempm[self.Hx, self.Hy] = 1 tempm = tempm.reshape(self.num_blocks, self.block_size, \ self.num_blocks, self.block_size).astype(int).sum(axis=(3,1)) self.mask = np.nansum(self._extend_vals((~self.dcloud).astype(int)).reshape(shape1), axis=(3,1)) self.mask = ((self.mask/((1.*self.block_size)**2)) > 0.) & ((tempm/((self.aero_res/500.)**2)) > 0.) & \ (np.any(~np.isnan([self.aot, self.tcwv, self.tco3, self.sza[0]]), axis = 0)) self.mask = binary_erosion(self.mask, structure=np.ones((5, 5)).astype(bool)) self.tcwv[~self.mask] = np.nanmean(self.tcwv) self.tco3[~self.mask] = np.nanmean(self.tco3) self.aero = solving_atmo_paras(self.boa, self.toa, self.sza, self.vza, self.saa, self.vaa, self.aot, self.tcwv, self.tco3, self.ele, self.aot_unc, self.tcwv_unc, self.tco3_unc, self.boa_unc, self.Hx, self.Hy, self.mask, (self.efull_res, self.efull_res), self.aero_res, self.emus, self.band_indexs, self.boa_bands, gamma = 2., alpha = -1.2, pix_res = 30) solved = self.aero._optimization() return solved