def _modis_aerosol(self, ): self.modis_logger.info('Getting emualated surface reflectance.') vza, sza, vaa, saa = self.modis_angle self.modis_boa, self.modis_boa_qa, self.brdf_stds = get_brdf_six( self.mcd43_file, angles=[vza, sza, vaa - saa], bands=(1, 2, 3, 4, 5, 6, 7), Linds=None) if self.mod_cloud is None: self.modis_cloud = np.zeros_like(self.modis_toa[0]).astype(bool) self.modis_logger.info('Getting elevation.') ele = reproject_data(self.global_dem, self.example_file) ele.get_it() mask = ~np.isfinite(ele.data) ele.data = np.ma.array(ele.data, mask=mask) self.elevation = ele.data / 1000. self.modis_logger.info('Getting pripors from ECMWF forcasts.') aod, tcwv, tco3 = self._read_cams(self.example_file) self.aod550 = aod * (1 - 0.14) # validation of +14% biase self.tco3 = tco3 * 46.698 * (1 - 0.05) self.tcwv = tcwv / 10. self.tco3_unc = np.ones(self.tco3.shape) * 0.2 self.aod550_unc = np.ones(self.aod550.shape) * 0.5 self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 qua_mask = np.all(self.modis_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.modis_boa.mask, axis = 0) &\ np.all(self.modis_boa>0, axis=0) &\ np.all(self.modis_boa<1, axis=0) toa_mask = np.all(np.isfinite(self.modis_toa), axis=0) &\ np.all(self.modis_toa>0, axis=0) & \ np.all(self.modis_toa<1, axis=0) self.modis_mask = qua_mask & boa_mask & toa_mask & (~self.modis_cloud) self.modis_AEE, self.modis_bounds = self._load_emus(self.modis_sensor) self.modis_solved = []
def _s2_aerosol(self, ): self.s2_logger.propagate = False self.s2_logger.info('Start to retrieve atmospheric parameters.') self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month, self.day, self.s2_u_bands) self.s2_logger.info('Reading in TOA reflectance.') selected_img = self.s2.get_s2_toa() self.s2_file_dir = self.s2.s2_file_dir self.s2.get_s2_cloud() self.s2_logger.info( 'Find corresponding pixels between S2 and MODIS tiles') tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2', destination_res=500) if len(tiles.keys()) > 1: self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' % len(tiles.keys())) self.mcd43_files = [] szas, vzas, saas, vaas, raas = [], [], [], [], [] boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], [] for key in tiles.keys(): #h,v = int(key[1:3]), int(key[-2:]) self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key) mcd43_file = glob(self.mcd43_tmp % (self.mcd43_dir, self.year, self.doy, key))[0] self.mcd43_files.append(mcd43_file) self.H_inds, self.L_inds = tiles[key] Lx, Ly = self.L_inds Hx, Hy = self.H_inds Hxs.append(Hx) Hys.append(Hy) self.s2_logger.info( 'Getting the angles and simulated surface reflectance.') self.s2.get_s2_angles(self.reconstruct_s2_angle) self.s2_angles = np.zeros((4, 6, len(Hx))) for j, band in enumerate(self.s2_u_bands[:-2]): self.s2_angles[[0,2],j,:] = self.s2.angles['vza'][band][Hx, Hy], \ self.s2.angles['vaa'][band][Hx, Hy] self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \ self.s2.angles['saa'][Hx, Hy] #use mean value to fill bad values for i in range(4): mask = ~np.isfinite(self.s2_angles[i]) if mask.sum() > 0: self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), self.s2_angles[i][~mask]) # simple interpolation vza, sza = self.s2_angles[:2] vaa, saa = self.s2_angles[2:] raa = vaa - saa szas.append(sza) vzas.append(vza) raas.append(raa) vaas.append(vaa) saas.append(saa) # get the simulated surface reflectance s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\ bands=(3,4,1,2,6,7), Linds= [Lx, Ly]) boas.append(s2_boa) boa_qas.append(s2_boa_qa) brdf_stds.append(brdf_std) self.s2_boa = np.hstack(boas) self.s2_boa_qa = np.hstack(boa_qas) self.brdf_stds = np.hstack(brdf_stds) self.Hx = np.hstack(Hxs) self.Hy = np.hstack(Hys) vza = np.hstack(vzas) sza = np.hstack(szas) vaa = np.hstack(vaas) saa = np.hstack(saas) raa = np.hstack(raas) self.s2_angles = np.array([vza, sza, vaa, saa]) #self.s2_boa, self.s2_boa_qa = self.s2_boa.flatten(), self.s2_boa_qa.flatten() self.s2_logger.info('Applying spectral transform.') self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \ np.array(self.s2_spectral_transform)[1,:-1][...,None] self.s2_logger.info('Getting elevation.') ele_data = reproject_data(self.global_dem, self.s2.s2_file_dir + '/B04.jp2', outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.elevation = ele_data[self.Hx, self.Hy] self.s2_logger.info('Getting pripors from ECMWF forcasts.') sen_time_str = json.load( open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp'] self.sen_time = datetime.datetime.strptime(sen_time_str, u'%Y-%m-%dT%H:%M:%S.%fZ') example_file = self.s2.s2_file_dir + '/B04.jp2' aod, tcwv, tco3 = np.array(self._read_cams(example_file))[:, self.Hx, self.Hy] self.s2_aod550 = aod #* (1-0.14) # validation of +14% biase self.s2_tco3 = tco3 * 46.698 #* (1 - 0.05) tcwv = tcwv / 10. self.s2_tco3_unc = np.ones(self.s2_tco3.shape) * 0.2 self.s2_aod550_unc = np.ones(self.s2_aod550.shape) * 0.5 self.s2_logger.info( 'Trying to get the tcwv from the emulation of sen2cor look up table.' ) #try: self._get_tcwv(selected_img, vza, sza, raa, ele_data) #except: # self.s2_logger.warning('Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.') # self.s2_tcwv = tcwv # self.s2_tcwv_unc = np.ones(self.s2_tcwv.shape) * 0.2 self.s2_logger.info('Trying to get the aod from ddv method.') try: solved = self._get_ddv_aot(self.s2.angles, example_file, self.s2_tcwv, ele_data, selected_img) if solved[0] < 0: self.s2_logger.warning( 'DDV failed and only cams data used for the prior.') else: self.s2_logger.info( 'DDV solved aod is %.02f, and it will used as the mean value of cams prediction.' % solved[0]) self.s2_aod550 += (solved[0] - self.s2_aod550.mean()) except: self.s2_logger.warning('Getting aod from ddv failed.') self.s2_logger.info('Applying PSF model.') if self.s2_psf is None: self.s2_logger.info('No PSF parameters specified, start solving.') high_img = np.repeat( np.repeat(selected_img['B11'], 2, axis=0), 2, axis=1) * 0.0001 high_indexs = self.Hx, self.Hy low_img = self.s2_boa[4] qa, cloud = self.s2_boa_qa[4], self.s2.cloud psf = psf_optimize(high_img, high_indexs, low_img, qa, cloud, 2) xs, ys = psf.fire_shift_optimize() xstd, ystd = 29.75, 39 ang = 0 self.s2_logger.info('Solved PSF parameters are: %.02f, %.02f, %d, %d, %d, and the correlation is: %f.' \ %(xstd, ystd, 0, xs, ys, 1-psf.costs.min())) else: xstd, ystd, ang, xs, ys = self.s2_psf # apply psf shifts without going out of the image extend shifted_mask = np.logical_and.reduce( ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.s2_full_res[0]), (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.s2_full_res[0]))) self.Hx, self.Hy = self.Hx[shifted_mask] + int( xs), self.Hy[shifted_mask] + int(ys) #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask] self.s2_boa = self.s2_boa[:, shifted_mask] self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask] self.s2_angles = self.s2_angles[:, :, shifted_mask] self.elevation = self.elevation[shifted_mask] self.s2_aod550 = self.s2_aod550[shifted_mask] self.s2_tcwv = self.s2_tcwv[shifted_mask] self.s2_tco3 = self.s2_tco3[shifted_mask] self.s2_aod550_unc = self.s2_aod550_unc[shifted_mask] self.s2_tcwv_unc = self.s2_tcwv_unc[shifted_mask] self.s2_tco3_unc = self.s2_tco3_unc[shifted_mask] self.brdf_stds = self.brdf_stds[:, shifted_mask] self.s2_logger.info('Getting the convolved TOA reflectance.') self.valid_pixs = sum( shifted_mask) # count how many pixels is still within the s2 tile ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd))) self.bad_pixs = np.zeros(self.valid_pixs).astype(bool) imgs = [] for i, band in enumerate(self.s2_u_bands[:-2]): if selected_img[band].shape != self.s2_full_res: selected_img[band] = self.repeat_extend(selected_img[band], shape=self.s2_full_res) else: pass selected_img[band][0, :] = -9999 selected_img[band][-1, :] = -9999 selected_img[band][:, 0] = -9999 selected_img[band][:, -1] = -9999 imgs.append(selected_img[band]) # filter out the bad pixels self.bad_pixs |= cloud_dilation(self.s2.cloud |\ (selected_img[band] <= 0) | \ (selected_img[band] >= 10000),\ iteration= ker_size/2)[self.Hx, self.Hy] del selected_img del self.s2.selected_img del high_img del self.s2.angles del self.s2.sza del self.s2.saa del self.s2 ker = self.gaussian(xstd, ystd, ang) f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self .Hy] * 0.0001 half = parmap(f, imgs[:3]) self.s2_toa = np.array(half + parmap(f, imgs[3:])) #self.s2_toa = np.array(parmap(f,imgs)) del imgs # get the valid value masks qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\ np.all(self.s2_boa > 0, axis = 0) &\ np.all(self.s2_boa < 1, axis = 0) toa_mask = (~self.bad_pixs) &\ np.all(self.s2_toa > 0, axis = 0) &\ np.all(self.s2_toa < 1, axis = 0) self.s2_mask = boa_mask & toa_mask & qua_mask & (~self.elevation.mask) self.s2_AEE, self.s2_bounds = self._load_emus(self.s2_sensor)
def _s2_aerosol(self, ): self.s2_logger.propagate = False self.s2_logger.info('Start to retrieve atmospheric parameters.') self.s2 = read_s2(self.s2_toa_dir, self.s2_tile, self.year, self.month, self.day, self.s2_u_bands) self.s2_logger.info('Reading in TOA reflectance.') selected_img = self.s2.get_s2_toa() self.s2_file_dir = self.s2.s2_file_dir self.s2.get_s2_cloud() self.s2_logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.s2_logger.info( 'Find corresponding pixels between S2 and MODIS tiles') tiles = Find_corresponding_pixels(self.s2.s2_file_dir + '/B04.jp2', destination_res=500) if len(tiles.keys()) > 1: self.s2_logger.info('This sentinel 2 tile covers %d MODIS tile.' % len(tiles.keys())) self.mcd43_files = [] boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], [] self.s2_logger.info( 'Getting the angles and simulated surface reflectance.') for key in tiles.keys(): self.s2_logger.info('Getting BOA from MODIS tile: %s.' % key) mcd43_file = glob(self.mcd43_tmp % (self.mcd43_dir, self.year, self.doy, key))[0] self.mcd43_files.append(mcd43_file) self.H_inds, self.L_inds = tiles[key] Lx, Ly = self.L_inds Hx, Hy = self.H_inds Hxs.append(Hx) Hys.append(Hy) self.s2.get_s2_angles(self.reconstruct_s2_angle) self.s2_angles = np.zeros((4, 6, len(Hx))) for j, band in enumerate(self.s2_u_bands[:-2]): self.s2_angles[[0,2],j,:] = (self.s2.angles['vza'][band])[Hx, Hy], \ (self.s2.angles['vaa'][band])[Hx, Hy] self.s2_angles[[1,3],j,:] = self.s2.angles['sza'][Hx, Hy], \ self.s2.angles['saa'][Hx, Hy] #use mean value to fill bad values for i in range(4): mask = ~np.isfinite(self.s2_angles[i]) if mask.sum() > 0: self.s2_angles[i][mask] = np.interp(np.flatnonzero(mask), \ np.flatnonzero(~mask), \ self.s2_angles[i][~mask]) # simple interpolation vza, sza = self.s2_angles[:2] vaa, saa = self.s2_angles[2:] raa = vaa - saa # get the simulated surface reflectance s2_boa, s2_boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\ bands=(3,4,1,2,6,7), Linds= [Lx, Ly]) boas.append(s2_boa) boa_qas.append(s2_boa_qa) brdf_stds.append(brdf_std) self.s2_boa = np.hstack(boas) self.s2_boa_qa = np.hstack(boa_qas) self.brdf_stds = np.hstack(brdf_stds) self.s2_logger.info('Applying spectral transform.') self.s2_boa = self.s2_boa*np.array(self.s2_spectral_transform)[0,:-1][...,None] + \ np.array(self.s2_spectral_transform)[1,:-1][...,None] self.Hx = np.hstack(Hxs) self.Hy = np.hstack(Hys) del sza del vza del saa del vaa del raa del mask del boas del boa_qas del brdf_stds del Hxs del Hys shape = (self.num_blocks, self.s2.angles['sza'].shape[0] / self.num_blocks, \ self.num_blocks, self.s2.angles['sza'].shape[1] / self.num_blocks) self.sza = self.s2.angles['sza'].reshape(shape).mean(axis=(3, 1)) self.saa = self.s2.angles['saa'].reshape(shape).mean(axis=(3, 1)) self.vza = [] self.vaa = [] for band in self.s2_u_bands[:-2]: self.vza.append( self.s2.angles['vza'][band].reshape(shape).mean(axis=(3, 1))) self.vaa.append( self.s2.angles['vaa'][band].reshape(shape).mean(axis=(3, 1))) self.vza = np.array(self.vza) self.vaa = np.array(self.vaa) self.raa = self.saa[None, ...] - self.vaa self.s2_logger.info('Getting elevation.') example_file = self.s2.s2_file_dir + '/B04.jp2' ele_data = reproject_data(self.global_dem, example_file, outputType=gdal.GDT_Float32).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.elevation = ele_data.reshape((self.num_blocks, ele_data.shape[0] / self.num_blocks, \ self.num_blocks, ele_data.shape[1] / self.num_blocks)).mean(axis=(3,1)) self.s2_logger.info('Getting pripors from ECMWF forcasts.') sen_time_str = json.load( open(self.s2.s2_file_dir + '/tileInfo.json', 'r'))['timestamp'] self.sen_time = datetime.datetime.strptime(sen_time_str, u'%Y-%m-%dT%H:%M:%S.%fZ') aot, tcwv, tco3 = np.array(self._read_cams(example_file)).reshape((3, self.num_blocks, \ self.block_size, self.num_blocks, self.block_size)).mean(axis=(4, 2)) self.aot = aot #* (1-0.14) # validation of +14% biase self.tco3 = tco3 * 46.698 #* (1 - 0.05) tcwv = tcwv / 10. self.tco3_unc = np.ones(self.tco3.shape) * 0.2 self.aot_unc = np.ones(self.aot.shape) * 0.5 self.s2_logger.info( 'Trying to get the tcwv from the emulation of sen2cor look up table.' ) try: self._get_tcwv(selected_img, self.s2.angles['vza'], self.s2.angles['vaa'], self.s2.angles['sza'], self.s2.angles['saa'], ele_data) except: self.s2_logger.warning( 'Getting tcwv from the emulation of sen2cor look up table failed, ECMWF data used.' ) self.tcwv = tcwv self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 self.s2_logger.info('Trying to get the aot from ddv method.') try: solved = self._get_ddv_aot(selected_img) if solved[0] < 0: self.s2_logger.warning( 'DDV failed and only cams data used for the prior.') else: self.s2_logger.info( 'DDV solved aot is %.02f, and it will used as the mean value of cams prediction.' % solved[0]) self.aot += (solved[0] - self.aot.mean()) except: self.s2_logger.warning('Getting aot from ddv failed.') self.s2_logger.info('Applying PSF model.') if self.s2_psf is None: xstd, ystd, ang, xs, ys = self._get_psf(selected_img) else: xstd, ystd, ang, xs, ys = self.s2_psf # apply psf shifts without going out of the image extend shifted_mask = np.logical_and.reduce( ((self.Hx + int(xs) >= 0), (self.Hx + int(xs) < self.full_res[0]), (self.Hy + int(ys) >= 0), (self.Hy + int(ys) < self.full_res[0]))) self.Hx, self.Hy = self.Hx[shifted_mask] + int( xs), self.Hy[shifted_mask] + int(ys) #self.Lx, self.Ly = self.Lx[shifted_mask], self.Ly[shifted_mask] self.s2_boa = self.s2_boa[:, shifted_mask] self.s2_boa_qa = self.s2_boa_qa[:, shifted_mask] self.brdf_stds = self.brdf_stds[:, shifted_mask] self.s2_logger.info('Getting the convolved TOA reflectance.') self.valid_pixs = sum( shifted_mask) # count how many pixels is still within the s2 tile ker_size = 2 * int(round(max(1.96 * xstd, 1.96 * ystd))) self.bad_pixs = np.zeros(self.valid_pixs).astype(bool) imgs = [] for i, band in enumerate(self.s2_u_bands[:-2]): if selected_img[band].shape != self.full_res: imgs.append( self.repeat_extend(selected_img[band], shape=self.full_res)) else: imgs.append(selected_img[band]) border_mask = np.zeros(self.full_res).astype(bool) border_mask[[0, -1], :] = True border_mask[:, [0, -1]] = True self.bad_pixs = cloud_dilation(self.s2.cloud | border_mask, iteration=ker_size / 2)[self.Hx, self.Hy] del selected_img del self.s2.selected_img del self.s2.angles['vza'] del self.s2.angles['vaa'] del self.s2.angles['sza'] del self.s2.angles['saa'] del self.s2.sza del self.s2.saa del self.s2 ker = self.gaussian(xstd, ystd, ang) f = lambda img: signal.fftconvolve(img, ker, mode='same')[self.Hx, self .Hy] * 0.0001 half = parmap(f, imgs[:3]) self.s2_toa = np.array(half + parmap(f, imgs[3:])) del imgs # get the valid value masks qua_mask = np.all(self.s2_boa_qa <= self.qa_thresh, axis=0) boa_mask = np.all(~self.s2_boa.mask,axis = 0 ) &\ np.all(self.s2_boa > 0, axis = 0) &\ np.all(self.s2_boa < 1, axis = 0) toa_mask = (~self.bad_pixs) &\ np.all(self.s2_toa > 0, axis = 0) &\ np.all(self.s2_toa < 1, axis = 0) self.s2_mask = boa_mask & toa_mask & qua_mask self.Hx = self.Hx[self.s2_mask] self.Hy = self.Hy[self.s2_mask] self.s2_toa = self.s2_toa[:, self.s2_mask] self.s2_boa = self.s2_boa[:, self.s2_mask] self.s2_boa_qa = self.s2_boa_qa[:, self.s2_mask] self.brdf_stds = self.brdf_stds[:, self.s2_mask] self.s2_boa_unc = grab_uncertainty(self.s2_boa, self.boa_bands, self.s2_boa_qa, self.brdf_stds).get_boa_unc() self.s2_logger.info('Solving...') self.aero = solving_atmo_paras( self.s2_boa, self.s2_toa, self.sza, self.vza, self.saa, self.vaa, self.aot, self.tcwv, self.tco3, self.elevation, self.aot_unc, self.tcwv_unc, self.tco3_unc, self.s2_boa_unc, self.Hx, self.Hy, self.full_res, self.aero_res, self.emus, self.band_indexs, self.boa_bands) solved = self.aero._optimization() return solved
def S2_PSF_optimization(self,): # open the created vrt file with 10 meter, 20 meter and 60 meter # grouped togehter and use gdal memory map to open it g = gdal.Open(self.s2_dir+'10meter.vrt') data= g.GetVirtualMemArray() b2,b3,b4,b8 = data g1 = gdal.Open(self.s2_dir+'20meter.vrt') data1 = g1.GetVirtualMemArray() b8a, b11, b12 = data1[-3:,:,:] img = dict(zip(self.bands, [b2,b3,b4,b8, b11, b12, b8a])) if glob(self.s2_dir+'cloud.tiff')==[]: cl = classification(img = img) cl.Get_cm_p() g=None; g1=None self.cloud = cl.cm g = gdal.Open(self.s2_dir+'B04.jp2') driver = gdal.GetDriverByName('GTiff') g1 = driver.Create(self.s2_dir+'cloud.tiff', g.RasterXSize, g.RasterYSize, 1, gdal.GDT_Byte) projection = g.GetProjection() geotransform = g.GetGeoTransform() g1.SetGeoTransform( geotransform ) g1.SetProjection( projection ) gcp_count = g.GetGCPs() if gcp_count != 0: g1.SetGCPs( gcp_count, g.GetGCPProjection() ) g1.GetRasterBand(1).WriteArray(self.cloud) g1=None; g=None del cl else: self.cloud = cloud = gdal.Open(self.s2_dir+'cloud.tiff').ReadAsArray().astype(bool) cloud_cover = 1.*self.cloud.sum()/self.cloud.size cloud_cover = 1.*self.cloud.sum()/self.cloud.size if cloud_cover > 0.2: print 'Too much cloud, cloud proportion: %.03f !!'%cloud_cover return [] else: mete = readxml('%smetadata.xml'%self.s2_dir) self.sza = np.zeros(7) self.sza[:] = mete['mSz'] self.saa = self.sza.copy() self.saa[:] = mete['mSa'] try: self.vza = (mete['mVz'])[[1,2,3,7,11,12,8],] self.vaa = (mete['mVa'])[[1,2,3,7,11,12,8],] except: self.vza = np.repeat(np.nanmean(mete['mVz']), 7) self.vaa = np.repeat(np.nanmean(mete['mVa']), 7) self.angles = (self.sza, self.vza, (self.vaa - self.saa)) tiles = Find_corresponding_pixels(self.s2_dir+'B04.jp2', destination_res=500) self.h,self.v = int(self.Lfile.split('.')[-4][1:3]), int(self.Lfile.split('.')[-4][4:]) self.H_inds, self.L_inds = tiles['h%02dv%02d'%(self.h, self.v)] self.Lx, self.Ly = self.L_inds self.Hx, self.Hy = self.H_inds angles = (self.sza[-2], self.vza[-2], (self.vaa - self.saa)[-2]) self.brdf, self.qa = get_brdf_six(self.Lfile, angles=angles, bands=(7,), Linds=list(self.L_inds)) self.brdf, self.qa = self.brdf.flatten(), self.qa.flatten() # convolve band 12 using the generally used PSF value self.H_data = np.repeat(np.repeat(b12, 2, axis=1), 2, axis=0) size = 2*int(round(max(1.96*50, 1.96*50)))# set the largest possible PSF size self.H_data[0,:]=self.H_data[-1,:]=self.H_data[:,0]=self.H_data[:,-1]=0 self.bad_pixs = cloud_dilation( (self.H_data <= 0) | self.cloud , iteration=size/2) xstd, ystd = 29.75, 39 ker = self.gaussian(xstd, ystd, 0) self.conved = signal.fftconvolve(self.H_data, ker, mode='same') m_mask = np.all(~self.brdf.mask,axis=0 ) & np.all(self.qa<=self.qa_thresh, axis=0) s_mask = ~self.bad_pixs[self.Hx, self.Hy] self.ms_mask = s_mask & m_mask '''self.in_patch_m = np.logical_and.reduce(((self.Hx>=self.patch[1]),
def _l8_aerosol(self, ): self.logger.propagate = False self.logger.info('Start to retrieve atmospheric parameters.') l8 = read_l8(self.l8_toa_dir, self.l8_tile, self.year, self.month, self.day, bands=self.bands) l8._get_angles() self.logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.logger.info( 'Find corresponding pixels between L8 and MODIS tiles') self.example_file = self.l8_toa_dir + '/%s_b%d.tif' % (l8.header, 1) tiles = Find_corresponding_pixels(self.example_file, destination_res=500) if len(tiles.keys()) > 1: self.logger.info('This Landsat 8 tile covers %d MODIS tile.' % len(tiles.keys())) self.mcd43_files = [] boas, boa_qas, brdf_stds, Hxs, Hys = [], [], [], [], [] for key in tiles.keys()[1:]: self.logger.info('Getting BOA from MODIS tile: %s.' % key) mcd43_file = glob(self.mcd43_tmp % (self.mcd43_dir, self.year, self.doy, key))[0] self.mcd43_files.append(mcd43_file) self.H_inds, self.L_inds = tiles[key] Lx, Ly = self.L_inds Hx, Hy = self.H_inds Hxs.append(Hx) Hys.append(Hy) vza, sza = l8.vza[:, Hx, Hy], l8.sza[:, Hx, Hy] vaa, saa = l8.vaa[:, Hx, Hy], l8.saa[:, Hx, Hy] raa = vaa - saa boa, boa_qa, brdf_std = get_brdf_six(mcd43_file, angles=[vza, sza, raa],\ bands=(3,4,1,2,6,7), Linds= [Lx, Ly]) boas.append(boa) boa_qas.append(boa_qa) brdf_stds.append(brdf_std) self.boa = np.hstack(boas) self.boa_qa = np.hstack(boa_qas) self.brdf_stds = np.hstack(brdf_stds) self.logger.info('Applying spectral transform.') self.boa = self.boa*np.array(self.spectral_transform)[0][...,None] + \ np.array(self.spectral_transform)[1][...,None] self.Hx = np.hstack(Hxs) self.Hy = np.hstack(Hys) self.sza = l8.sza[:, self.Hx, self.Hy] self.vza = l8.vza[:, self.Hx, self.Hy] self.saa = l8.saa[:, self.Hx, self.Hy] self.vaa = l8.vaa[:, self.Hx, self.Hy] self.logger.info('Reading in TOA reflectance.') toa = l8._get_toa() self.toa = toa[:, self.Hx, self.Hy] self.sen_time = l8.sen_time self.logger.info('Getting elevation.') ele_data = reproject_data(self.global_dem, self.example_file).data mask = ~np.isfinite(ele_data) ele_data = np.ma.array(ele_data, mask=mask) / 1000. self.logger.info('Getting pripors from ECMWF forcasts.') aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)) self._get_ddv_aot(toa, l8, tcwv, tco3, ele_data) aot, tcwv, tco3 = np.array(self._read_cams(self.example_file)) self.aot = aot[self.Hx, self.Hy] #* (1-0.14) # validation of +14% biase self.tco3 = tco3[self.Hx, self.Hy] #* (1 - 0.05) self.tcwv = tcwv[self.Hx, self.Hy] self.aot_unc = np.ones(self.aot.shape) * 0.5 self.tcwv_unc = np.ones(self.tcwv.shape) * 0.2 self.tco3_unc = np.ones(self.tco3.shape) * 0.2