def _get_boa(self, ): pix_mem = 180. if 'jasmin_memory_limit' in os.environ: jasmin_memory_limit = float( os.environ['jasmin_memory_limit']) - 10000 else: jasmin_memory_limit = psutil.virtual_memory().available + 10000 av_ram = min(psutil.virtual_memory().available - 10000, jasmin_memory_limit) needed = np.array( [i.RasterXSize * i.RasterYSize * pix_mem for i in self._toa_bands]) u_need = np.unique(needed) procs = av_ram / u_need if av_ram > sum(needed): #ret = parmap(self._do_band, range(len(self.toa_bands))) self._chunks = 1 ret = parmap(self._do_chunk, range(len(self.toa_bands))) else: ret = [] index = [] for i, proc in enumerate(procs): bands_to_do = np.arange(len( self.toa_bands))[needed == u_need[i]] if int(proc) >= 1: self._chunks = 1 re = parmap(self._do_chunk, bands_to_do, min(int(proc), len(bands_to_do))) else: self._chunks = int(np.ceil(1. / proc)) re = list(map(self._do_chunk, bands_to_do)) ret += re index += (np.where(needed == u_need[i])[0]).tolist() ret = list(zip(*sorted(zip(index, ret))))[1] self.boa_rgb = ret[self.ri], ret[self.gi], ret[self.bi]
def _fill_nan(self, ): self._vza = np.array(parmap(fill_nan, list(self._vza))) self._vaa = np.array(parmap(fill_nan, list(self._vaa))) self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3 = \ parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3]) self._aot = self._aot self._aot = np.maximum(self._aot, 0.01)
def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None, global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', \ cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', jasmin = False): if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') if len(glob(file_path + '/emus/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12: url = 'http://www2.geog.ucl.ac.uk/~ucfafyi/emus/' req = requests.get(url) to_down = [] for line in req.text.split(): if 'MSI' in line: fname = line.split('"')[1].split('<')[0] if 'MSI' in fname: to_down.append([fname, url]) f = lambda fname_url: downloader(fname_url[0], fname_url[1], file_path + '/emus/') parmap(f, to_down) rets = s2_pre_processing(s2_t) aero_atmos = [] for ret in rets: ret += (mcd43, vrt_dir, aoi, global_dem, cams_dir, jasmin) aero_atmo = do_correction(*ret) if send_back: aero_atmos.append(aero_atmo) if send_back: return aero_atmos
def SIAC_L8(l8_dir, send_back=False, mcd43=home + '/MCD43/', vrt_dir=home + '/MCD43_VRT/', aoi=None): file_path = os.path.dirname(os.path.realpath(__file__)) if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') #print(file_path) if len(glob(file_path + '/emus/' + 'isotropic_OLI_emulators_*_x?p_L8.pkl')) < 6: url = 'http://www2.geog.ucl.ac.uk/~ucfafyi/emus/' req = requests.get(url) to_down = [] for line in req.text.split(): if 'OLI' in line: fname = line.split('"')[1].split('<')[0] if ('OLI' in fname) & ('L8' in fname): to_down.append([fname, url]) f = lambda fname_url: downloader(fname_url[0], fname_url[1], file_path + '/emus/') parmap(f, to_down) rets = l8_pre_processing(l8_dir) aero_atmos = [] for ret in rets: ret += (mcd43, vrt_dir, aoi) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: aero_atmos.append(aero_atmo) if send_back: return aero_atmos
def _fill_nan(self,): self._vza = np.array(parmap(fill_nan, list(self._vza))) self._vaa = np.array(parmap(fill_nan, list(self._vaa))) self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc = \ parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc]) self._aot_unc = array_to_raster(self._aot_unc, self.example_file) self._tcwv_unc = array_to_raster(self._tcwv_unc, self.example_file) self._tco3_unc = array_to_raster(self._tco3_unc, self.example_file)
def _get_convolved_toa(self, ): imgs = [band_g.ReadAsArray() for band_g in self._toa_bands] self.bad_pixs = self.bad_pix[self.hx, self.hy] if self.full_res[0] % 2 != 0: xgaus = np.exp(-2. * (np.pi**2) * (self.psf_xstd**2) * ((0.5 * np.arange(self.full_res[0] + 1) / (self.full_res[0] + 1))**2)) else: xgaus = np.exp( -2. * (np.pi**2) * (self.psf_xstd**2) * ((0.5 * np.arange(self.full_res[0]) / self.full_res[0])**2)) if self.full_res[1] % 2 != 0: ygaus = np.exp(-2. * (np.pi**2) * (self.psf_ystd**2) * ((0.5 * np.arange(self.full_res[1] + 1) / (self.full_res[1] + 1))**2)) else: ygaus = np.exp( -2. * (np.pi**2) * (self.psf_ystd**2) * ((0.5 * np.arange(self.full_res[1]) / self.full_res[1])**2)) gaus_2d = np.outer(xgaus, ygaus) par = partial(convolve, gaus_2d=gaus_2d, hx=self.hx, hy=self.hy) if np.array(self.ref_scale).ndim == 2: self.ref_scale = self.ref_scale[self.hx, self.hy] if np.array(self.ref_off).ndim == 2: self.ref_off = self.ref_off[self.hx, self.hy] self.toa = np.array(parmap(par, imgs)) * self.ref_scale + self.ref_off
def _fill_nan(self,): def fill_nan(array): x_shp, y_shp = array.shape mask = ~np.isnan(array) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ np.tile (range(y_shp), x_shp).reshape(x_shp, y_shp) array = griddata(valid, value, mesh, method='nearest') return array self._vza = np.array(parmap(fill_nan, list(self._vza))) self._vaa = np.array(parmap(fill_nan, list(self._vaa))) self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3 = \ parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3]) self._aot = self._aot self._aot = np.maximum(self._aot, 0.01)
def _get_convolved_toa(self,): imgs = [band_g.ReadAsArray() for band_g in self._toa_bands] self.bad_pixs = self.bad_pix[self.hx, self.hy] if self.full_res[0] %2 != 0: xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0] + 1) /(self.full_res[0] + 1))**2)) else: xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0]) /self.full_res[0])**2)) if self.full_res[1] %2 != 0: ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1] + 1) /(self.full_res[1] + 1))**2)) else: ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1]) /self.full_res[1])**2)) gaus_2d = np.outer(xgaus, ygaus) def convolve(img, gaus_2d, hx, hy): x_size, y_size = img.shape if x_size % 2 != 0: img = np.insert(img, -1, img[-1, :], axis=0) if y_size % 2 != 0: img = np.insert(img, -1, img[:, -1], axis=1) dat = idct(idct(dct(dct(img, axis=0, norm = 'ortho'), axis=1, \ norm='ortho') * gaus_2d, axis=1, norm='ortho'), axis=0, norm='ortho')[hx, hy] return dat par = partial(convolve, gaus_2d = gaus_2d, hx = self.hx, hy = self.hy) if np.array(self.ref_scale).ndim ==2: self.ref_scale = self.ref_scale[self.hx, self.hy] if np.array(self.ref_off).ndim == 2: self.ref_off = self.ref_off[self.hx, self.hy] self.toa = np.array(parmap(par,imgs)) * self.ref_scale+self.ref_off
def fire_shift_optimize(self, ): #self.S2_PSF_optimization() self._preprocess() if self.lh_mask.sum() == 0: self.costs = np.array([ 100000000000., ]) return 0, 0 min_val = [-50, -50] max_val = [50, 50] ps, distributions = create_training_set(['xs', 'ys'], min_val, max_val, n_train=50) self.shift_solved = parmap(self.shift_optimize, ps) self.paras, self.costs = np.array([i[0] for i in self.shift_solved]), \ np.array([i[1] for i in self.shift_solved]) if (1 - self.costs.min()) >= 0.6: xs, ys = self.paras[self.costs == np.nanmin(self.costs)][0].astype( int) else: xs, ys = 0, 0 #print 'Best shift is ', xs, ys, 'with the correlation of', 1-self.costs.min() return xs, ys
def fire_gaus_optimize(self, ): xs, ys = self.fire_shift_optimize() if self.costs.min() < 0.1: min_val = [4, 4, -15, xs - 2, ys - 2] max_val = [40, 40, 15, xs + 2, ys + 2] self.bounds = [4, 40], [4, 40], [-15, 15], [xs - 2, xs + 2], [ys - 2, ys + 2] ps, distributions = create_training_set(self.parameters, min_val, max_val, n_train=50) print('Start solving...') self.gaus_solved = parmap(self.gaus_optimize, ps, nprocs=5) result = np.array( [np.hstack((i[0], i[1])) for i in self.gaus_solved]) print( 'solved psf', dict( zip(self.parameters + [ 'cost', ], result[np.argmin(result[:, -1])]))) return result[np.argmin(result[:, -1]), :] else: print('Cost is too large, plese check!') return []
def _fill_nan(self, ): def fill_nan(array): x_shp, y_shp = array.shape mask = ~np.isnan(array) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ np.tile (range(y_shp), x_shp).reshape(x_shp, y_shp) array = griddata(valid, value, mesh, method='nearest') return array self._vza = np.array(parmap(fill_nan, list(self._vza))) self._vaa = np.array(parmap(fill_nan, list(self._vaa))) self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc = \ parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc]) self._aot_unc = array_to_raster(self._aot_unc, self.example_file) self._tcwv_unc = array_to_raster(self._tcwv_unc, self.example_file) self._tco3_unc = array_to_raster(self._tco3_unc, self.example_file)
def _load_xa_xb_xc_emus(self, ): xap_emu = glob(self.emus_dir + '/isotropic_%s_emulators_correction_xap_%s.pkl' % (self.sensor, self.satellite))[0] xbp_emu = glob(self.emus_dir + '/isotropic_%s_emulators_correction_xbp_%s.pkl' % (self.sensor, self.satellite))[0] xcp_emu = glob(self.emus_dir + '/isotropic_%s_emulators_correction_xcp_%s.pkl' % (self.sensor, self.satellite))[0] if sys.version_info >= (3, 0): f = lambda em: pkl.load(open(em, 'rb'), encoding='latin1') else: f = lambda em: pkl.load(open(str(em), 'rb')) self.emus = parmap(f, [xap_emu, xbp_emu, xcp_emu])
def _read_MCD43(self, fnames): def warp_data(fname, aoi, xRes, yRes): g = gdal.Warp('',fname, format = 'MEM', srcNodata = 32767, dstNodata=0, \ cutlineDSName=aoi, xRes = xRes, yRes = yRes, cropToCutline=True, resampleAlg = 0) # weird adaptation for gdal 2.3, this should be a bug in gdal 2.3 return g.ReadAsArray( ) # no reason you have to specify the srcNodata to use dstNodata par = partial(warp_data, aoi=self.aoi, xRes=self.aero_res * 0.5, yRes=self.aero_res * 0.5) n_files = int(len(fnames) / 2) ret = parmap(par, fnames) das = np.array(ret[:n_files]) qas = np.array(ret[n_files:]) ws = 0.618034**qas ws[qas == 255] = 0 das[das == 32767] = 0 return das, ws
def _get_convolved_toa(self, ): imgs = [band_g.ReadAsArray() for band_g in self._toa_bands] self.bad_pixs = self.bad_pix[self.hx, self.hy] xgaus = np.exp( -2. * (np.pi**2) * (self.psf_xstd**2) * ((0.5 * np.arange(self.full_res[0]) / self.full_res[0])**2)) ygaus = np.exp( -2. * (np.pi**2) * (self.psf_ystd**2) * ((0.5 * np.arange(self.full_res[1]) / self.full_res[1])**2)) gaus_2d = np.outer(xgaus, ygaus) def convolve(img, gaus_2d, hx, hy): dat = idct(idct(dct(dct(img, axis=0, norm = 'ortho'), axis=1, \ norm='ortho') * gaus_2d, axis=1, norm='ortho'), axis=0, norm='ortho')[hx, hy] return dat par = partial(convolve, gaus_2d=gaus_2d, hx=self.hx, hy=self.hy) if np.array(self.ref_scale).ndim == 2: self.ref_scale = self.ref_scale[self.hx, self.hy] if np.array(self.ref_off).ndim == 2: self.ref_off = self.ref_off[self.hx, self.hy] self.toa = np.array(parmap(par, imgs)) * self.ref_scale + self.ref_off
def _solving(self, ): self.logger.propagate = False self.logger.info('Set AOI.') self._create_base_map() self.logger.info('Get corresponding bands.') self._find_boa_bands() self.logger.info('Slice TOA bands based on AOI.') self._create_band_gs() self._resamplers() self.logger.info('Parsing angles.') self._parse_angles() self.logger.info('Mask bad pixeles.') self._mask_bad_pix() if np.sum(~self.bad_pix) > 10: self.logger.info('Get simulated BOA.') self._get_boa() self.logger.info('Get PSF.') self._get_psf() self.logger.info('Get simulated TOA reflectance.') self._get_convolved_toa() self.logger.info('Filtering data.') self._re_mask() if self.mask is not False: self.logger.info('Loading emulators.') self._load_xa_xb_xc_emus() self.logger.info('Reading priors and elevation.') self._read_aux() self._fill_nan() self.logger.info( 'Mean values for prior AOT: %.02f and TCWV: %.02f' % (self._aot.mean(), self._tcwv.mean())) if self.mask.sum() == 0: self.logger.info( 'No valid value is found for retrieval of atmospheric parameters and priors are stored.' ) ret = np.array( [[self._aot, self._tcwv, self._tco3], [self._aot_unc, self._tcwv_unc, self._tco3_unc]]) self.aero_res /= 2 #self.ySize *=2 #self.xSize *=2 self.ySize, self.xSize = self._aot.shape else: self.aero = solving_atmo_paras(self.boa, self.toa, self._sza, self._vza, self._saa, self._vaa, self._aot, self._tcwv, self._tco3, self._ele, self._aot_unc, self._tcwv_unc, self._tco3_unc, self.boa_unc, self.hx, self.hy, self.mask, self.full_res, self.aero_res, self.emus, self.band_index, self.boa_wv, pix_res=self.pixel_res, gamma=self.gamma, log_file=self.log_file) ret = self.aero._multi_grid_solver() else: self.logger.info( 'No valid value is found for retrieval of atmospheric parameters and priors are stored.' ) self._read_aux() self._fill_nan() self.logger.info( 'Mean values for prior AOT: %.02f and TCWV: %.02f' % (self._aot.mean(), self._tcwv.mean())) ret = np.array( [[self._aot, self._tcwv, self._tco3], [self._aot_unc, self._tcwv_unc, self._tco3_unc]]) self.aero_res /= 2 #self.ySize *=2 #self.xSize *=2 self.ySize, self.xSize = self._aot.shape else: self.logger.info( 'No valid value is found for retrieval of atmospheric parameters and priors are stored.' ) self._read_aux() self._fill_nan() self.logger.info( 'Mean values for prior AOT: %.02f and TCWV: %.02f' % (self._aot.mean(), self._tcwv.mean())) ret = np.array([[self._aot, self._tcwv, self._tco3], [self._aot_unc, self._tcwv_unc, self._tco3_unc]]) self.aero_res /= 2 self.ySize, self.xSize = self._aot.shape solved = ret[0].reshape(3, self.ySize, self.xSize) unc = ret[1].reshape(3, self.ySize, self.xSize) self.logger.info( 'Finished retrieval and saving them into local files.') para_names = 'aot', 'tcwv', 'tco3', 'aot_unc', 'tcwv_unc', 'tco3_unc' toa_dir = self.toa_dir + '/' + 'B'.join( self.toa_bands[0].split('/')[-1].split('B')[:-1]) name_arrays = zip(para_names, list(solved) + list(unc)) par = partial(save_posterior, g=self.example_file, aero_res=self.aero_res, toa_dir=toa_dir) parmap(par, name_arrays) self.post_aot, self.post_tcwv, self.post_tco3, = solved self.post_aot_unc, self.post_tcwv_unc, self.post_tco3_unc = unc handlers = self.logger.handlers[:] for handler in handlers: handler.close() self.logger.removeHandler(handler)
aot_unc[:] = 0.5 tcwv_unc[:] = 0.2 tco3_unc[:] = 0.2 toa = np.random.rand(6, 50000) y = toa * 2.639794 - 0.038705 boa = y / (1 + 0.068196 * y) boa_unc = np.ones(50000) * 0.05 Hx = np.random.choice(10980, 50000) Hy = np.random.choice(10980, 50000) full_res = (10980, 10980) aero_res = 600 emus_dir = '~/DATA/Multiply/emus/' sensor = 'MSI' xap_emu = glob(emus_dir + '/isotropic_%s_emulators_*xap*.pkl' % (sensor))[0] xbp_emu = glob(emus_dir + '/isotropic_%s_emulators_*xbp*.pkl' % (sensor))[0] xcp_emu = glob(emus_dir + '/isotropic_%s_emulators_*xcp*.pkl' % (sensor))[0] f = lambda em: pkl.load(open(em, 'rb')) emus = parmap(f, [xap_emu, xbp_emu, xcp_emu]) band_indexs = [1, 2, 3, 7, 11, 12] band_wavelength = [469, 555, 645, 869, 1640, 2130] mask = np.zeros((10980, 10980)).astype(bool) mask[1, 1] = True aero = solving_atmo_paras(boa, toa, sza, vza, saa, vaa, aot, tcwv, tco3, ele, aot_unc, tcwv_unc, tco3_unc, boa_unc, Hx, Hy, mask, full_res, aero_res, emus, band_indexs, band_wavelength) solved = aero._multi_grid_solver()
def _obs_cost_test(self, p, is_full=True, do_unc=False): p = np.array(p).reshape(2, -1) X = self.control_variables.reshape(self.boa.shape[0], 7, -1) X[:, 3:5, :] = np.array(p) xap_H, xbp_H, xcp_H = [], [], [] xap_dH, xbp_dH, xcp_dH = [], [], [] emus = list(self.xap_emus) + list(self.xbp_emus) + list(self.xcp_emus) Xs = list(X) + list(X) + list(X) inps = list(zip(emus, Xs)) if self._coarse_mask.sum() > 1000: ret = np.array(parmap(self._helper, inps)) else: ret = np.array(list(map(self._helper, inps))) xap_H, xbp_H, xcp_H = ret[:, :, 0].reshape(3, self.boa.shape[0], len(self.Hx)) xap_dH, xbp_dH, xcp_dH = ret[:, :, 1:].reshape(3, self.boa.shape[0], len(self.Hx), 2) y = xap_H * self.toa - xbp_H sur_ref = y / (1 + xcp_H * y) diff = sur_ref - self.boa full_J = np.nansum(0.5 * self.band_weights[..., None] * (diff)**2 / self.boa_unc**2, axis=0) J = np.zeros(self.full_res) dH = -1 * (-self.toa[...,None] * xap_dH - \ 2 * self.toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \ self.toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \ xbp_dH + \ xbp_H[...,None]**2 * xcp_dH) / \ (self.toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \ xbp_H[...,None] * xcp_H[...,None] + 1)**2 full_dJ = [ self.band_weights[..., None] * dH[:, :, i] * diff / (self.boa_unc**2) for i in range(2) ] if is_full: dJ = np.nansum(np.array(full_dJ), axis=(1, )) dJ[np.isnan(dJ)] = 0 full_J[np.isnan(full_J)] = 0 #J_ = np.zeros((2,) + self.full_res) #J_[:, self.Hx, self.Hy] = dJ #subs1 = [np.array_split(sub, self.num_blocks_y, axis=2) for sub in np.array_split(J_, self.num_blocks_x, axis=1)] #J = np.zeros(self.full_res) #J[self.Hx, self.Hy] = full_J #subs2 = [np.array_split(sub, self.num_blocks_y, axis=1) for sub in np.array_split(J, self.num_blocks_x, axis=0)] #J_ = np.zeros((2, self.num_blocks_x, self.num_blocks_y)) #J = np.zeros(( self.num_blocks_x, self.num_blocks_y)) nx, ny = (np.ceil(np.array(self.full_res) / np.array([self.num_blocks_x, self.num_blocks_y])) \ * np.array([self.num_blocks_x, self.num_blocks_y])).astype(int) #end_x, end_y = np.array(self.full_res) - np.array([nx, ny]) x_size, y_size = int(nx / self.num_blocks_x), int( ny / self.num_blocks_y) J_ = np.zeros((2, nx, ny)) J = np.zeros((nx, ny)) #J_[:], J[:] = np.nan, np.nan J_[:, self.Hx, self.Hy] = dJ J[self.Hx, self.Hy] = full_J J_ = J_.reshape(2, self.num_blocks_x, x_size, self.num_blocks_y, y_size) J = J.reshape(self.num_blocks_x, x_size, self.num_blocks_y, y_size) J_ = np.sum(J_, axis=(2, 4)) J = np.sum(J, axis=(1, 3)) #for i in range(self.num_blocks_x): # for j in range(self.num_blocks_y): # J_[:, i,j] = np.nansum(subs1[i][j], axis=(1,2)) # J [ i,j] = np.nansum(subs2[i][j], axis=(0,1)) J_[:, ~self._coarse_mask] = 0 J[~self._coarse_mask] = 0 J_ = J_.reshape(2, -1) if do_unc: #comb_unc = np.nansum([self.band_weights[...,None] * (dH[:, :, i] ** 2) * (self.boa_unc ** -2) for i in range(2)], axis = 1) #comb_unc[comb_unc==0] = np.nan #self.obs_unc = np.zeros((2,) + self.full_res) #self.obs_unc[:] = np.nan #self.obs_unc[:, self.Hx, self.Hy] = comb_unc comb_unc = np.nansum([ self.band_weights[..., None] * (dH[:, :, i]**2) * (self.boa_unc**-2) for i in range(2) ], axis=1) comb_unc[comb_unc == 0] = np.nan self.obs_unc = np.zeros((2, nx, ny)) self.obs_unc[:] = np.nan self.obs_unc[:, self.Hx, self.Hy] = comb_unc self.obs_unc = self.obs_unc.reshape(2, self.num_blocks_x, x_size, self.num_blocks_y, y_size) self.obs_unc = np.nanmean(self.obs_unc, axis=(2, 4)) #subs = [np.array_split(sub, self.num_blocks_y, axis=2) for sub in np.array_split(self.obs_unc, self.num_blocks_x, axis=1)] #self.obs_unc = np.zeros((2, self.num_blocks_x, self.num_blocks_y)) #for i in range(self.num_blocks_x): # for j in range(self.num_blocks_y): # self.obs_unc[:, i,j] = np.nanmean(subs[i][j], axis=(1,2)) self.obs_unc[:, ~self._coarse_mask] = np.nan return self.obs_unc else: J = np.nansum(np.array(full_J)) J_ = np.nansum(np.array(full_dJ), axis=(1, 2)) return J, J_
def _get_boa(self, temporal_filling=16): qa_temp = 'MCD43_%s_BRDF_Albedo_Band_Mandatory_Quality_Band%d.vrt' da_temp = 'MCD43_%s_BRDF_Albedo_Parameters_Band%d.vrt' doy = self.obs_time.strftime('%Y%j') if temporal_filling == True: temporal_filling = 16 if temporal_filling: days = [(self.obs_time - timedelta(days = int(i))) for i in np.arange(temporal_filling, 0, -1)] + \ [(self.obs_time + timedelta(days = int(i))) for i in np.arange(0, temporal_filling+1, 1)] fnames = [] for temp in [da_temp, qa_temp]: for day in days: for band in self.boa_bands: fname = self.mcd43_dir + '/'.join([ day.strftime('%Y-%m-%d'), temp % (day.strftime('%Y%j'), band) ]) fnames.append(fname) else: fnames = [] for temp in [da_temp, qa_temp]: for band in self.boa_bands: fname = self.MCD43_dir + '/'.join([ datetime.strftime(self.obs_time, '%Y-%m-%d'), temp % (doy, band) ]) fnames.append(fname) das, ws = self._read_MCD43(fnames) mg = gdal.Warp('',fnames[0], format = 'MEM', dstNodata= None, xRes = self.aero_res*0.5, yRes = \ self.aero_res*0.5, cutlineDSName=self.aoi, cropToCutline=True, resampleAlg = 0) hg = self.example_file self.hx, self.hy, hmask, rmask = self._get_index(mg, hg) No_band = len(self.toa_bands) mask = ~(mg.ReadAsArray()[0] == 0) self._annoying_angles(mg) sza = np.repeat(self._sza[None, ...], len(self._vza), axis=0)[:, mask][:, hmask][:, rmask] saa = np.repeat(self._saa[None, ...], len(self._vza), axis=0)[:, mask][:, hmask][:, rmask] angles = self._vza[:, mask][:, hmask][:, rmask], sza, self._vaa[:, mask][:, hmask][:, rmask] - saa kk = get_kk(angles) k_vol = kk.Ross k_geo = kk.Li kers = np.array([np.ones(k_vol.shape), k_vol, k_geo]) surs = [] for i in range(No_band): surs.append( (das[i::No_band][:, :, mask][:, :, hmask][:, :, rmask] * kers[:, i] * 0.001).sum(axis=1)) if temporal_filling: def smooth(da_w): da, w = da_w mid = int(da.shape[0] / 2) if (da.shape[-1] == 0) | (w.shape[-1] == 0): return da[mid], w[mid] data = np.array( smoothn(da, s=10., smoothOrder=1., axis=0, TolZ=0.001, verbose=False, isrobust=True, W=w))[[0, 3], ] return data[0][mid], data[1][mid] boa = [] w = [] for i in range(No_band): das = surs[i] Ws = ws[i::No_band][:, mask][:, hmask][:, rmask] chunks = zip(np.array_split(das, 18, axis=1), np.array_split(Ws, 18, axis=1)) ret = parmap(smooth, chunks) _b = np.hstack([i[0] for i in ret]) _w = np.hstack([i[1] for i in ret]) boa.append(_b) w.append(_w) boa = np.array(boa) w = np.array(w) unc = 0.015 / w else: boa = np.array(surs) unc = 0.015 / ws self.boa = boa self.boa_unc = np.minimum(unc, 0.5)