def _create_band_gs(self,):
'''
Create a lost of boa gs and cut them with AOI.
'''
self._toa_bands = []
for band in self.toa_bands:
pixel_res = abs(gdal.Open(band).GetGeoTransform()[1])
g = gdal.Warp('', band, format = 'MEM', xRes = pixel_res, yRes = pixel_res, srcNodata = 0, dstNodata=0, warpOptions = \
['NUM_THREADS=ALL_CPUS'], cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0)
self._toa_bands.append(g)
self.example_file = self._toa_bands[0]
if self.cloud_mask is None:
self.cloud_mask = False
self.mask = self.example_file.ReadAsArray() >0# & (~self.cloud_mask)
self.mask_g = array_to_raster(self.mask.astype(float), self.example_file)
if isinstance(self.ref_scale, np.ndarray):
assert(self.ref_scale.shape == self.mask.shape), \
'Reflectance scale should have the same shape as reflectance, but got ' + str(self.ref_scale.shape) + ' and ' + str(self.mask.shape)
self.ref_scale_g = array_to_raster(self.ref_scale, self.example_file)
if isinstance(self.ref_off, np.ndarray):
assert(self.ref_off.shape == self.mask.shape), \
'Reflectance offset should have the same shape as reflectance, but got ' + str(self.ref_off.shape) + ' and ' + str(self.mask.shape)
self.ref_off_g = array_to_raster(self.ref_off, self.example_file)
self.mask = self.mask.astype(bool)
self.mg = gdal.Warp('', band, format = 'MEM', srcNodata = None, xRes = self.block_size, yRes = \
self.block_size, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0)
def _get_xps(self,):
p = [np.cos(self._sza), None, None, self._aot, self._tcwv, self._tco3, self._ele]
raa = np.cos(self._vaa - self._saa)
vza = np.cos(self._vza)
xap, xap_dH = [], []
xbp, xbp_dH = [], []
xcp, xcp_dH = [], []
xps = [xap, xbp, xcp]
xhs = [xap_dH, xbp_dH, xcp_dH]
for i in range(len(self.toa_bands)):
emus_index = self.band_index[i]
X = copy(p)
X[1:3] = vza[i], raa[i]
X = np.array(X).reshape(7, -1)[:, self._cmask.ravel()]
for ei, emu in enumerate([self.xap_emus, self.xbp_emus, self.xcp_emus]):
#H, _, dH = emu[emus_index].predict(X.T, do_unc=True)
H, dH = emu[emus_index].predict(X.T, cal_jac=True)[0]
temp = np.zeros_like(self._sza)
temp[self._cmask] = H.ravel()
xps[ei].append(array_to_raster(temp, self.example_file))
temp = np.zeros(self._sza.shape + (3,))
temp[self._cmask] = dH[:, 3:6]
xhs[ei].append(array_to_raster(temp, self.example_file))
self._saa; del self._sza; del self._ele; del self._aot; del self._tcwv; del self._tco3#; del self._aot_unc; del self._tcwv_unc; del self._tco3_unc
self.xap_H, self.xbp_H, self.xcp_H = np.array(xps)
self.xap_dH, self.xbp_dH, self.xcp_dH = np.array(xhs)
def _fill_nan(self,):
self._vza = np.array(list(map(fill_nan, list(self._vza))))
self._vaa = np.array(list(map(fill_nan, list(self._vaa))))
self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc = \
list(map(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc]))
self._aot_unc = array_to_raster(self._aot_unc, self.example_file)
self._tcwv_unc = array_to_raster(self._tcwv_unc, self.example_file)
self._tco3_unc = array_to_raster(self._tco3_unc, self.example_file)
def _var_parser(self, var):
if os.path.exists(str(var)):
var_g = gdal.Open(str(var))
elif type(var).__module__== 'numpy':
var_g = array_to_raster(var, self.example_file)
elif ('/vsicurl/' in str(var)) or ('/vsizip/') in str(var):
var_g = gdal.Open(str(var))
else:
var = float(var)
var_array = np.zeros((10,10))
var_array[:] = var
var_g = array_to_raster(var_array, self.example_file)
return var_g
def _fill_nan(self, ):
def fill_nan(array):
x_shp, y_shp = array.shape
mask = ~np.isnan(array)
valid = np.array(np.where(mask)).T
value = array[mask]
mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \
np.tile (range(y_shp), x_shp).reshape(x_shp, y_shp)
array = griddata(valid, value, mesh, method='nearest')
return array
self._vza = np.array(parmap(fill_nan, list(self._vza)))
self._vaa = np.array(parmap(fill_nan, list(self._vaa)))
self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc = \
parmap(fill_nan, [self._saa, self._sza, self._ele, self._aot, self._tcwv, self._tco3, self._aot_unc, self._tcwv_unc, self._tco3_unc])
self._aot_unc = array_to_raster(self._aot_unc, self.example_file)
self._tcwv_unc = array_to_raster(self._tcwv_unc, self.example_file)
self._tco3_unc = array_to_raster(self._tco3_unc, self.example_file)
def _create_band_gs(self,):
'''
Create a lost of boa gs and cut them with AOI.
'''
self._toa_bands = []
for band in self.toa_bands:
g = gdal.Warp('', band, format = 'MEM', srcNodata = 0, dstNodata=0, warpOptions = \
['NUM_THREADS=ALL_CPUS'], cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0)
self._toa_bands.append(g)
self.example_file = self._toa_bands[0]
if self.cloud_mask is None:
self.cloud_mask = False
self.mask = self.example_file.ReadAsArray() >0# & (~self.cloud_mask)
self.mask_g = array_to_raster(self.mask.astype(float), self.example_file)
self.mask = self.mask.astype(bool)
self.mg = gdal.Warp('', band, format = 'MEM', srcNodata = None, xRes = self.block_size, yRes = \
self.block_size, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0)
def _mask_bad_pix(self):
#snow_mask = blue > 0.6
if os.path.exists(str(self.cloud_mask)):
cloud_g = gdal.Open(str(self.cloud_mask))
elif type(self.cloud_mask).__module__ == 'numpy':
cloud_g = array_to_raster(self.cloud_mask, self.example_file)
cloud = reproject_data(cloud_g, \
self.example_file, \
xRes=self.pixel_res, \
yRes=self.pixel_res, \
xSize=self.full_res[1], \
ySize=self.full_res[0], \
srcNodata = np.nan,\
outputType= gdal.GDT_Float32,\
resample = 0).data
cloud = cloud.astype(bool)
RED = None
BLUE = None
SWIR_1 = None
NIR = None
GREEN = None
if 3 in self.boa_bands:
BLUE = self._toa_bands[np.where(self.boa_bands == 3)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if BLUE is not None:
water_mask = BLUE < 0.05
snow_mask = BLUE > 0.6
#del BLUE
else:
self.logger.error(
'Blue band is needed for the retirval of aerosol.')
if 2 in self.boa_bands:
NIR = self._toa_bands[np.where(self.boa_bands == 2)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if 1 in self.boa_bands:
RED = self._toa_bands[np.where(self.boa_bands == 1)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if (RED is not None) & (NIR is not None):
NDVI = (NIR - RED) / (NIR + RED)
water_mask = ((NDVI < 0.01) & (NIR < 0.11)) | ((NDVI < 0.1) & (NIR < 0.05)) \
| (NIR <= 0.0001) | (RED <= 0.0001) | np.isnan(NIR) | np.isnan(RED)
del NIR
del RED
del NDVI
elif NIR is not None:
water_mask = NIR < 0.05
del NIR
elif RED is not None:
water_mask = RED < 0.05
del RED
if 6 in self.boa_bands:
SWIR_1 = self._toa_bands[np.where(self.boa_bands == 6)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if 1 in self.boa_bands:
GREEN = self._toa_bands[np.where(self.boa_bands == 4)[0][
0]].ReadAsArray() * self.ref_scale + self.ref_off
if (SWIR_1 is not None) & (GREEN is not None):
NDSI = (GREEN - SWIR_1) / (SWIR_1 + GREEN)
snow_mask = (NDSI > 0.42) | (SWIR_1 <= 0.0001) | (
GREEN <= 0.0001) | np.isnan(SWIR_1) | np.isnan(GREEN)
del SWIR_1
del GREEN
del NDSI
mask = water_mask | snow_mask | cloud | (BLUE > 1)
ker_size = int(2 * 1.96 * self.psf_ystd)
mask = binary_erosion(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=5).astype(bool)
#mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool)
mask = binary_dilation(mask,
structure=np.ones((3, 3)).astype(bool),
iterations=30 + ker_size).astype(bool)
mask[:30, :] = mask[:, :30] = mask[:, -30:] = mask[-30:, :] = True
self.bad_pix = mask
