Esempio n. 1
0
 def _annoying_angles(self,):
     _sun_angles = [] 
     _view_angles = []
     for fname in self._sun_angles:     
         #nodatas = [float(i.split("=")[1]) for i in gdal.Info(fname).split('\n') if' NoData' in i] 
         try:
             nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i])
         except:
             nodatas = None
         ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \
                              0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
         _sun_angles.append(ang)        
     for fname in self._view_angles:    
         try:                           
             nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i])
         except:                        
             nodatas = None 
         ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \
                              0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
         _view_angles.append(ang)       
     _view_angles = np.array(_view_angles)
     _sun_angles = np.squeeze(np.array(_sun_angles))
     if len(self._view_angles) == len(self.toa_bands):
         if self.a_z_order == 1:        
             self._vaa = _view_angles[:,0] 
             self._vza = _view_angles[:,1]
         else:                          
             self._vaa = _view_angles[:,1] 
             self._vza = _view_angles[:,0]
     elif len(self._view_angles) == 2*len(self.toa_bands):
         self._vaa = _view_angles[:len(self.toa_bands)]           
         self._vza = _view_angles[len(self.toa_bands):]           
      
     if os.path.exists(str(self.sun_angles)):
         if self.a_z_order == 1:        
             self._saa = _sun_angles[0] 
             self._sza = _sun_angles[1] 
         else:                          
             self._saa = _sun_angles[1] 
             self._sza = _sun_angles[0] 
     elif len(self.sun_angles) == 2:    
         self._saa = _sun_angles[0]     
         self._sza = _sun_angles[1]
     self._sza = self._sza * self.ang_scale
     self._saa = self._saa * self.ang_scale
     self._vza = self._vza * self.ang_scale
     self._vaa = self._vaa * self.ang_scale
     self._vaa, self._vza, self._saa, self._sza = map(np.deg2rad, [self._vaa, self._vza, self._saa, self._sza])
Esempio n. 2
0
def do_cloud(cloud_bands, cloud_name=None):
    toas = [
        reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data
        for band in cloud_bands
    ]
    toas = np.array(toas) / 10000.
    mask = np.all(toas >= 0.0001, axis=0)
    valid_pixel = mask.sum()
    cloud_proba = cl.predict_proba(toas[:, mask].T)
    cloud_mask = np.zeros_like(toas[0])
    cloud_mask[mask] = cloud_proba[:, 1]
    cloud_mask[~mask] = -256
    if cloud_name is None:
        return cloud_mask
    else:
        g = gdal.Open(cloud_bands[0])
        dst = gdal.GetDriverByName('GTiff').Create(
            cloud_name,
            g.RasterXSize,
            g.RasterYSize,
            1,
            gdal.GDT_Byte,
            options=["TILED=YES", "COMPRESS=DEFLATE"])
        dst.SetGeoTransform(g.GetGeoTransform())
        dst.SetProjection(g.GetProjection())
        dst.GetRasterBand(1).WriteArray((cloud_mask * 100).astype(int))
        dst.GetRasterBand(1).SetNoDataValue(-256)
        dst = None
        g = None
        return cloud_mask
Esempio n. 3
0
 def _parse_aux(self,):
     auxs =  [self.global_dem, self.mask.astype(float)]
     scales = [self.ele_scale, 1]
     for i  in range(len(auxs)):
         var_g = self._var_parser(auxs[i])
         dat = reproject_data(var_g, self.mg, srcNodata = 0, resample = \
                              0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data
         auxs[i] = dat * scales[i]
     self._ele, self._cmask = auxs
     self._ele[np.isnan(self._ele)] = 0
     self._cmask[np.isnan(self._cmask)] = 0
     self._cmask = binary_dilation(self._cmask, structure = np.ones((3,3)).astype(bool)).astype(bool)
Esempio n. 4
0
    def _resamplers(self, ):
        """
        Define the sresamplers used for resampling different gdal files or objects to required resolution and size.
        """
        self.nearest_resampler = lambda fname: reproject_data(
            fname,
            self.example_file,
            xRes=self.aero_res * 0.5,
            yRes=self.aero_res * 0.5,
            srcNodata=None,
            outputType=gdal.GDT_Float32,
            resample=0,
        ).g  # GRIORA_NearestNeighbour due to version changes...

        self.bilinear_resampler = lambda fname: reproject_data(
            fname,
            self.example_file,
            xRes=self.aero_res * 0.5,
            yRes=self.aero_res * 0.5,
            srcNodata=0,
            outputType=gdal.GDT_Float32,
            resample=1,
        ).g  # GRIORA_Bilinear
Esempio n. 5
0
    def _annoying_angles(self, destination):
        mg = destination
        _sun_angles = []
        _view_angles = []
        for fname in self._sun_angles:
            try:
                nodatas = " ".join([
                    i.split("=")[1] for i in gdal.Info(fname).split("\n")
                    if " NoData" in i
                ])
            except:
                nodatas = None
            ang = reproject_data(
                fname,
                mg,
                srcNodata=None,
                resample=0,
                dstNodata=np.nan,
                outputType=gdal.GDT_Float32,
            ).data
            _sun_angles.append(ang)
        for fname in self._view_angles:
            try:
                nodatas = " ".join([
                    i.split("=")[1] for i in gdal.Info(fname).split("\n")
                    if " NoData" in i
                ])
            except:
                nodatas = None
            ang = reproject_data(
                fname,
                mg,
                srcNodata=None,
                resample=0,
                dstNodata=np.nan,
                outputType=gdal.GDT_Float32,
            ).data
            _view_angles.append(ang)
        _view_angles = np.squeeze(np.array(_view_angles))
        _sun_angles = np.squeeze(np.array(_sun_angles))
        if len(self.view_angles) == len(self.toa_bands):
            if self.a_z_order == 1:
                self._vaa = _view_angles[:, 0]
                self._vza = _view_angles[:, 1]
            else:
                self._vaa = _view_angles[:, 1]
                self._vza = _view_angles[:, 0]
        elif len(self.view_angles) == 2 * len(self.toa_bands):
            self._vaa = _view_angles[:len(self.toa_bands)]
            self._vza = _view_angles[len(self.toa_bands):]

        if os.path.exists(str(self.sun_angles)):
            if self.a_z_order == 1:
                self._saa = _sun_angles[0]
                self._sza = _sun_angles[1]
            else:
                self._saa = _sun_angles[1]
                self._sza = _sun_angles[0]
        elif len(self.sun_angles) == 2:
            self._saa = _sun_angles[0]
            self._sza = _sun_angles[1]
        self._sza = self._sza * self.ang_scale
        self._saa = self._saa * self.ang_scale
        self._vza = self._vza * self.ang_scale
        self._vaa = self._vaa * self.ang_scale
Esempio n. 6
0
    def _mask_bad_pix(self):
        # snow_mask = blue > 0.6
        if os.path.exists(str(self.cloud_mask)):
            cloud_g = gdal.Open(str(self.cloud_mask))
        elif type(self.cloud_mask).__module__ == "numpy":
            cloud_g = array_to_raster(self.cloud_mask, self.example_file)
        cloud = reproject_data(
            cloud_g,
            self.example_file,
            xRes=self.pixel_res,
            yRes=self.pixel_res,
            xSize=self.full_res[1],
            ySize=self.full_res[0],
            srcNodata=np.nan,
            outputType=gdal.GDT_Float32,
            resample=0,
        ).data
        cloud = cloud.astype(bool)
        RED = None
        BLUE = None
        SWIR_1 = None
        NIR = None
        GREEN = None

        if 3 in self.boa_bands:
            BLUE = (self._toa_bands[np.where(
                self.boa_bands == 3)[0][0]].ReadAsArray() * self.ref_scale +
                    self.ref_off)
        if BLUE is not None:
            water_mask = BLUE < 0.05
            snow_mask = BLUE > 0.6
            # del BLUE
        else:
            self.logger.error(
                "Blue band is needed for the retirval of aerosol.")

        if 2 in self.boa_bands:
            NIR = (self._toa_bands[np.where(
                self.boa_bands == 2)[0][0]].ReadAsArray() * self.ref_scale +
                   self.ref_off)
        if 1 in self.boa_bands:
            RED = (self._toa_bands[np.where(
                self.boa_bands == 1)[0][0]].ReadAsArray() * self.ref_scale +
                   self.ref_off)
        if (RED is not None) & (NIR is not None):
            NDVI = (NIR - RED) / (NIR + RED)
            water_mask = (((NDVI < 0.01) & (NIR < 0.11))
                          | ((NDVI < 0.1) & (NIR < 0.05))
                          | (NIR <= 0.0001)
                          | (RED <= 0.0001)
                          | np.isnan(NIR)
                          | np.isnan(RED))
            del NIR
            del RED
            del NDVI
        elif NIR is not None:
            water_mask = NIR < 0.05
            del NIR
        elif RED is not None:
            water_mask = RED < 0.05
            del RED

        if 6 in self.boa_bands:
            SWIR_1 = (self._toa_bands[np.where(
                self.boa_bands == 6)[0][0]].ReadAsArray() * self.ref_scale +
                      self.ref_off)
        if 1 in self.boa_bands:
            GREEN = (self._toa_bands[np.where(
                self.boa_bands == 4)[0][0]].ReadAsArray() * self.ref_scale +
                     self.ref_off)
        if (SWIR_1 is not None) & (GREEN is not None):
            NDSI = (GREEN - SWIR_1) / (SWIR_1 + GREEN)
            snow_mask = ((NDSI > 0.42)
                         | (SWIR_1 <= 0.0001)
                         | (GREEN <= 0.0001)
                         | np.isnan(SWIR_1)
                         | np.isnan(GREEN))
            del SWIR_1
            del GREEN
            del NDSI
        mask = water_mask | snow_mask | cloud | (BLUE > 1)
        ker_size = int(2 * 1.96 * self.psf_ystd)
        mask = binary_erosion(mask,
                              structure=np.ones((3, 3)).astype(bool),
                              iterations=5).astype(bool)
        # mask     = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool)
        mask = binary_dilation(mask,
                               structure=np.ones((3, 3)).astype(bool),
                               iterations=30 + ker_size).astype(bool)
        mask[:30, :] = mask[:, :30] = mask[:, -30:] = mask[-30:, :] = True
        self.bad_pix = mask
Esempio n. 7
0
def do_aot_tcwv(aot_bands,
                tcwv_bands,
                cams_dir,
                obs_time,
                sun_ang_name,
                view_ang_name,
                dem,
                tcwv_name=None,
                aot_name=None):

    toas = [
        reproject_data(str(band),
                       tcwv_bands[0],
                       dstNodata=0,
                       resample=5,
                       xRes=120,
                       yRes=120).data for band in aot_bands
    ]
    toas = np.array(toas) / 10000.

    mask = np.all(toas >= 0.0001, axis=0)

    time_ind = np.abs((obs_time.hour + obs_time.minute / 60. +
                       obs_time.second / 3600.) -
                      np.arange(0, 25, 3)).argmin()
    prior_uncs = 0.1
    prior_scale = 46.698
    prior_f = cams_dir + '/'.join([
        datetime.datetime.strftime(obs_time, '%Y_%m_%d'),
        datetime.datetime.strftime(obs_time, '%Y_%m_%d') + '_gtco3.tif'
    ])
    var_g = gdal.Open(prior_f)
    prior_g = reproject_data(prior_f,
                             tcwv_bands[0],
                             dstNodata=0,
                             resample=1,
                             xRes=120,
                             yRes=120).g
    g = var_g.GetRasterBand(int(time_ind + 1))
    offset = g.GetOffset()
    scale = g.GetScale()
    tco3 = prior_g.GetRasterBand(
        int(time_ind + 1)).ReadAsArray() * scale + offset
    tco3[:] = np.nanmean(tco3) * prior_scale

    saa, sza = reproject_data(str(sun_ang_name),
                              tcwv_bands[0],
                              dstNodata=0,
                              resample=1,
                              xRes=120,
                              yRes=120).data / 100.
    vaa, vza = reproject_data(str(view_ang_name),
                              tcwv_bands[0],
                              dstNodata=0,
                              resample=1,
                              xRes=120,
                              yRes=120).data / 100.
    raa = vaa - saa

    sza = np.cos(np.deg2rad(sza))
    vza = np.cos(np.deg2rad(vza))
    raa = np.cos(np.deg2rad(raa))

    ele = reproject_data(
        dem, tcwv_bands[0], dstNodata=0, resample=1, xRes=120,
        yRes=120).data / 10000.

    X = np.vstack([toas[[9, 8]],
                   np.array([sza, vza, raa, tco3, ele])]).reshape(7, -1).T
    iso_tcwv = Two_NN(np_model_file=file_path + '/emus/S2_TCWV.npz')
    tcwv = iso_tcwv.predict(X)[0].reshape(toas[0].shape).astype(float)

    bad = (tcwv < 0) | (tcwv > 8) | (~mask) | np.isnan(tcwv)
    tcwv[bad] = np.nanmedian(tcwv[~bad])

    bands_min = [
        0.11572497, 0.08986528, 0.07280412, 0.05007033, 0.06228712, 0.06849915,
        0.07015634, 0.06554198, 0.06809415, 0.02089167, 0.00080242, 0.04392302,
        0.03012728
    ]
    bands_max = [
        0.32072931, 0.30801709, 0.32420026, 0.38214899, 0.3951169, 0.41631785,
        0.44391895, 0.42444658, 0.46161492, 0.19927658, 0.00870671, 0.39533241,
        0.32505772
    ]

    for _, toa in enumerate(toas):
        mask = mask & (toa >= bands_min[_]) & (toa <= bands_max[_])

    X = np.vstack([toas, np.array([sza, vza, raa, tco3,
                                   ele])]).reshape(18, -1).T
    gbm = joblib.load(file_path + '/emus/lgb.pkl')
    aot = gbm.predict(X).reshape(toas[0].shape).astype(float)
    aot = np.exp(-1 * aot)
    shape = toas[0].shape
    if mask.sum() > 3:
        aot_min, aot_median, aot_max = np.nanpercentile(aot[mask], [5, 50, 95])
        good_aot = (aot >= aot_min) & (aot <= aot_max)
        indx, indy = np.where(good_aot.reshape(shape))
        myInterpolator = NearestNDInterpolator((indx, indy), aot[good_aot])
        grid_x, grid_y = np.mgrid[0:shape[0]:1, 0:shape[1]:1, ]
        aot = myInterpolator(grid_x, grid_y)
        aot = smoothn(aot,
                      isrobust=True,
                      TolZ=1e-6,
                      W=100 * ((aot >= aot_min) & (aot <= aot_max)),
                      s=10,
                      MaxIter=1000)[0]
    else:
        aot = np.nanmedian(aot) * np.ones(shape)

    if tcwv_name is not None:
        g = gdal.Open(tcwv_bands[0])
        ySize, xSize = tcwv.shape
        dst = gdal.GetDriverByName('GTiff').Create(
            tcwv_name,
            xSize,
            ySize,
            1,
            gdal.GDT_UInt16,
            options=["TILED=YES", "COMPRESS=DEFLATE"])
        dst.SetGeoTransform(g.GetGeoTransform())
        dst.SetProjection(g.GetProjection())
        dst.GetRasterBand(1).WriteArray((tcwv * 1000).astype(int))
        dst.GetRasterBand(1).SetNoDataValue(65535)
        dst = None
        g = None

    if aot_name is not None:
        g = gdal.Open(aot_bands[0])
        ySize, xSize = aot.shape
        dst = gdal.GetDriverByName('GTiff').Create(
            aot_name,
            xSize,
            ySize,
            1,
            gdal.GDT_UInt16,
            options=["TILED=YES", "COMPRESS=DEFLATE"])
        dst.SetGeoTransform(g.GetGeoTransform())
        dst.SetProjection(g.GetProjection())
        dst.GetRasterBand(1).WriteArray((aot * 1000).astype(int))
        dst.GetRasterBand(1).SetNoDataValue(65535)
        dst = None
        g = None

    return aot, tcwv
Esempio n. 8
0
    def _do_chunk(self, ind): 
        toa_g = self._toa_bands[ind]       
        geo = toa_g.GetGeoTransform()
        x_size, y_size = toa_g.RasterXSize, toa_g.RasterYSize
        boas = []
        uncs = []
        for i in range(self._chunks):
            r = i * 1. /  self._chunks
            x_start = int(r * x_size)
            xmin = geo[0] + r * x_size * geo[1]

            r = (i+1.)/ self._chunks
            x_end = int(r * x_size)
            xmax = geo[0] + r * x_size * geo[1]

            x_off = x_end - x_start
            toa = toa_g.ReadAsArray(x_start, 0, x_off, int(y_size))
            _g = gdal.Warp('', toa_g, format = 'MEM',  outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = 0)
            xRes = yRes = int(geo[1])
            xps  = [self.xap_H[ind], self.xbp_H[ind], self.xcp_H[ind]] 
            for j in range(3): 
                xps[j]  = reproject_data(xps[j], _g,xRes=xRes, yRes = yRes,xSize=x_off, \
                                         ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data
             
            xap_H, xbp_H, xcp_H = xps
            toa = toa * self.ref_scale + self.ref_off                             
            y   = xap_H * toa - xbp_H
            boa = y / (1 + xcp_H * y)
             
            xhs = [self.xap_dH[ind], self.xbp_dH[ind], self.xcp_dH[ind]]
            for j in range(3):         
                #var_g = xhs[j]
                xhs[j]  = reproject_data(xhs[j], _g, xRes=xRes, yRes = yRes, xSize=x_off, \
                                         ySize=y_size, resample = 0, \
                                         outputType= gdal.GDT_Float32).data.transpose(1,2,0)
             
            xap_dH, xbp_dH, xcp_dH = xhs
            del xps; del xhs   
             
            dH           = -1 * (-toa[...,None] * xap_dH - \
                            2 * toa[...,None] * xap_H[...,None] * xbp_H[...,None] * xcp_dH + \
                            toa[...,None]**2 * xap_H[...,None]**2 * xcp_dH + \
                            xbp_dH + \
                            xbp_H[...,None]**2 * xcp_dH) / \
                            (toa[...,None] * xap_H[...,None] * xcp_H[...,None] - \
                                xbp_H[...,None] * xcp_H[...,None] + 1)**2
             
            toa_dH = xap_H / (xcp_H*(toa * xap_H - xbp_H) + 1)**2
            del xap_H; del xbp_H; del xcp_H; del xap_dH; del xbp_dH; del xcp_dH
            aot_dH, tcwv_dH, tco3_dH = [dH[:, :,i] for i in range(3)]
            del dH
             
            tmp = []
            for unc_g in [self._aot_unc, self._tcwv_unc, self._tco3_unc]:
                unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data
                tmp.append(unc)
            aot_unc, tcwv_unc, tco3_unc = tmp; del tmp
            unc  = np.sqrt(aot_dH ** 2 * aot_unc**2 + tcwv_dH ** 2 * tcwv_unc**2 + tco3_dH ** 2 * tco3_unc**2 + toa_dH**2 * 0.015**2)
            del aot_unc; del tcwv_unc; del tco3_unc
            boas.append(boa)
            uncs.append(unc)
        boas, uncs = np.hstack(boas), np.hstack(uncs)
        boa_name = self.toa_dir + '/' + '.'.join(self.toa_bands[ind].split('/')[-1].split('.')[0:-1]) + '_sur.tif'
        unc_name  = boa_name.replace('_sur.tif', '_sur_unc.tif')
        projectionRef = toa_g.GetProjectionRef()
        geotransform  = toa_g.GetGeoTransform()
        self._save_band(boas, boa_name, projectionRef, geotransform)
        self._save_band(uncs, unc_name, projectionRef, geotransform)
        if ind in [self.ri, self.gi, self.bi]:
            return boas
        else:
            return None