示例#1
0
    def test_cloud_detector(self):
        cloud_detector = S2PixelCloudDetector(all_bands=True)
        cloud_probs = cloud_detector.get_cloud_probability_maps(
            self.templates['s2_im'])
        cloud_mask = cloud_detector.get_cloud_masks(self.templates['s2_im'])
        self.assertTrue(np.array_equal(cloud_probs,
                                       self.templates['cl_probs']))
        self.assertTrue(np.array_equal(cloud_mask, self.templates['cl_mask']))

        cloud_detector = S2PixelCloudDetector(all_bands=False)
        self.assertRaises(
            ValueError, lambda: cloud_detector.get_cloud_probability_maps(
                self.templates['s2_im']))
示例#2
0
    def detectClouds(self, threshold):
        '''
        Use s2cloudless to detect clouds
        
        Parameters
        ----------
        threshold : float
            Minimum probability of cloud in order to mask a pixel
        '''
        self.threshold = threshold
        
        # Load any bands that are not already there
        
        load_required = 0
        for band in self.band_list:
            if (band not in self.layers):
                load_required = 1
                
        if (load_required == 1):
            self.loadLayers(self.band_list)
            
        # Load numpy.ndarray for use with s2cloudless
        numpy_bands = np.empty((1, self.size_x, self.size_y, 10))
        numpy_bands[0,:,:,0] = self.numpy_layers['B01']
        numpy_bands[0,:,:,1] = self.numpy_layers['B02']
        numpy_bands[0,:,:,2] = self.numpy_layers['B04']
        numpy_bands[0,:,:,3] = self.numpy_layers['B05']
        numpy_bands[0,:,:,4] = self.numpy_layers['B08']
        numpy_bands[0,:,:,5] = self.numpy_layers['B8A']
        numpy_bands[0,:,:,6] = self.numpy_layers['B09']
        numpy_bands[0,:,:,7] = self.numpy_layers['B10']
        numpy_bands[0,:,:,8] = self.numpy_layers['B11']
        numpy_bands[0,:,:,9] = self.numpy_layers['B12']
        
        cloud_detector   = S2PixelCloudDetector(threshold=threshold, average_over=4, dilation_size=2)
        self.cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(numpy_bands))
        self.cloud_masks = cloud_detector.get_cloud_masks(numpy_bands)
        
        # Save cloud mask to disk as black PNG that is transparent where there is no cloud,
        # and opaque where the probability of cloud > threshold
        save_location = os.path.join("masks", "cloud_masks")
        os.makedirs(save_location, exist_ok=True)
    
        img = Image.new('RGBA', (self.size_x, self.size_y), color = 'black')

        self.cloudCount = 0
        
        for x in range(self.size_x):
            for y in range(self.size_y):
                if (self.cloud_masks[0][y][x] > 0):
                    img.putpixel((x, y), (0,0,0,255)) # Opaque
                    self.cloudCount += 1
                else:
                    img.putpixel((x, y), (0,0,0,0))   # Transparent
    
        output_file = os.path.join(save_location, "mask-x" + str(self.tile_x) + "-y" + str(self.tile_y) + "-" + self.dateStr + ".png")

        print("Writing Cloud Mask [" + output_file + "]")    
        img.save(output_file)
        self.metadata.setCloudCount(self.cloudCount)
def create_mask(band_array, path):
    """Crete the mask."""
    stacked = np.stack(band_array, -1)
    # shape (1, y_pixels, x_pixels, n_bands)
    # s2cloudless requires binary map
    arr4d = np.expand_dims(stacked / 10000, 0)
    cloud_detector = S2PixelCloudDetector(threshold=0.7,
                                          average_over=4,
                                          dilation_size=2)
    cloud_prob_map = cloud_detector.get_cloud_probability_maps(np.array(arr4d))
    cloud_masks = cloud_detector.get_cloud_masks(np.array(arr4d))
    template_file = [
        os.path.join(path, item) for item in os.listdir(path)
        if item.endswith("01.jp2")
    ][0]
    # save files
    out_prob = os.path.dirname(path) + "_s2cloudless_prob.tif"
    out_mask = os.path.dirname(path) + "_s2cloudless_mask.tif"
    output = gdal_array.SaveArray(cloud_prob_map[0, :, :],
                                  out_prob,
                                  format="GTiff",
                                  prototype=template_file)
    output = None
    output = gdal_array.SaveArray(cloud_masks[0, :, :],
                                  out_mask,
                                  format="GTiff",
                                  prototype=template_file)
    output = None
    return (out_prob, out_mask)
def createCloudMask(band_array, path):
    stacked = np.stack(band_array, -1)
    # print(stacked / 10000)
    # swapped = np.moveaxis(stacked, 0, 2)  # shape (y_pixels, x_pixels, n_bands)

    arr4d = np.expand_dims(
        stacked / 10000, 0
    )  # shape (1, y_pixels, x_pixels, n_bands) # s2cloudless requires binary map
    cloud_detector = S2PixelCloudDetector(threshold=0.7,
                                          average_over=4,
                                          dilation_size=2)
    cloud_prob_map = cloud_detector.get_cloud_probability_maps(np.array(arr4d))
    cloud_masks = cloud_detector.get_cloud_masks(np.array(arr4d))
    # show files
    plt.figure(figsize=(50, 50))
    plt.imshow(np.squeeze(
        cloud_masks))  # check whz process not stopping, when closing
    plt.show()
    # save files

    template_file = "/home/Usuario/Documents/sentinelHub/S2A_MSIL1C_20190904T140051_N0208_R067_T21LWH_20190904T172119.SAFE/GRANULE/L1C_T21LWH_A021943_20190904T140439/IMG_DATA/T21LWH_20190904T140051_B01.jp2"
    print(template_file)
    out_prob = path[:-31] + "out_prob.tif"
    out_mask = path[:-31] + "out_mask.tif"
    output = gdal_array.SaveArray(cloud_prob_map[0, :, :],
                                  out_prob,
                                  format="GTiff",
                                  prototype=template_file)
    output = None
    output = gdal_array.SaveArray(cloud_masks[0, :, :],
                                  out_mask,
                                  format="GTiff",
                                  prototype=template_file)
    output = None
示例#5
0
    def get_cloud_saturation_mask(self, ignore_detection=False):
        cloud_detector = S2PixelCloudDetector(**self.cloud_detection_config)
        true_color, bands, all_bands, dates = self.load_data()
        print("Downloaded data")
        print("Detecting clouds, this may take some time...")
        if True or not ignore_detection:
            cloud_masks_orig = cloud_detector.get_cloud_masks(np.array(bands))
            # upscale cloud masks
            cloud_masks = []
            for i in range(len(cloud_masks_orig)):
                cloud_masks.append(
                    self.upscale_image(cloud_masks_orig[i], self.cloud_scale))
            cloud_masks = np.array(cloud_masks)
        else:
            cloud_masks = np.array(bands) * 0
        print("Cloud detection finished")
        # Images might be slightly out of scale, crop them
        x, y, _ = true_color[0].shape
        cloud_masks = cloud_masks[:, -x:, :y]
        off_image_detection_mask = sum(map(self.get_image_mask, true_color))
        # Just sum how many times we detect white pixels

        full_cloud_mask = sum(cloud_masks)
        # Get total mask by dividing number of cloud detections by number of all sensible pixels
        total_mask_w = (
            full_cloud_mask /
            (len(cloud_masks) - off_image_detection_mask)).astype(float)

        self.memo_data = CloudSaturation.MemoData(total_mask_w, true_color,
                                                  bands, all_bands, dates,
                                                  cloud_masks)

        return total_mask_w, np.array(true_color), np.array(
            all_bands), np.array(dates), np.array(cloud_masks)
def test_sentinelhub_cloud_detector(display=False):
    """
    Runs the classification on the test scene and compares the results with the provided
    cloud probability map and probability mask. Used to verify the installation of the
    package.

    :param display: flag to turn on or off the display of results.
    :type display: bool, default is False
    """
    # Load arrays
    package_dir = os.path.dirname(__file__)
    templates_filename = os.path.join(package_dir,
                                      './TestInputs/input_arrays.npz')
    templates = np.load(templates_filename)

    # Classifier instance, image has all bands
    cloud_detector = S2PixelCloudDetector(all_bands=True)

    # Compute cloud probabilities
    cloud_probs = cloud_detector.get_cloud_probability_maps(templates['s2_im'])

    # Compute cloud mask
    cloud_mask = cloud_detector.get_cloud_masks(templates['s2_im'])

    # Check predicted results match templates
    probs_ok = np.isclose(cloud_probs, templates['cl_probs']).all()
    mask_ok = np.array_equal(cloud_mask, templates['cl_mask'])

    if not probs_ok:
        LOGGER.info(
            'Test FAILED.\nCloud probabilities DO NOT match templates.')

    if not mask_ok:
        LOGGER.info('Test FAILED.\nCloud masks DO NOT match templates.')

    if mask_ok and probs_ok:
        LOGGER.info(
            'Test OK.\nCloud probabilities and cloud masks match templates.')

    # Display results
    if display:
        fig = plt.figure(figsize=(20, 10))
        plt.subplot(131)
        plt.imshow(np.squeeze(templates['s2_im'])[:, :, [3, 2, 1]])
        plt.title('RGB image')
        plt.axis('off')
        plt.subplot(132)
        plt.imshow(np.squeeze(templates['s2_im'])[:, :, [3, 2, 1]])
        plt.imshow(np.squeeze(templates['cl_mask']), vmin=0, vmax=1, alpha=.4)
        plt.title('RGB image with template mask overlay')
        plt.axis('off')
        plt.subplot(133)
        plt.imshow(np.squeeze(templates['s2_im'])[:, :, [3, 2, 1]])
        plt.imshow(np.squeeze(cloud_mask), vmin=0, vmax=1, alpha=.4)
        plt.title('RGB image with predicted mask overlay')
        plt.axis('off')
        plt.show()
示例#7
0
def get_s2_pixel_cloud_detector(threshold=0.4,
                                average_over=4,
                                dilation_size=2,
                                all_bands=True):
    """ Wrapper function for pixel-based S2 cloud detector `S2PixelCloudDetector`
    """
    return S2PixelCloudDetector(threshold=threshold,
                                average_over=average_over,
                                dilation_size=dilation_size,
                                all_bands=all_bands)
示例#8
0
def test_no_data_available_request(config):
    """ Tests an exception raised by CloudMaskRequest
    """
    cloud_detector = S2PixelCloudDetector()
    with pytest.raises(NoDataAvailableException):
        CloudMaskRequest(cloud_detector,
                         bbox=BBOX1,
                         time=('2021-01-01', '2021-01-10'),
                         size=(250, 250),
                         maxcc=0.01,
                         config=config)
示例#9
0
    def cloud_mask(fpath):
        '''
        internal function. creates the cloud mask using s2cloudless package
        '''

        fname = fpath.split('/')[-1]

        aband_list = [
            'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A',
            'B09', 'B10', 'B11', 'B12', 'BQA', 'BQA', 'BQA', 'BQA'
        ]
        cband_list = [
            'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11',
            'B12'
        ]
        ds = gdal.Open(fpath, 0)
        ctr = 0
        for cband in cband_list:
            cind = aband_list.index(cband)
            cdata = ds.GetRasterBand(cind + 1).ReadAsArray()
            if ctr == 0:
                ndata = np.zeros(
                    (cdata.shape[0], cdata.shape[1], len(cband_list)))
            ndata[:, :, ctr] = cdata
            ctr = ctr + 1
        ndata = ndata.astype(float)
        ndata = ndata * 1.0 / 10000
        ndata = np.expand_dims(ndata, 0)
        # print(time.time()-start_time)
        cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                              average_over=4,
                                              dilation_size=2)
        cloud_matrix = cloud_detector.get_cloud_probability_maps(
            np.array(ndata))
        cloud_matrix = np.squeeze(cloud_matrix)
        cloud_matrix = np.round(cloud_matrix * 100).astype(np.uint16)

        ctr = 0
        for cband in aband_list:
            cind = aband_list.index(cband)
            cdata = ds.GetRasterBand(cind + 1).ReadAsArray()
            if ctr == 0:
                odata = np.zeros(
                    (cdata.shape[0], cdata.shape[1], len(aband_list) + 1))
            odata[:, :, ctr] = cdata
            ctr = ctr + 1

        odata[:, :, -1] = cloud_matrix
        odata = odata.astype(np.uint16)
        CreateGeoTiffs(odata, fpath, fpath, gdal.GDT_UInt16)
示例#10
0
def test_pixel_cloud_detector():
    data_path = os.path.join(os.path.dirname(__file__), 'TestInputs',
                             'input_arrays.npz')
    data = np.load(data_path)

    cloud_detector = S2PixelCloudDetector(all_bands=True)
    cloud_probs = cloud_detector.get_cloud_probability_maps(data['s2_im'])
    cloud_mask = cloud_detector.get_cloud_masks(data['s2_im'])

    assert np.allclose(cloud_probs, data['cl_probs'], atol=1e-14)
    assert np.array_equal(cloud_mask, data['cl_mask'])

    single_temporal_cloud_probs = cloud_detector.get_cloud_probability_maps(
        data['s2_im'][0, ...])
    single_temporal_cloud_mask = cloud_detector.get_cloud_masks(
        data['s2_im'][0, ...])

    assert np.array_equal(single_temporal_cloud_probs, cloud_probs[0, ...])
    assert np.array_equal(single_temporal_cloud_mask, cloud_mask[0, ...])

    cloud_detector = S2PixelCloudDetector(all_bands=False)
    with pytest.raises(ValueError):
        cloud_detector.get_cloud_probability_maps(data['s2_im'])
示例#11
0
def compute_cloud_mask(x: np.ndarray,
                       threshold: float = 0.4,
                       average_over: int = 4,
                       dilation_size: int = 2,
                       all_bands: bool = True):
    z = sentinel2_to_cloud_mask_preprocess(x)
    cloud_detector = S2PixelCloudDetector(threshold=threshold,
                                          average_over=average_over,
                                          dilation_size=dilation_size,
                                          all_bands=all_bands)

    cloud_mask = cloud_detector.get_cloud_probability_maps(z)
    cloud_mask = cloud_mask.squeeze()
    return cloud_mask
示例#12
0
 def __init__(self,
              clp_feature: Tuple = (FeatureType.DATA, 'CLP'),
              clm_feature: Tuple = (FeatureType.MASK, 'CLM'),
              average_over: int = 24,
              max_clp: float = 255.):
     """
     :param clp_feature: Feature type and name of input CLP mask
     :param clm_feature: Feature type and name of output CLM mask
     :param average_over: Parameter used ot smooth the CLP data map
     :param max_clp: Maximum value of CLP map used for normalization
     """
     self.clm_feature = next(self._parse_features(clm_feature)())
     self.clp_feature = next(self._parse_features(clp_feature)())
     self.s2_cd = S2PixelCloudDetector(average_over=average_over)
     self.max_clp = max_clp
示例#13
0
    def calculate_cloud_mask(self):
        bands_array = self._calculate_bands_array(self)
        cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                              average_over=4,
                                              dilation_size=2,
                                              all_bands=False)

        # cloud_prob = cloud_detector.get_cloud_probability_maps(bands_array)
        cloud_mask = cloud_detector.get_cloud_masks(bands_array)

        resized_mask = scipy.ndimage.zoom(cloud_mask[0], 6, mode='nearest')
        temp = np.multiply(resized_mask, -1)

        resized_mask_invertida = np.add(temp, 1)

        return resized_mask_invertida
示例#14
0
    def compute_cloud_mask(self,
                           threshold=0.4,
                           average_over=4,
                           dilation_size=2):
        cloud_detector = S2PixelCloudDetector(threshold=threshold,
                                              average_over=average_over,
                                              dilation_size=dilation_size,
                                              all_bands=False)
        bands_for_cloud_detector = [
            "B1", "B2", "B4", "B5", "B8", "B8A", "B9", "B10", "B11", "B12"
        ]
        cloud_detector_input = np.zeros((self.rgb.shape[0], self.rgb.shape[1],
                                         len(bands_for_cloud_detector)))
        for i, band in enumerate(bands_for_cloud_detector):
            cloud_detector_input[:, :, i] = self.bands_rhot[band]

        return cloud_detector.get_cloud_masks(cloud_detector_input)
示例#15
0
def detect_clouds(merged_tif_path, save_file):
    print('================================')
    print('Cloud detection.')
    cloud_detector = S2PixelCloudDetector()

    img = rasterio.open(merged_tif_path).read()
    img = reshape_as_image(img)
    img = np.expand_dims(img, axis=0) / 10000
    print('predict.')
    cloud_probs = cloud_detector.get_cloud_probability_maps(img)
    width = int(img.shape[2] * 4)
    height = int(img.shape[1] * 4)
    print('resize.')
    cloud_probs = cloud_probs.reshape((img.shape[1], img.shape[2]))
    cloud_probs = cv2.resize(cloud_probs, (width, height),
                             interpolation=cv2.INTER_CUBIC)

    print('save cloud.')
    imageio.imwrite(save_file, cloud_probs)
    def load_eo_data(self):
        all_bands_request, cloud_bands_request = self.create_requests()

        all_bands = all_bands_request.get_data(
            save_data=True, redownload=self.settings.redownload)
        print("downloaded all bands")

        cloud_bands = cloud_bands_request.get_data(
            save_data=True, redownload=self.settings.redownload)
        print("downloaded cloud bands")
        print(cloud_bands[0].shape)

        dates1 = all_bands_request.get_dates()
        dates = [j.date() for j in dates1]
        out = []
        for j in range(1, len(dates)):
            if dates[j] == dates[j - 1]:
                out.append(j)

        all_bands = np.delete(all_bands, out, 0)
        cloud_detector = S2PixelCloudDetector(
            average_over=self.settings.cloud_detection_settings.average_over,
            dilation_size=self.settings.cloud_detection_settings.dilation_size,
            threshold=self.settings.cloud_detection_settings.threshold,
            all_bands=True)
        cloud_masks_orig = cloud_detector.get_cloud_masks(np.array(all_bands))
        # upscale cloud masks
        cloud_masks = []
        for i in range(len(cloud_masks_orig)):
            cloud_masks.append(
                self.upscale_image(
                    cloud_masks_orig[i],
                    self.settings.cloud_detection_settings.x_scale))
        cloud_masks = np.array(cloud_masks)

        for j in out:
            dates1.pop(j)

        return all_bands, cloud_masks, dates1
    def get_cloud_saturation_mask(self):
        """
        Main acquisition method
        Acquires required data, generates cloud detections masks and saves them
        in memo_data for further use
        :return: Tuple[cloud_saturation_mask, true_color_bands, all_bands,
        data_acquisiton_dates, cloud_masks]
        """
        cloud_detector = S2PixelCloudDetector(**self.cloud_detection_config)
        true_color, bands, all_bands, dates = self.load_data()
        print("Downloaded")
        cloud_masks_orig = cloud_detector.get_cloud_masks(np.array(bands))
        # upscale cloud masks
        cloud_masks = []
        print(cloud_masks_orig[0].shape)
        for i in range(len(cloud_masks_orig)):
            cloud_masks.append(
                self.upscale_image(cloud_masks_orig[i], self.cloud_scale))
        cloud_masks = np.array(cloud_masks)
        print("Detected")
        # Images might be slightly out of scale, crop them
        x, y, _ = true_color[0].shape
        cloud_masks = cloud_masks[:, -x:, :y]
        off_image_detection_mask = sum(map(self.get_image_mask, true_color))
        # Just sum how many times we detect white pixels

        full_cloud_mask = sum(cloud_masks)
        # Get total mask by dividing number of cloud detections by number of all sensible pixels
        total_mask_w = (
            full_cloud_mask /
            (len(cloud_masks) - off_image_detection_mask)).astype(float)

        self.memo_data = CloudSaturation.MemoData(total_mask_w, true_color,
                                                  bands, all_bands, dates,
                                                  cloud_masks)

        # TODO: Refactor me to some nicer object
        return (total_mask_w, np.array(true_color), np.array(all_bands),
                np.array(dates), np.array(cloud_masks))
示例#18
0
#Calculo de probabilidades y obtención de mascaras de nubes
bands_script = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]'
wms_bands_request = WmsRequest(layer=LAYER_NAME,
                               custom_url_params={
                                   CustomUrlParam.EVALSCRIPT: bands_script,
                                   CustomUrlParam.ATMFILTER: 'NONE'
                               },
                               bbox=bounding_box,
                               time=(Date_Ini, Date_Fin),
                               width=x_width,
                               height=y_height,
                               image_format=MimeType.TIFF_d32f,
                               instance_id=INSTANCE_ID)
wms_bands = wms_bands_request.get_data()
cloud_detector = S2PixelCloudDetector(
    threshold=0.35, average_over=8, dilation_size=3)  #change threshold to test
cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands))
cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands))
all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
                                   threshold=0.1)

#folder de imagenes nubes
Path('output_clouds/' + analysis_area).mkdir(parents=True, exist_ok=True)
if not os.path.exists(analysis_area):
    os.makedirs(analysis_area)

#Mostrar las probabilidades de nubes para cada imagen por fecha en el rango de analisis
fig = plt.figure(figsize=(15, 10))
n_cols = 4
n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))
for idx, [prob, mask, data] in enumerate(all_cloud_masks):
示例#19
0
def apply_cloudmask(S2DotSafefp=None,
                    bands=None,
                    threshold=0.4,
                    dilation_size=3,
                    average_over=1,
                    resolution_safe=100,
                    save_mask=None,
                    crs=None,
                    affine=None):
    """
    Function for applying cloud mask to either S2 product in safe format or a multiple rasters
    in numpy ndarray format. ndarray format is reccomended

    :param S2DotSafefp: path to S2 file
    :param bands: ndarray (N, H, W, B) N is the number of rasters, H is height, W is width, B are bands
    :param resolution_safe: Pixel resolution for cloud mask for .safe format in meters,
                            recommended to use higher than 60 as it's much more efficient.
    :param dilation_size: dilation for cloud pixels
    :param threshold: threshold for the cloud probability
    :param average_over: neighborhood of averaging
    :param save_mask:
    :param crs: If save_mask => crs for the saved mask
    :param affine: Affine transformation for the saved mask
    :return: cloud mask (N, H, W) cloud mask for each raster, if .safe format the output shape will be
            (1, H, W)

    """
    # If the input is in safe format
    if S2DotSafefp is not None:
        # resolutions for each band
        res = np.array([60, 10, 10, 20, 10, 20, 60, 60, 20, 20])
        # Valid bands
        bands_ = [
            'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11',
            'B12'
        ]
        # list of rasters
        S2rasters = [
            glob(
                os.path.join(S2DotSafefp, 'GRANULE', 'L*', 'IMG_DATA',
                             '*{}*'.format(band)))[0] for band in bands_
        ]
        # Affine transformation list
        afftr = res / resolution_safe
        S2_cloud_bands = []

        # iterate through band rasters
        for i, raster in enumerate(S2rasters):
            with rasterio.open(raster) as src:
                # Divide by 10000 to get TOA values
                arr = src.read().astype('float32') / 10000
                aff = src.transform
                # Define new affine transform
                new_aff = Affine(aff.a / afftr[i], aff.b, aff.c, aff.d,
                                 aff.e / afftr[i], aff.f)
                # Empty array for destination
                newarr = np.empty(shape=(arr.shape[0],
                                         int(round(arr.shape[1] * afftr[i])),
                                         int(round(arr.shape[2] * afftr[i]))),
                                  dtype='float32')
                # Apply downsampling
                warp.reproject(arr,
                               newarr,
                               src_transform=aff,
                               dst_transform=new_aff,
                               src_crs=src.crs,
                               dst_crs=src.crs)
                # Append to list
                S2_cloud_bands.append(newarr)
        # Concatenate the list and add a dim for compatibility with S2Cloudless function
        S2_cloud_bands = np.concatenate(S2_cloud_bands, axis=0)

        S2_cloud_bands = np.expand_dims(S2_cloud_bands, axis=0)
        # Apply cloudmask
        cloudmask = S2PixelCloudDetector(
            threshold=threshold,
            dilation_size=dilation_size,
            average_over=average_over,
            all_bands=False).get_cloud_masks(S2_cloud_bands)

        # save mask if path is specified
        if save_mask is not None:
            with rasterio.open(S2rasters[0]) as src:
                kwargs = src.profile
                kwargs['width'] = cloudmask.shape[1]
                kwargs['height'] = cloudmask.shape[2]
                kwargs['transform'] = new_aff
                kwargs['driver'] = 'Gtiff'
                kwargs['dtype'] = cloudmask.dtype
                kwargs['blockxsize'] = 128
                kwargs['blockysize'] = 128

                with rasterio.open(save_mask, 'w', **kwargs) as dst:
                    dst.write(cloudmask)
        # return the cloudmask as np ndarray
        return cloudmask
    # If valid bands are given
    elif bands is not None:

        cloudmask = S2PixelCloudDetector(
            threshold=threshold,
            dilation_size=dilation_size,
            average_over=average_over,
            all_bands=False).get_cloud_masks(bands)

        # save mask if path is specified
        if save_mask is not None and crs is not None:
            kwargs = {
                'driver': 'GTiff',
                'interleave': 'band',
                'tiled': True,
                'blockxsize': 128,
                'blockysize': 128,
                'nodata': 0,
                'dtype': cloudmask.dtype,
                'height': cloudmask.shape[1],
                'width': cloudmask.shape[2],
                'crs': crs,
                'transform': affine
            }

            with rasterio.open(save_mask, 'w', **kwargs) as dst:
                dst.write(cloudmask)
        elif save_mask is not None and crs is None:
            raise ValueError("To save the mask, crs needs to be provided")
        # return the cloudmask as np ndarray
        return cloudmask
    else:
        raise ValueError(
            "You need to provide a path to a Sentinel-2 .safe file, or a numpy ndarray consisting of the bands"
        )
示例#20
0
wms_bands_request = WmsRequest(layer=LAYER_NAME,
                               custom_url_params={
                                   CustomUrlParam.EVALSCRIPT: bands_script,
                                   CustomUrlParam.ATMFILTER: 'NONE'
                               },
                               bbox=bounding_box,
                               time=('2017-12-01', '2017-12-31'),
                               width=600,
                               height=None,
                               image_format=MimeType.TIFF_d32f,
                               instance_id=INSTANCE_ID)
wms_bands = wms_bands_request.get_data()

cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                      average_over=4,
                                      dilation_size=2)
cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands))
cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands))
image_idx = 0
overlay_cloud_mask(wms_true_color_imgs[image_idx], cloud_masks[image_idx])
plot_probability_map(wms_true_color_imgs[image_idx], cloud_probs[image_idx])
plot_cloud_mask(cloud_masks[image_idx])

all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
                                   threshold=0.1)
fig = plt.figure(figsize=(15, 10))
n_cols = 4
n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))

for idx, [prob, mask, data] in enumerate(all_cloud_masks):
示例#21
0
        width_16 = width // 16
        height_16 = height // 16

        small_data = np.zeros((13, width_16, height_16), dtype=np.uint16)
        rasterio.warp.reproject(data[0:13],
                                small_data,
                                src_transform=geoTransform,
                                src_crs=crs,
                                dst_transform=geoTransform *
                                rasterio.transform.Affine.scale(16, 16),
                                dst_crs=crs,
                                resampling=rasterio.enums.Resampling.nearest)
        small_data = np.transpose(small_data, axes=(1, 2, 0))
        small_data = small_data.reshape(1, width_16, height_16, 13) / 1e5
        cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                              average_over=4,
                                              dilation_size=1,
                                              all_bands=True)
        small_tmp = cloud_detector.get_cloud_probability_maps(small_data)
        try:
            quantile = np.quantile(
                np.extract(small_tmp > np.min(small_tmp), small_tmp), 0.20)
        except:
            quantile = 0.0033
        cutoff = max(0.0033, quantile)
        small_tmp = (small_tmp > cutoff).astype(np.uint16)

        tmp = np.zeros((1, width, height), dtype=np.uint16)
        rasterio.warp.reproject(small_tmp,
                                tmp,
                                src_transform=geoTransform *
                                rasterio.transform.Affine.scale(16, 16),
示例#22
0
    def cloud_detection(self, input_file):
        print("cloud_detection", input_file)
        input_dir = os.path.join(input_file, "GRANULE")
        sub_directories = utils.get_immediate_subdirectories(input_dir)
        image_dir = os.path.join(input_dir, sub_directories[0], "IMG_DATA")

        input_bands = [
            'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11',
            'B12'
        ]  # Band order is strict
        # num_bands = len(input_bands)
        scale_factor = 10000.0  #Read from metadata ?
        band_paths = self.get_band_paths(input_file, input_bands)
        for band_ind, img_filename in enumerate(band_paths):
            with rasterio.open(img_filename) as ds:
                img = ds.read()
                if band_ind == 0:  # First band need to be 60m
                    tmparr = np.empty_like(img)
                    aff60 = ds.transform
                    img_stack = np.zeros(
                        (img.shape[0], img.shape[1], img.shape[2],
                         len(input_bands)))
                    img_stack[:, :, :, band_ind] = img / scale_factor
                elif input_bands[band_ind].upper(
                ) == "B09" or input_bands[band_ind].upper() == "B10":  # 60m
                    img_stack[:, :, :, band_ind] = img / scale_factor
                else:
                    reproject(img,
                              tmparr,
                              src_transform=ds.transform,
                              dst_transform=aff60,
                              src_crs=ds.crs,
                              dst_crs=ds.crs,
                              resampling=Resampling.bilinear)
                    img_stack[:, :, :, band_ind] = tmparr / scale_factor

                if input_bands[band_ind].upper() == "B02":  # 10m
                    aff10 = ds.transform
                    nrows10 = img.shape[1]
                    ncols10 = img.shape[2]
                    ds10 = ds

        cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                              average_over=4,
                                              dilation_size=2)
        cloud_probs = cloud_detector.get_cloud_probability_maps(img_stack)
        cloud_mask = cloud_detector.get_cloud_masks(img_stack).astype(
            rasterio.uint8)

        cloud_probs_10 = np.zeros((1, nrows10, ncols10))
        reproject(cloud_probs,
                  cloud_probs_10,
                  src_transform=aff60,
                  dst_transform=aff10,
                  src_crs=ds.crs,
                  dst_crs=ds.crs,
                  resampling=Resampling.cubic_spline)

        cloud_mask_10 = np.zeros((1, nrows10, ncols10))
        reproject(cloud_mask,
                  cloud_mask_10,
                  src_transform=aff60,
                  dst_transform=aff10,
                  src_crs=ds.crs,
                  dst_crs=ds.crs,
                  resampling=Resampling.nearest)

        return (cloud_probs_10, cloud_mask_10, ds10)
示例#23
0
def cloud_masking(gdir):
    """
    Masks cloudy pixels with s2cloudless algorithm

    - Reads necessary bands from ekstrand corrected netCDF file
    - Applies s2cloudless algorithm to scene to get cloud mask
    (https://github.com/sentinel-hub/sentinel2-cloud-detector)
    - Applies cloud mask to scene
    - saves processed scene with all bands in cloud_masked.nc file

    Parameters:
    -----------
    gdir: :py:class:`crampon.GlacierDirectory`
        A GlacierDirectory instance.

    Returns:
    --------
    None

    """

    cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                          average_over=2,
                                          dilation_size=2)

    try:
        sentinel = xr.open_dataset(gdir.get_filepath('ekstrand'))
    except FileNotFoundError:
        # Check if uncorrected sentinel file exists:
        try:
            sentinel = xr.open_dataset(
                gdir.get_filepath('sentinel'))  #, chunks={'time':10})
        except FileNotFoundError:  # No data exists at all
            # 1st time step: no data for this glacier available
            log.info('No data for this glacier - Abort Cloud Masking')
            return

    try:
        wms_bands = sentinel.sel(
        band=['B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12'],
        time=cfg.PARAMS['date'][0])\
        .img_values.values
    except KeyError:
        # Check if uncorrected Sentinel File exists/has data:
        try:
            sentinel = xr.open_dataset(
                gdir.get_filepath('sentinel'))  #,chunks={'time':10} )
            wms_bands = sentinel.sel(
            band=['B01', 'B02', 'B04','B05','B08','B8A','B09', 'B10', 'B11', 'B12'],
            time=cfg.PARAMS['date'][0])\
                    .img_values.values
        except KeyError:
            # No data for this glacier & date
            log.info('No data for this glacier and date - Abort Cloud Masking')
            return

    # rearrange dimensions, goal :[height, width, channels].
    #  now: [channels, height, width] and correct into  float (divide by factor 10000)
    wms_bands = [np.transpose(wms_bands / 10000, (1, 2, 0)) for _ in range(1)]
    cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands))

    #    cloud_probability = cloud_detector.get_cloud_probability_maps(np.array(wms_bands))
    #    plot_cloud_mask(cloud_probability ,wms_bands)

    ##### Deal with MemoryError for large files: remove last entry
    #sentinel=sentinel.isel(time=range(0,sentinel.time.values.size-1))

    for band_id in sentinel['band'].values:
        band_array = sentinel.sel(band=[band_id],
                                  time=cfg.PARAMS['date'][0]).img_values.values
        # Set threshold to exclude glaciers with more than 60% cloud cover
        #  -> no useful classification possible
        image = sentinel.sel(band=[band_id],
                             time=cfg.PARAMS['date'][0]).img_values.values
        band_array[cloud_masks == 1] = 0
        try:
            cloud_cover = 1 - len(band_array[band_array > 0]) / len(
                image[image > 0])
        except ZeroDivisionError:
            # for 100 % cloud cover:
            log.info: ('Cloud Cover is 100%- abort Cloud Masking')
            return

        if cloud_cover > 0.7:
            # -> set all pixels to 0
            log.info('Cloud COver over 70% - abort Cloud Masking')
            return
        band_array = band_array[0, :, :]

        if os.path.isfile(gdir.get_filepath('cloud_masked')):
            cloud_mask = xr.open_dataset(gdir.get_filepath('cloud_masked'))
            cloud_mask_new = cloud_mask.copy(deep=True)
            cloud_mask_new = cloud_mask_new.isel(time=0)
            cloud_mask_new.coords['time'] = np.array([cfg.PARAMS['date'][0]])
            cloud_mask = xr.concat([cloud_mask, cloud_mask_new], dim='time')
            os.remove(gdir.get_filepath('cloud_masked'))
        else:
            if not 'cloud_mask' in locals():
                # File does not exist: Create new as copy from sentinel.nc file:
                sentinel_copy = sentinel.copy(deep=True)
                # Remove all dates except current:
                sentinel_copy = sentinel_copy.drop(
                    [time_id for time_id in sentinel['time'].values][:-1],
                    dim='time').squeeze('time', drop=True)
                sentinel_copy = sentinel_copy.assign_coords(
                    time=cfg.PARAMS['date'][0])
                sentinel_copy = sentinel_copy.expand_dims('time')
                cloud_mask = sentinel_copy.copy(deep=True)
        cloud_mask['img_values'].loc[(
                     dict(band = band_id, time=cfg.PARAMS['date'][0]))] \
                    = np.ones([band_array.shape[0], band_array.shape[1]])

        cloud_mask['img_values'].loc[(
                    dict(band=band_id, time=cfg.PARAMS['date'][0]))] \
                    = band_array

    print("Cloud cover: ", cloud_cover)

    # Write Updated DataSet to file
    cloud_mask.to_netcdf(gdir.get_filepath('cloud_masked'), 'w')
    cloud_mask.close()
def convert_sentinel2(path_to_SAFE_folder,
                      output_resolution=10,
                      cloud_detection_resolution=60,
                      n_threads=10,
                      delete_safe_file=True):

    warnings.filterwarnings("ignore", category=DeprecationWarning)

    if path_to_SAFE_folder[-1] == '/':
        path_to_SAFE_folder = path_to_SAFE_folder[:-1]
    tile_name = path_to_SAFE_folder.split('/')[-1].replace('.SAFE', '')

    print('Processing:', tile_name)

    #Explore content in folder to find image-folder:
    input_dir = os.path.join(path_to_SAFE_folder, "GRANULE")
    sub_directories = [
        name for name in os.listdir(input_dir)
        if os.path.isdir(os.path.join(input_dir, name))
    ]
    image_dir = os.path.join(input_dir, sub_directories[0], "IMG_DATA")

    #Read all band
    scale_factor = 10000
    band_data = {}
    t = time.time()
    time_left = 0

    for img_dir, dirnames, filenames in os.walk(image_dir):
        for band_ind, band_name in enumerate(_all_bands):
            if band_name in _bands_10m:
                resolution = 10
            elif band_name in _bands_20m:
                resolution = 20
            elif band_name in _bands_60m:
                resolution = 60

            for filename in filenames:

                #L2a products have multiple resolutions for each band.
                l2a_prod_res = filename.split('.jp2')[0].split('_')[-1]
                if l2a_prod_res in ['10m', '20m', '60m']:
                    l2a_prod_res = int(l2a_prod_res.split('m')[0])
                    if l2a_prod_res > output_resolution and l2a_prod_res > resolution:
                        continue

                if fnmatch.filter([filename],
                                  "*" + band_name.upper() + "*.jp2"):
                    path_to_jp2 = os.path.join(img_dir, filename)

                    # Assuming that os.walk loop through folders alphabetically, this ensure that the higest resolution is chosen if there exist multiple resolutions pr file
                    if band_name not in band_data:
                        print(' - Reading', band_name, '-',
                              len(band_data) + 1, 'of', len(_all_bands), '-',
                              'Estimated time left:',
                              str(datetime.timedelta(seconds=time_left)))

                        with rasterio.open(path_to_jp2) as ds:
                            band_data[band_name] = {
                                'data': ds.read() / scale_factor,
                                'transform': ds.transform,
                                'crs': ds.crs,
                                'res': resolution
                            }

                        time_left = (time.time() - t) / (len(band_data)) * (
                            len(_all_bands) - len(band_data))

    #Collect sentral parameters for each resolution
    transforms = {
        10: band_data[_bands_10m[0]]['transform'],
        20: band_data[_bands_20m[0]]['transform'],
        60: band_data[_bands_60m[0]]['transform'],
    }
    crss = {
        10: band_data[_bands_10m[0]]['crs'],
        20: band_data[_bands_20m[0]]['crs'],
        60: band_data[_bands_60m[0]]['crs'],
    }
    shapes = {
        10: band_data[_bands_10m[0]]['data'].shape,
        20: band_data[_bands_20m[0]]['data'].shape,
        60: band_data[_bands_60m[0]]['data'].shape,
    }
    target_transform = transforms[output_resolution]
    target_crs = crss[output_resolution]

    #Reprojections
    def reproject_to_resolution(data, res_from, res_to):

        if res_to == res_from:
            return data
        tmp_array = np.zeros(shapes[res_to])
        reproject(data,
                  tmp_array,
                  src_transform=transforms[res_from],
                  dst_transform=transforms[res_to],
                  src_crs=crss[res_from],
                  dst_crs=crss[res_to],
                  num_threads=n_threads,
                  resampling=Resampling.bilinear)

        return tmp_array

    ####################################################################################################################
    # Perform cloud detection using
    # https://github.com/sentinel-hub/sentinel2-cloud-detector/blob/master/examples/sentinel2-cloud-detector-example.ipynb

    #Convert all bands to 60m for cloud-detection
    print(" - Cloud detection running")
    bands_for_cloud_detection = [
        reproject_to_resolution(band_data[band_name]['data'],
                                band_data[band_name]['res'],
                                cloud_detection_resolution)
        for band_name in _bands_used_in_cloud_detection
    ]
    bands_for_cloud_detection = np.concatenate(
        [np.expand_dims(np.squeeze(b), -1) for b in bands_for_cloud_detection],
        -1)
    bands_for_cloud_detection = np.expand_dims(bands_for_cloud_detection, 0)
    cloud_detector = S2PixelCloudDetector(threshold=0.4,
                                          all_bands=False,
                                          average_over=4,
                                          dilation_size=2)
    cloud_mask = cloud_detector.get_cloud_masks(bands_for_cloud_detection)[0]

    #Save to output resolution
    cloud_mask = reproject_to_resolution(cloud_mask,
                                         cloud_detection_resolution,
                                         output_resolution)

    print(" - Cloud detection finished")

    # Save memory maps
    print(" - Creating 10m resolution data-cube")
    bands = [
        "b02", "b03", "b04", "b08", "b05", "b06", "b07", "b8a", "b11", "b12",
        "b01", "b09", "b10"
    ]
    bands_data = []
    for band_name in bands:
        data_at_res = reproject_to_resolution(band_data[band_name]['data'],
                                              band_data[band_name]['res'],
                                              output_resolution)
        data_at_res[data_at_res == 0] = np.nan  #Insert np.nan as nodata value
        bands_data.append(data_at_res.squeeze().astype('float16'))

    bands_data = [d[:, :, None] for d in bands_data]
    bands_data = np.concatenate(bands_data, -1)

    if delete_safe_file:
        print(' - Deleting SAFE file', path_to_SAFE_folder)
        shutil.rmtree(path_to_SAFE_folder)

    warnings.filterwarnings("default", category=DeprecationWarning)

    return bands_data, cloud_mask, target_transform, target_crs
示例#25
0
    def mask_s2(data,
                solar_za,
                solar_az,
                cloud_heights=None,
                nodata=None,
                scale_factor=1,
                num_workers=1,
                **kwargs):
        """
        Masks Sentinel 2 data

        Args:
            data (DataArray): The Sentinel 2 data to mask.
            solar_za (DataArray): The solar zenith angle.
            solar_az (DataArray): The solar azimuth angle.
            cloud_heights (Optional[list]): A list of potential cloud heights.
            nodata (Optional[int or float]): A 'no data' value to fill NAs with.
            scale_factor (Optional[float]): A scale factor to apply to the data.
            num_workers (Optional[int]): The number of parallel compute workers.
            kwargs (Optional[dict]): Keyword arguments for ``s2cloudless.S2PixelCloudDetector``.

        Returns:

            ``xarray.DataArray``:

                Data range: 0 to 4, where 0=clear; 2=shadow; 4=cloud

        Example:
            >>> import geowombat as gw
            >>> from geowombat.radiometry import mask_s2
            >>>
            >>> with gw.config.update(sensor='s2f', scale_factor=0.0001):
            >>>
            >>>     with gw.open('image.tif') as src, \
            >>>         gw.open('solar_zenith.tif') as sza, \
            >>>             gw.open('solar_azimuth.tif') as saa:
            >>>
            >>>         s2_mask = mask_s2(src, sza, saa)
        """

        # from ..radiometry.mask import CloudShadowMasker
        # mask_s2 = CloudShadowMasker().mask_s2
        #
        # mask = mask_s2(data,
        #                sza,
        #                saa,
        #                scale_factor=0.0001,
        #                nodata=0,
        #                num_workers=num_threads)
        #
        # fnmask = Path(load_bands_names[0]).name.split('.')[0]
        # mask.gw.to_raster(f'/media/jcgr/data/projects/global_fields/data/grids/ms/test/000960/{fnmask}_mask.tif',
        #                   n_workers=1, n_threads=1)
        #
        # if bands_out:
        #     data = _assign_attrs(data, attrs, bands_out)

        new_attrs = data.attrs.copy()

        if not cloud_heights:
            cloud_heights = list(range(500, 2000, 500))

        if not isinstance(nodata, int) and not isinstance(nodata, float):
            nodata = data.gw.nodata

        if scale_factor == 1.0:
            scale_factor = data.gw.scale_factor

        if S2CLOUDLESS_INSTALLED:

            if not kwargs:

                kwargs = dict(threshold=0.4,
                              average_over=4,
                              dilation_size=5,
                              all_bands=False)

            cloud_detector = S2PixelCloudDetector(**kwargs)

            # Get the S2Cloudless bands
            data_cloudless = data.sel(band=[
                'coastal', 'blue', 'red', 'nir1', 'nir', 'rededge', 'water',
                'cirrus', 'swir1', 'swir2'
            ])

            # Scale from 0-10000 to 0-1
            if isinstance(nodata, int) or isinstance(nodata, float):
                data_cloudless = xr.where(data_cloudless != nodata,
                                          data_cloudless * scale_factor,
                                          nodata).clip(0, 1).astype('float64')
            else:
                data_cloudless = (data_cloudless * scale_factor).clip(
                    0, 1).astype('float64')

            # Reshape for predictions ..
            #   from [bands x rows x columns]
            #   to [images x rows x columns x bands]
            X = data_cloudless.transpose('y', 'x', 'band').data.compute(
                num_workers=num_workers)[np.newaxis, :, :, :]

            ################
            # Predict clouds
            ################

            # Convert from NumPy array to DataArray
            # clear=0, clouds=1
            cloud_mask = ndarray_to_xarray(data,
                                           cloud_detector.get_cloud_masks(X),
                                           [1])

            #################
            # Predict shadows
            #################

            # Scale the angles to degrees
            sza = solar_za * 0.01
            sza.coords['band'] = [1]

            saa = solar_az * 0.01
            saa.coords['band'] = [1]

            # Convert to radians
            rad_sza = xr.ufuncs.deg2rad(sza)
            rad_saa = xr.ufuncs.deg2rad(saa)

            # non-shadow=0, shadows=1
            shadow_mask = estimate_shadows(data, cloud_mask, rad_sza, rad_saa,
                                           cloud_heights, nodata, scale_factor,
                                           num_workers)

            # Recode for final output
            mask = xr.where(
                cloud_mask.sel(band=1) == 1, 4,
                xr.where(
                    shadow_mask.sel(band=1) == 1, 2,
                    xr.where(data.max(dim='band') == nodata, 255,
                             0))).expand_dims(dim='band').astype('uint8')

            mask = mask.assign_coords(coords={'band': ['mask']})

            new_attrs['nodatavals'] = (255)
            new_attrs['scales'] = (1.0)
            new_attrs['offsets'] = (0.0)
            new_attrs['pre-scaling'] = scale_factor
            new_attrs['sensor'] = 's2'
            new_attrs['clearval'] = (0)
            new_attrs['shadowval'] = (2)
            new_attrs['cloudval'] = (4)
            new_attrs['fillval'] = (255)

            mask = mask.assign_attrs(**new_attrs)

        else:

            logger.warning('  S2Cloudless is not installed.')

            mask = None

        return mask
def cloud_detection(input_path, cloudmask, cloudprobs):
    for file in sorted(os.listdir(input_path)):
        if file.endswith("B01.jp2"):
            ds = gdal.Open(os.path.join(input_path, file), gdal.GA_ReadOnly)
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B01 = scl.read()
                tmparr = np.empty_like(B01)
                aff = scl.transform
                print(B01.shape)
        if file.endswith("B02.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B02 = scl.read()
                reproject(
                    B02, tmparr,
                    src_transform=scl.transform,
                    dst_transform=aff,
                    src_crs=scl.crs,
                    dst_crs=scl.crs,
                    resampling=Resampling.bilinear)
                B02 = tmparr
                print(B02.shape)
        if file.endswith("B04.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B04 = scl.read()
                reproject(
                   B04, tmparr,
                   src_transform=scl.transform,
                   dst_transform=aff,
                   src_crs=scl.crs,
                   dst_crs=scl.crs,
                   resampling=Resampling.bilinear)
                B04 = tmparr
                print(B04.shape)
        if file.endswith("B05.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B05 = scl.read()
                reproject(
                   B05, tmparr,
                   src_transform=scl.transform,
                   dst_transform=aff,
                   src_crs=scl.crs,
                   dst_crs=scl.crs,
                   resampling=Resampling.bilinear)
                B05 = tmparr
                print(B05.shape)
        if file.endswith("B08.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B08 = scl.read()
                reproject(
                   B08, tmparr,
                   src_transform=scl.transform,
                   dst_transform=aff,
                   src_crs=scl.crs,
                   dst_crs=scl.crs,
                   resampling=Resampling.bilinear)
                B08 = tmparr
                print(B08.shape)
        if file.endswith("B8A.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B8A = scl.read()
                reproject(
                   B8A, tmparr,
                   src_transform=scl.transform,
                   dst_transform=aff,
                   src_crs=scl.crs,
                   dst_crs=scl.crs,
                   resampling=Resampling.bilinear)
                B8A = tmparr
                print(B8A.shape)
        if file.endswith("B09.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B09 = scl.read()
        if file.endswith("B10.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B10 = scl.read()
        if file.endswith("B11.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B11 = scl.read()
                reproject(
                   B11, tmparr,
                   src_transform=scl.transform,
                   dst_transform=aff,
                   src_crs=scl.crs,
                   dst_crs=scl.crs,
                   resampling=Resampling.bilinear)
                B11 = tmparr
                print(B11.shape)
        if file.endswith("B12.jp2"):
            with rasterio.open(os.path.join(input_path, file)) as scl:
                B12 = scl.read()
                reproject(
                   B12, tmparr,
                   src_transform=scl.transform,
                   dst_transform=aff,
                   src_crs=scl.crs,
                   dst_crs=scl.crs,
                   resampling=Resampling.bilinear)
                B12 = tmparr
                print(B12.shape)
            print(B12.shape)

    bands = np.array([np.dstack((B01[0] / 10000.0, B02[0] / 10000.0, B04[0] / 10000.0, B05[0] / 10000.0, B08[0] / 10000.0,
                                B8A[0] / 10000.0, B09[0] / 10000.0, B10[0] / 10000.0, B11[0] / 10000.0,
                                B12[0] / 10000.0))])
    print(bands.shape)
    cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2)
    cloud_probs = cloud_detector.get_cloud_probability_maps(bands)
    mask = cloud_detector.get_cloud_masks(bands)#.astype(rasterio.uint8)

    # write the cloud mask and cloud probability map
    driver = gdal.GetDriverByName("GTiff")
    outdata = driver.Create(cloudmask, ds.RasterXSize, ds.RasterYSize, 1, gdal.GDT_UInt16)
    outdata.SetGeoTransform(ds.GetGeoTransform())
    outdata.SetProjection(ds.GetProjection())##sets same projection as input
    outband = outdata.GetRasterBand(1)
    outband.WriteArray(mask[0])
    outband.FlushCache() ##saves to disk!!
    del outband
    del outdata
    trim_processed(shapefile_link, cloudmask, cloudmask_clipped)
    os.remove(cloudmask)
    outdata = driver.Create(cloudprobs, ds.RasterXSize, ds.RasterYSize, 1, gdal.GDT_Float32)
    outdata.SetGeoTransform(ds.GetGeoTransform())
    outdata.SetProjection(ds.GetProjection())##sets same projection as input
    outband = outdata.GetRasterBand(1)
    outband.WriteArray(cloud_probs[0])
    outband.FlushCache() ##saves to disk!!
    del outband
    del outdata
    trim_processed(shapefile_link, cloudprobs, cloudprobs_clipped)
    os.remove(cloudprobs)
示例#27
0

@pytest.fixture(name='config')
def config_fixture():
    config = SHConfig()

    for param in config.get_params():
        env_variable = param.upper()
        if os.environ.get(env_variable):
            setattr(config, param, os.environ.get(env_variable))

    return config


@pytest.mark.parametrize('input_params,stats', [
    (dict(cloud_detector=S2PixelCloudDetector(
        threshold=0.6, average_over=2, dilation_size=5, all_bands=True),
          bbox=BBOX1,
          time=('2017-12-01', '2017-12-31'),
          size=(60, 81)),
     dict(clm_min=0,
          clm_max=1,
          clm_mean=0.343827,
          clm_median=0,
          clp_min=0.00011,
          clp_max=0.99999,
          clp_mean=0.23959,
          clp_median=0.01897,
          mask_shape=(7, 81, 60))),
    (dict(cloud_detector=S2PixelCloudDetector(),
          bbox=BBOX2,
          time=[dt.datetime(2016, 7, 18, 7, 14, 4)],
def generate_cloud_masks(array: np.ndarray) -> np.ndarray:
    """ Generate binary cloud masks with s2cloudless """
    cloud_detector = S2PixelCloudDetector(threshold=CLOUD_THRESHOLD,
                                          all_bands=True)
    return np.squeeze(cloud_detector.get_cloud_masks(array)).astype("byte")
示例#29
0
    def cloud_process(bounding_box, Date_Ini, Date_Fin, x_width, y_height,
                      analysis_area, clouds_folder, lote_aoi, municipio,
                      departamento):
        INSTANCE_ID = '3a63d637-11ad-493a-b921-91be7c4da68d'  #From Sentinel HUB Python Instance ID /change to dynamic user input
        LAYER_NAME = 'TRUE-COLOR-S2-L1C'  # e.g. TRUE-COLOR-S2-L1C
        #Obtener imagenes por fecha (dentro de rango) dentro de box de interés
        wms_true_color_request = WmsRequest(
            layer=LAYER_NAME,
            bbox=bounding_box,
            time=(Date_Ini, Date_Fin),  #cambiar a fechas de interés
            width=x_width,
            height=y_height,
            image_format=MimeType.PNG,
            time_difference=datetime.timedelta(hours=2),
            instance_id=INSTANCE_ID)
        wms_true_color_imgs = wms_true_color_request.get_data()
        #Cloudless_tools.plot_previews(np.asarray(wms_true_color_imgs), wms_true_color_request.get_dates(), cols=4, figsize=(15, 10))

        #count of 0's to know how empty is the image
        count_of_zeros = []
        for n in range(0, len(wms_true_color_imgs)):
            # zeros / 4 channels * width * height (pixels)
            count_of_zeros.append(
                (np.count_nonzero(wms_true_color_imgs[n] == 0)) /
                (4 * wms_true_color_imgs[n][:, :, 0].shape[0] *
                 wms_true_color_imgs[n][:, :, 0].shape[1]))

        #Calculo de probabilidades y obtención de mascaras de nubes
        bands_script = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]'
        wms_bands_request = WmsRequest(
            layer=LAYER_NAME,
            custom_url_params={
                CustomUrlParam.EVALSCRIPT: bands_script,
                CustomUrlParam.ATMFILTER: 'NONE'
            },
            bbox=bounding_box,
            time=(Date_Ini, Date_Fin),
            width=x_width,
            height=y_height,
            image_format=MimeType.TIFF_d32f,
            time_difference=datetime.timedelta(hours=2),
            instance_id=INSTANCE_ID)
        wms_bands = wms_bands_request.get_data()
        #wms_bands_request.get_filename_list()
        #wms_bands_request.get_url_list()
        #wms_bands_request.get_dates()
        cloud_detector = S2PixelCloudDetector(
            threshold=0.35, average_over=8,
            dilation_size=3)  #change threshold to test
        #cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands))
        cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands))
        all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
                                           threshold=0.35)
        #cloud_masks = all_cloud_masks.get_cloud_masks()

        #Mostrar las probabilidades de nubes para cada imagen por fecha en el rango de analisis
        n_cols = 4
        n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))
        fig = plt.figure(figsize=(n_cols * 4,
                                  n_rows * 3))  #, constrained_layout=False
        for idx, [prob, mask, data] in enumerate(all_cloud_masks):
            ax = fig.add_subplot(n_rows, n_cols, idx + 1)
            image = wms_true_color_imgs[idx]
            Cloudless_tools.overlay_cloud_mask(image, mask, factor=1, fig=fig)
        plt.tight_layout()
        plt.savefig(clouds_folder + analysis_area + '/real_and_cloud.png')

        #Mostrar las mascaras de nubes para cada imagen por fecha en el rango de analisis
        n_cols = 4
        n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))
        fig = plt.figure(figsize=(n_cols * 4, n_rows * 3))
        #each_cld_mask = all_cloud_masks.get_cloud_masks(threshold=0.35)
        cld_per_idx = []
        for idx, cloud_mask in enumerate(
                all_cloud_masks.get_cloud_masks(threshold=0.35)):
            ax = fig.add_subplot(n_rows, n_cols, idx + 1)
            #correct mask, when no data is in the image, to mask non values
            cloud_mask[wms_true_color_imgs[idx][:, :, 0] == 0] = 1
            Cloudless_tools.plot_cloud_mask(cloud_mask, fig=fig)
            n_cloud_mask = np.shape(np.concatenate(cloud_mask))
            cloud_perc = sum(np.concatenate(cloud_mask) == 1) / n_cloud_mask
            cld_per_idx.append(cloud_perc.astype(float))
        plt.tight_layout()
        plt.savefig(clouds_folder + analysis_area + '/cloud_masks.png')

        #Calculo y extracción de imagenes con cobertura de nubes menor a x%
        x = pd.DataFrame(
            cld_per_idx
        ) < 0.6  #Menor a 60% de cobertura de nubes // or lote is visible TO ADD
        all_dates = pd.DataFrame(all_cloud_masks.get_dates())
        valid_dates = all_dates[x[0]]
        all_dates['year-month'] = all_dates[0].dt.to_period('M')
        all_dates['cld_percent'] = cld_per_idx
        all_dates['empty_percent'] = count_of_zeros
        all_dates = all_dates.rename(columns={0: 'dates'})
        #summary
        '''
        summary_clds = all_dates[['year-month','cld_percent','dates']].groupby('year-month').agg({'dates':lambda x: x.diff().mean(), 'cld_percent': ['count', lambda x: (x<0.6).sum(), lambda x: x.mean(), 'min']}) \
            .reset_index()
        '''
        def f_mi(x):
            d = []
            d.append(x['dates'].diff().mean())
            d.append(x['cld_percent'].count())
            d.append((x['cld_percent'] < 0.6).sum())
            d.append(x['cld_percent'].mean())
            d.append(x['cld_percent'].min())
            d.append(x[x['cld_percent'] < 0.6]['dates'].max())
            d.append(x[x['cld_percent'] < 0.6]['dates'].min())
            d.append(x['empty_percent'].max())
            d.append(x['empty_percent'].min())
            return pd.Series(d,
                             index=[
                                 'time_between_pass', 'count_pass',
                                 'clear_images', 'mean_cloud_cover',
                                 'min_cloud_cover', 'last_good_date',
                                 'first_good_date', 'max_empty_space',
                                 'min_empty_space'
                             ])  #

        summary_clds = all_dates.groupby('year-month').apply(f_mi)
        summary_clds['centroid_x'], summary_clds['centroid_y'], summary_clds[
            'terrain_name'], summary_clds['terrain_code'], summary_clds[
                'municipio'], summary_clds['departamento'] = lote_aoi['x'][
                    0], lote_aoi['y'][0], lote_aoi['name'][
                        0], analysis_area, municipio, departamento
        #export data
        summary_clds.to_csv(clouds_folder + analysis_area +
                            '/Analisis_nubes.csv',
                            index=True,
                            header=True)

        #filter clouds dataframe with only valid dates
        clouds_data = cloud_masks[x[0]]
        minIndex = cld_per_idx.index(min(cld_per_idx))
        best_date = valid_dates[valid_dates.index == minIndex]
        best_date = best_date.iloc[0, 0]

        #Mostrar las mascaras de nubes para cada imagen por fecha valida
        n_cols = 4
        n_rows = int(np.ceil(len(clouds_data) / n_cols))
        fig = plt.figure(figsize=(n_cols * 4, n_rows * 3))
        for idx, cloud_mask in enumerate(clouds_data):
            ax = fig.add_subplot(n_rows, n_cols, idx + 1)
            Cloudless_tools.plot_cloud_mask(cloud_mask, fig=fig)
        plt.tight_layout()
        plt.savefig(clouds_folder + analysis_area + '/cloud_masks_valid.png')

        clear_pct = len(valid_dates) / len(cld_per_idx)
        number_cld_analysis = len(cld_per_idx)
        return best_date, valid_dates, clouds_data, clear_pct, number_cld_analysis