def test_cloud_detector(self): cloud_detector = S2PixelCloudDetector(all_bands=True) cloud_probs = cloud_detector.get_cloud_probability_maps( self.templates['s2_im']) cloud_mask = cloud_detector.get_cloud_masks(self.templates['s2_im']) self.assertTrue(np.array_equal(cloud_probs, self.templates['cl_probs'])) self.assertTrue(np.array_equal(cloud_mask, self.templates['cl_mask'])) cloud_detector = S2PixelCloudDetector(all_bands=False) self.assertRaises( ValueError, lambda: cloud_detector.get_cloud_probability_maps( self.templates['s2_im']))
def detectClouds(self, threshold): ''' Use s2cloudless to detect clouds Parameters ---------- threshold : float Minimum probability of cloud in order to mask a pixel ''' self.threshold = threshold # Load any bands that are not already there load_required = 0 for band in self.band_list: if (band not in self.layers): load_required = 1 if (load_required == 1): self.loadLayers(self.band_list) # Load numpy.ndarray for use with s2cloudless numpy_bands = np.empty((1, self.size_x, self.size_y, 10)) numpy_bands[0,:,:,0] = self.numpy_layers['B01'] numpy_bands[0,:,:,1] = self.numpy_layers['B02'] numpy_bands[0,:,:,2] = self.numpy_layers['B04'] numpy_bands[0,:,:,3] = self.numpy_layers['B05'] numpy_bands[0,:,:,4] = self.numpy_layers['B08'] numpy_bands[0,:,:,5] = self.numpy_layers['B8A'] numpy_bands[0,:,:,6] = self.numpy_layers['B09'] numpy_bands[0,:,:,7] = self.numpy_layers['B10'] numpy_bands[0,:,:,8] = self.numpy_layers['B11'] numpy_bands[0,:,:,9] = self.numpy_layers['B12'] cloud_detector = S2PixelCloudDetector(threshold=threshold, average_over=4, dilation_size=2) self.cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(numpy_bands)) self.cloud_masks = cloud_detector.get_cloud_masks(numpy_bands) # Save cloud mask to disk as black PNG that is transparent where there is no cloud, # and opaque where the probability of cloud > threshold save_location = os.path.join("masks", "cloud_masks") os.makedirs(save_location, exist_ok=True) img = Image.new('RGBA', (self.size_x, self.size_y), color = 'black') self.cloudCount = 0 for x in range(self.size_x): for y in range(self.size_y): if (self.cloud_masks[0][y][x] > 0): img.putpixel((x, y), (0,0,0,255)) # Opaque self.cloudCount += 1 else: img.putpixel((x, y), (0,0,0,0)) # Transparent output_file = os.path.join(save_location, "mask-x" + str(self.tile_x) + "-y" + str(self.tile_y) + "-" + self.dateStr + ".png") print("Writing Cloud Mask [" + output_file + "]") img.save(output_file) self.metadata.setCloudCount(self.cloudCount)
def create_mask(band_array, path): """Crete the mask.""" stacked = np.stack(band_array, -1) # shape (1, y_pixels, x_pixels, n_bands) # s2cloudless requires binary map arr4d = np.expand_dims(stacked / 10000, 0) cloud_detector = S2PixelCloudDetector(threshold=0.7, average_over=4, dilation_size=2) cloud_prob_map = cloud_detector.get_cloud_probability_maps(np.array(arr4d)) cloud_masks = cloud_detector.get_cloud_masks(np.array(arr4d)) template_file = [ os.path.join(path, item) for item in os.listdir(path) if item.endswith("01.jp2") ][0] # save files out_prob = os.path.dirname(path) + "_s2cloudless_prob.tif" out_mask = os.path.dirname(path) + "_s2cloudless_mask.tif" output = gdal_array.SaveArray(cloud_prob_map[0, :, :], out_prob, format="GTiff", prototype=template_file) output = None output = gdal_array.SaveArray(cloud_masks[0, :, :], out_mask, format="GTiff", prototype=template_file) output = None return (out_prob, out_mask)
def createCloudMask(band_array, path): stacked = np.stack(band_array, -1) # print(stacked / 10000) # swapped = np.moveaxis(stacked, 0, 2) # shape (y_pixels, x_pixels, n_bands) arr4d = np.expand_dims( stacked / 10000, 0 ) # shape (1, y_pixels, x_pixels, n_bands) # s2cloudless requires binary map cloud_detector = S2PixelCloudDetector(threshold=0.7, average_over=4, dilation_size=2) cloud_prob_map = cloud_detector.get_cloud_probability_maps(np.array(arr4d)) cloud_masks = cloud_detector.get_cloud_masks(np.array(arr4d)) # show files plt.figure(figsize=(50, 50)) plt.imshow(np.squeeze( cloud_masks)) # check whz process not stopping, when closing plt.show() # save files template_file = "/home/Usuario/Documents/sentinelHub/S2A_MSIL1C_20190904T140051_N0208_R067_T21LWH_20190904T172119.SAFE/GRANULE/L1C_T21LWH_A021943_20190904T140439/IMG_DATA/T21LWH_20190904T140051_B01.jp2" print(template_file) out_prob = path[:-31] + "out_prob.tif" out_mask = path[:-31] + "out_mask.tif" output = gdal_array.SaveArray(cloud_prob_map[0, :, :], out_prob, format="GTiff", prototype=template_file) output = None output = gdal_array.SaveArray(cloud_masks[0, :, :], out_mask, format="GTiff", prototype=template_file) output = None
def get_cloud_saturation_mask(self, ignore_detection=False): cloud_detector = S2PixelCloudDetector(**self.cloud_detection_config) true_color, bands, all_bands, dates = self.load_data() print("Downloaded data") print("Detecting clouds, this may take some time...") if True or not ignore_detection: cloud_masks_orig = cloud_detector.get_cloud_masks(np.array(bands)) # upscale cloud masks cloud_masks = [] for i in range(len(cloud_masks_orig)): cloud_masks.append( self.upscale_image(cloud_masks_orig[i], self.cloud_scale)) cloud_masks = np.array(cloud_masks) else: cloud_masks = np.array(bands) * 0 print("Cloud detection finished") # Images might be slightly out of scale, crop them x, y, _ = true_color[0].shape cloud_masks = cloud_masks[:, -x:, :y] off_image_detection_mask = sum(map(self.get_image_mask, true_color)) # Just sum how many times we detect white pixels full_cloud_mask = sum(cloud_masks) # Get total mask by dividing number of cloud detections by number of all sensible pixels total_mask_w = ( full_cloud_mask / (len(cloud_masks) - off_image_detection_mask)).astype(float) self.memo_data = CloudSaturation.MemoData(total_mask_w, true_color, bands, all_bands, dates, cloud_masks) return total_mask_w, np.array(true_color), np.array( all_bands), np.array(dates), np.array(cloud_masks)
def test_sentinelhub_cloud_detector(display=False): """ Runs the classification on the test scene and compares the results with the provided cloud probability map and probability mask. Used to verify the installation of the package. :param display: flag to turn on or off the display of results. :type display: bool, default is False """ # Load arrays package_dir = os.path.dirname(__file__) templates_filename = os.path.join(package_dir, './TestInputs/input_arrays.npz') templates = np.load(templates_filename) # Classifier instance, image has all bands cloud_detector = S2PixelCloudDetector(all_bands=True) # Compute cloud probabilities cloud_probs = cloud_detector.get_cloud_probability_maps(templates['s2_im']) # Compute cloud mask cloud_mask = cloud_detector.get_cloud_masks(templates['s2_im']) # Check predicted results match templates probs_ok = np.isclose(cloud_probs, templates['cl_probs']).all() mask_ok = np.array_equal(cloud_mask, templates['cl_mask']) if not probs_ok: LOGGER.info( 'Test FAILED.\nCloud probabilities DO NOT match templates.') if not mask_ok: LOGGER.info('Test FAILED.\nCloud masks DO NOT match templates.') if mask_ok and probs_ok: LOGGER.info( 'Test OK.\nCloud probabilities and cloud masks match templates.') # Display results if display: fig = plt.figure(figsize=(20, 10)) plt.subplot(131) plt.imshow(np.squeeze(templates['s2_im'])[:, :, [3, 2, 1]]) plt.title('RGB image') plt.axis('off') plt.subplot(132) plt.imshow(np.squeeze(templates['s2_im'])[:, :, [3, 2, 1]]) plt.imshow(np.squeeze(templates['cl_mask']), vmin=0, vmax=1, alpha=.4) plt.title('RGB image with template mask overlay') plt.axis('off') plt.subplot(133) plt.imshow(np.squeeze(templates['s2_im'])[:, :, [3, 2, 1]]) plt.imshow(np.squeeze(cloud_mask), vmin=0, vmax=1, alpha=.4) plt.title('RGB image with predicted mask overlay') plt.axis('off') plt.show()
def get_s2_pixel_cloud_detector(threshold=0.4, average_over=4, dilation_size=2, all_bands=True): """ Wrapper function for pixel-based S2 cloud detector `S2PixelCloudDetector` """ return S2PixelCloudDetector(threshold=threshold, average_over=average_over, dilation_size=dilation_size, all_bands=all_bands)
def test_no_data_available_request(config): """ Tests an exception raised by CloudMaskRequest """ cloud_detector = S2PixelCloudDetector() with pytest.raises(NoDataAvailableException): CloudMaskRequest(cloud_detector, bbox=BBOX1, time=('2021-01-01', '2021-01-10'), size=(250, 250), maxcc=0.01, config=config)
def cloud_mask(fpath): ''' internal function. creates the cloud mask using s2cloudless package ''' fname = fpath.split('/')[-1] aband_list = [ 'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12', 'BQA', 'BQA', 'BQA', 'BQA' ] cband_list = [ 'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12' ] ds = gdal.Open(fpath, 0) ctr = 0 for cband in cband_list: cind = aband_list.index(cband) cdata = ds.GetRasterBand(cind + 1).ReadAsArray() if ctr == 0: ndata = np.zeros( (cdata.shape[0], cdata.shape[1], len(cband_list))) ndata[:, :, ctr] = cdata ctr = ctr + 1 ndata = ndata.astype(float) ndata = ndata * 1.0 / 10000 ndata = np.expand_dims(ndata, 0) # print(time.time()-start_time) cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2) cloud_matrix = cloud_detector.get_cloud_probability_maps( np.array(ndata)) cloud_matrix = np.squeeze(cloud_matrix) cloud_matrix = np.round(cloud_matrix * 100).astype(np.uint16) ctr = 0 for cband in aband_list: cind = aband_list.index(cband) cdata = ds.GetRasterBand(cind + 1).ReadAsArray() if ctr == 0: odata = np.zeros( (cdata.shape[0], cdata.shape[1], len(aband_list) + 1)) odata[:, :, ctr] = cdata ctr = ctr + 1 odata[:, :, -1] = cloud_matrix odata = odata.astype(np.uint16) CreateGeoTiffs(odata, fpath, fpath, gdal.GDT_UInt16)
def test_pixel_cloud_detector(): data_path = os.path.join(os.path.dirname(__file__), 'TestInputs', 'input_arrays.npz') data = np.load(data_path) cloud_detector = S2PixelCloudDetector(all_bands=True) cloud_probs = cloud_detector.get_cloud_probability_maps(data['s2_im']) cloud_mask = cloud_detector.get_cloud_masks(data['s2_im']) assert np.allclose(cloud_probs, data['cl_probs'], atol=1e-14) assert np.array_equal(cloud_mask, data['cl_mask']) single_temporal_cloud_probs = cloud_detector.get_cloud_probability_maps( data['s2_im'][0, ...]) single_temporal_cloud_mask = cloud_detector.get_cloud_masks( data['s2_im'][0, ...]) assert np.array_equal(single_temporal_cloud_probs, cloud_probs[0, ...]) assert np.array_equal(single_temporal_cloud_mask, cloud_mask[0, ...]) cloud_detector = S2PixelCloudDetector(all_bands=False) with pytest.raises(ValueError): cloud_detector.get_cloud_probability_maps(data['s2_im'])
def compute_cloud_mask(x: np.ndarray, threshold: float = 0.4, average_over: int = 4, dilation_size: int = 2, all_bands: bool = True): z = sentinel2_to_cloud_mask_preprocess(x) cloud_detector = S2PixelCloudDetector(threshold=threshold, average_over=average_over, dilation_size=dilation_size, all_bands=all_bands) cloud_mask = cloud_detector.get_cloud_probability_maps(z) cloud_mask = cloud_mask.squeeze() return cloud_mask
def __init__(self, clp_feature: Tuple = (FeatureType.DATA, 'CLP'), clm_feature: Tuple = (FeatureType.MASK, 'CLM'), average_over: int = 24, max_clp: float = 255.): """ :param clp_feature: Feature type and name of input CLP mask :param clm_feature: Feature type and name of output CLM mask :param average_over: Parameter used ot smooth the CLP data map :param max_clp: Maximum value of CLP map used for normalization """ self.clm_feature = next(self._parse_features(clm_feature)()) self.clp_feature = next(self._parse_features(clp_feature)()) self.s2_cd = S2PixelCloudDetector(average_over=average_over) self.max_clp = max_clp
def calculate_cloud_mask(self): bands_array = self._calculate_bands_array(self) cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2, all_bands=False) # cloud_prob = cloud_detector.get_cloud_probability_maps(bands_array) cloud_mask = cloud_detector.get_cloud_masks(bands_array) resized_mask = scipy.ndimage.zoom(cloud_mask[0], 6, mode='nearest') temp = np.multiply(resized_mask, -1) resized_mask_invertida = np.add(temp, 1) return resized_mask_invertida
def compute_cloud_mask(self, threshold=0.4, average_over=4, dilation_size=2): cloud_detector = S2PixelCloudDetector(threshold=threshold, average_over=average_over, dilation_size=dilation_size, all_bands=False) bands_for_cloud_detector = [ "B1", "B2", "B4", "B5", "B8", "B8A", "B9", "B10", "B11", "B12" ] cloud_detector_input = np.zeros((self.rgb.shape[0], self.rgb.shape[1], len(bands_for_cloud_detector))) for i, band in enumerate(bands_for_cloud_detector): cloud_detector_input[:, :, i] = self.bands_rhot[band] return cloud_detector.get_cloud_masks(cloud_detector_input)
def detect_clouds(merged_tif_path, save_file): print('================================') print('Cloud detection.') cloud_detector = S2PixelCloudDetector() img = rasterio.open(merged_tif_path).read() img = reshape_as_image(img) img = np.expand_dims(img, axis=0) / 10000 print('predict.') cloud_probs = cloud_detector.get_cloud_probability_maps(img) width = int(img.shape[2] * 4) height = int(img.shape[1] * 4) print('resize.') cloud_probs = cloud_probs.reshape((img.shape[1], img.shape[2])) cloud_probs = cv2.resize(cloud_probs, (width, height), interpolation=cv2.INTER_CUBIC) print('save cloud.') imageio.imwrite(save_file, cloud_probs)
def load_eo_data(self): all_bands_request, cloud_bands_request = self.create_requests() all_bands = all_bands_request.get_data( save_data=True, redownload=self.settings.redownload) print("downloaded all bands") cloud_bands = cloud_bands_request.get_data( save_data=True, redownload=self.settings.redownload) print("downloaded cloud bands") print(cloud_bands[0].shape) dates1 = all_bands_request.get_dates() dates = [j.date() for j in dates1] out = [] for j in range(1, len(dates)): if dates[j] == dates[j - 1]: out.append(j) all_bands = np.delete(all_bands, out, 0) cloud_detector = S2PixelCloudDetector( average_over=self.settings.cloud_detection_settings.average_over, dilation_size=self.settings.cloud_detection_settings.dilation_size, threshold=self.settings.cloud_detection_settings.threshold, all_bands=True) cloud_masks_orig = cloud_detector.get_cloud_masks(np.array(all_bands)) # upscale cloud masks cloud_masks = [] for i in range(len(cloud_masks_orig)): cloud_masks.append( self.upscale_image( cloud_masks_orig[i], self.settings.cloud_detection_settings.x_scale)) cloud_masks = np.array(cloud_masks) for j in out: dates1.pop(j) return all_bands, cloud_masks, dates1
def get_cloud_saturation_mask(self): """ Main acquisition method Acquires required data, generates cloud detections masks and saves them in memo_data for further use :return: Tuple[cloud_saturation_mask, true_color_bands, all_bands, data_acquisiton_dates, cloud_masks] """ cloud_detector = S2PixelCloudDetector(**self.cloud_detection_config) true_color, bands, all_bands, dates = self.load_data() print("Downloaded") cloud_masks_orig = cloud_detector.get_cloud_masks(np.array(bands)) # upscale cloud masks cloud_masks = [] print(cloud_masks_orig[0].shape) for i in range(len(cloud_masks_orig)): cloud_masks.append( self.upscale_image(cloud_masks_orig[i], self.cloud_scale)) cloud_masks = np.array(cloud_masks) print("Detected") # Images might be slightly out of scale, crop them x, y, _ = true_color[0].shape cloud_masks = cloud_masks[:, -x:, :y] off_image_detection_mask = sum(map(self.get_image_mask, true_color)) # Just sum how many times we detect white pixels full_cloud_mask = sum(cloud_masks) # Get total mask by dividing number of cloud detections by number of all sensible pixels total_mask_w = ( full_cloud_mask / (len(cloud_masks) - off_image_detection_mask)).astype(float) self.memo_data = CloudSaturation.MemoData(total_mask_w, true_color, bands, all_bands, dates, cloud_masks) # TODO: Refactor me to some nicer object return (total_mask_w, np.array(true_color), np.array(all_bands), np.array(dates), np.array(cloud_masks))
#Calculo de probabilidades y obtención de mascaras de nubes bands_script = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]' wms_bands_request = WmsRequest(layer=LAYER_NAME, custom_url_params={ CustomUrlParam.EVALSCRIPT: bands_script, CustomUrlParam.ATMFILTER: 'NONE' }, bbox=bounding_box, time=(Date_Ini, Date_Fin), width=x_width, height=y_height, image_format=MimeType.TIFF_d32f, instance_id=INSTANCE_ID) wms_bands = wms_bands_request.get_data() cloud_detector = S2PixelCloudDetector( threshold=0.35, average_over=8, dilation_size=3) #change threshold to test cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands)) cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands)) all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request, threshold=0.1) #folder de imagenes nubes Path('output_clouds/' + analysis_area).mkdir(parents=True, exist_ok=True) if not os.path.exists(analysis_area): os.makedirs(analysis_area) #Mostrar las probabilidades de nubes para cada imagen por fecha en el rango de analisis fig = plt.figure(figsize=(15, 10)) n_cols = 4 n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols)) for idx, [prob, mask, data] in enumerate(all_cloud_masks):
def apply_cloudmask(S2DotSafefp=None, bands=None, threshold=0.4, dilation_size=3, average_over=1, resolution_safe=100, save_mask=None, crs=None, affine=None): """ Function for applying cloud mask to either S2 product in safe format or a multiple rasters in numpy ndarray format. ndarray format is reccomended :param S2DotSafefp: path to S2 file :param bands: ndarray (N, H, W, B) N is the number of rasters, H is height, W is width, B are bands :param resolution_safe: Pixel resolution for cloud mask for .safe format in meters, recommended to use higher than 60 as it's much more efficient. :param dilation_size: dilation for cloud pixels :param threshold: threshold for the cloud probability :param average_over: neighborhood of averaging :param save_mask: :param crs: If save_mask => crs for the saved mask :param affine: Affine transformation for the saved mask :return: cloud mask (N, H, W) cloud mask for each raster, if .safe format the output shape will be (1, H, W) """ # If the input is in safe format if S2DotSafefp is not None: # resolutions for each band res = np.array([60, 10, 10, 20, 10, 20, 60, 60, 20, 20]) # Valid bands bands_ = [ 'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12' ] # list of rasters S2rasters = [ glob( os.path.join(S2DotSafefp, 'GRANULE', 'L*', 'IMG_DATA', '*{}*'.format(band)))[0] for band in bands_ ] # Affine transformation list afftr = res / resolution_safe S2_cloud_bands = [] # iterate through band rasters for i, raster in enumerate(S2rasters): with rasterio.open(raster) as src: # Divide by 10000 to get TOA values arr = src.read().astype('float32') / 10000 aff = src.transform # Define new affine transform new_aff = Affine(aff.a / afftr[i], aff.b, aff.c, aff.d, aff.e / afftr[i], aff.f) # Empty array for destination newarr = np.empty(shape=(arr.shape[0], int(round(arr.shape[1] * afftr[i])), int(round(arr.shape[2] * afftr[i]))), dtype='float32') # Apply downsampling warp.reproject(arr, newarr, src_transform=aff, dst_transform=new_aff, src_crs=src.crs, dst_crs=src.crs) # Append to list S2_cloud_bands.append(newarr) # Concatenate the list and add a dim for compatibility with S2Cloudless function S2_cloud_bands = np.concatenate(S2_cloud_bands, axis=0) S2_cloud_bands = np.expand_dims(S2_cloud_bands, axis=0) # Apply cloudmask cloudmask = S2PixelCloudDetector( threshold=threshold, dilation_size=dilation_size, average_over=average_over, all_bands=False).get_cloud_masks(S2_cloud_bands) # save mask if path is specified if save_mask is not None: with rasterio.open(S2rasters[0]) as src: kwargs = src.profile kwargs['width'] = cloudmask.shape[1] kwargs['height'] = cloudmask.shape[2] kwargs['transform'] = new_aff kwargs['driver'] = 'Gtiff' kwargs['dtype'] = cloudmask.dtype kwargs['blockxsize'] = 128 kwargs['blockysize'] = 128 with rasterio.open(save_mask, 'w', **kwargs) as dst: dst.write(cloudmask) # return the cloudmask as np ndarray return cloudmask # If valid bands are given elif bands is not None: cloudmask = S2PixelCloudDetector( threshold=threshold, dilation_size=dilation_size, average_over=average_over, all_bands=False).get_cloud_masks(bands) # save mask if path is specified if save_mask is not None and crs is not None: kwargs = { 'driver': 'GTiff', 'interleave': 'band', 'tiled': True, 'blockxsize': 128, 'blockysize': 128, 'nodata': 0, 'dtype': cloudmask.dtype, 'height': cloudmask.shape[1], 'width': cloudmask.shape[2], 'crs': crs, 'transform': affine } with rasterio.open(save_mask, 'w', **kwargs) as dst: dst.write(cloudmask) elif save_mask is not None and crs is None: raise ValueError("To save the mask, crs needs to be provided") # return the cloudmask as np ndarray return cloudmask else: raise ValueError( "You need to provide a path to a Sentinel-2 .safe file, or a numpy ndarray consisting of the bands" )
wms_bands_request = WmsRequest(layer=LAYER_NAME, custom_url_params={ CustomUrlParam.EVALSCRIPT: bands_script, CustomUrlParam.ATMFILTER: 'NONE' }, bbox=bounding_box, time=('2017-12-01', '2017-12-31'), width=600, height=None, image_format=MimeType.TIFF_d32f, instance_id=INSTANCE_ID) wms_bands = wms_bands_request.get_data() cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2) cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands)) cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands)) image_idx = 0 overlay_cloud_mask(wms_true_color_imgs[image_idx], cloud_masks[image_idx]) plot_probability_map(wms_true_color_imgs[image_idx], cloud_probs[image_idx]) plot_cloud_mask(cloud_masks[image_idx]) all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request, threshold=0.1) fig = plt.figure(figsize=(15, 10)) n_cols = 4 n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols)) for idx, [prob, mask, data] in enumerate(all_cloud_masks):
width_16 = width // 16 height_16 = height // 16 small_data = np.zeros((13, width_16, height_16), dtype=np.uint16) rasterio.warp.reproject(data[0:13], small_data, src_transform=geoTransform, src_crs=crs, dst_transform=geoTransform * rasterio.transform.Affine.scale(16, 16), dst_crs=crs, resampling=rasterio.enums.Resampling.nearest) small_data = np.transpose(small_data, axes=(1, 2, 0)) small_data = small_data.reshape(1, width_16, height_16, 13) / 1e5 cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=1, all_bands=True) small_tmp = cloud_detector.get_cloud_probability_maps(small_data) try: quantile = np.quantile( np.extract(small_tmp > np.min(small_tmp), small_tmp), 0.20) except: quantile = 0.0033 cutoff = max(0.0033, quantile) small_tmp = (small_tmp > cutoff).astype(np.uint16) tmp = np.zeros((1, width, height), dtype=np.uint16) rasterio.warp.reproject(small_tmp, tmp, src_transform=geoTransform * rasterio.transform.Affine.scale(16, 16),
def cloud_detection(self, input_file): print("cloud_detection", input_file) input_dir = os.path.join(input_file, "GRANULE") sub_directories = utils.get_immediate_subdirectories(input_dir) image_dir = os.path.join(input_dir, sub_directories[0], "IMG_DATA") input_bands = [ 'B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12' ] # Band order is strict # num_bands = len(input_bands) scale_factor = 10000.0 #Read from metadata ? band_paths = self.get_band_paths(input_file, input_bands) for band_ind, img_filename in enumerate(band_paths): with rasterio.open(img_filename) as ds: img = ds.read() if band_ind == 0: # First band need to be 60m tmparr = np.empty_like(img) aff60 = ds.transform img_stack = np.zeros( (img.shape[0], img.shape[1], img.shape[2], len(input_bands))) img_stack[:, :, :, band_ind] = img / scale_factor elif input_bands[band_ind].upper( ) == "B09" or input_bands[band_ind].upper() == "B10": # 60m img_stack[:, :, :, band_ind] = img / scale_factor else: reproject(img, tmparr, src_transform=ds.transform, dst_transform=aff60, src_crs=ds.crs, dst_crs=ds.crs, resampling=Resampling.bilinear) img_stack[:, :, :, band_ind] = tmparr / scale_factor if input_bands[band_ind].upper() == "B02": # 10m aff10 = ds.transform nrows10 = img.shape[1] ncols10 = img.shape[2] ds10 = ds cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2) cloud_probs = cloud_detector.get_cloud_probability_maps(img_stack) cloud_mask = cloud_detector.get_cloud_masks(img_stack).astype( rasterio.uint8) cloud_probs_10 = np.zeros((1, nrows10, ncols10)) reproject(cloud_probs, cloud_probs_10, src_transform=aff60, dst_transform=aff10, src_crs=ds.crs, dst_crs=ds.crs, resampling=Resampling.cubic_spline) cloud_mask_10 = np.zeros((1, nrows10, ncols10)) reproject(cloud_mask, cloud_mask_10, src_transform=aff60, dst_transform=aff10, src_crs=ds.crs, dst_crs=ds.crs, resampling=Resampling.nearest) return (cloud_probs_10, cloud_mask_10, ds10)
def cloud_masking(gdir): """ Masks cloudy pixels with s2cloudless algorithm - Reads necessary bands from ekstrand corrected netCDF file - Applies s2cloudless algorithm to scene to get cloud mask (https://github.com/sentinel-hub/sentinel2-cloud-detector) - Applies cloud mask to scene - saves processed scene with all bands in cloud_masked.nc file Parameters: ----------- gdir: :py:class:`crampon.GlacierDirectory` A GlacierDirectory instance. Returns: -------- None """ cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=2, dilation_size=2) try: sentinel = xr.open_dataset(gdir.get_filepath('ekstrand')) except FileNotFoundError: # Check if uncorrected sentinel file exists: try: sentinel = xr.open_dataset( gdir.get_filepath('sentinel')) #, chunks={'time':10}) except FileNotFoundError: # No data exists at all # 1st time step: no data for this glacier available log.info('No data for this glacier - Abort Cloud Masking') return try: wms_bands = sentinel.sel( band=['B01', 'B02', 'B04', 'B05', 'B08', 'B8A', 'B09', 'B10', 'B11', 'B12'], time=cfg.PARAMS['date'][0])\ .img_values.values except KeyError: # Check if uncorrected Sentinel File exists/has data: try: sentinel = xr.open_dataset( gdir.get_filepath('sentinel')) #,chunks={'time':10} ) wms_bands = sentinel.sel( band=['B01', 'B02', 'B04','B05','B08','B8A','B09', 'B10', 'B11', 'B12'], time=cfg.PARAMS['date'][0])\ .img_values.values except KeyError: # No data for this glacier & date log.info('No data for this glacier and date - Abort Cloud Masking') return # rearrange dimensions, goal :[height, width, channels]. # now: [channels, height, width] and correct into float (divide by factor 10000) wms_bands = [np.transpose(wms_bands / 10000, (1, 2, 0)) for _ in range(1)] cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands)) # cloud_probability = cloud_detector.get_cloud_probability_maps(np.array(wms_bands)) # plot_cloud_mask(cloud_probability ,wms_bands) ##### Deal with MemoryError for large files: remove last entry #sentinel=sentinel.isel(time=range(0,sentinel.time.values.size-1)) for band_id in sentinel['band'].values: band_array = sentinel.sel(band=[band_id], time=cfg.PARAMS['date'][0]).img_values.values # Set threshold to exclude glaciers with more than 60% cloud cover # -> no useful classification possible image = sentinel.sel(band=[band_id], time=cfg.PARAMS['date'][0]).img_values.values band_array[cloud_masks == 1] = 0 try: cloud_cover = 1 - len(band_array[band_array > 0]) / len( image[image > 0]) except ZeroDivisionError: # for 100 % cloud cover: log.info: ('Cloud Cover is 100%- abort Cloud Masking') return if cloud_cover > 0.7: # -> set all pixels to 0 log.info('Cloud COver over 70% - abort Cloud Masking') return band_array = band_array[0, :, :] if os.path.isfile(gdir.get_filepath('cloud_masked')): cloud_mask = xr.open_dataset(gdir.get_filepath('cloud_masked')) cloud_mask_new = cloud_mask.copy(deep=True) cloud_mask_new = cloud_mask_new.isel(time=0) cloud_mask_new.coords['time'] = np.array([cfg.PARAMS['date'][0]]) cloud_mask = xr.concat([cloud_mask, cloud_mask_new], dim='time') os.remove(gdir.get_filepath('cloud_masked')) else: if not 'cloud_mask' in locals(): # File does not exist: Create new as copy from sentinel.nc file: sentinel_copy = sentinel.copy(deep=True) # Remove all dates except current: sentinel_copy = sentinel_copy.drop( [time_id for time_id in sentinel['time'].values][:-1], dim='time').squeeze('time', drop=True) sentinel_copy = sentinel_copy.assign_coords( time=cfg.PARAMS['date'][0]) sentinel_copy = sentinel_copy.expand_dims('time') cloud_mask = sentinel_copy.copy(deep=True) cloud_mask['img_values'].loc[( dict(band = band_id, time=cfg.PARAMS['date'][0]))] \ = np.ones([band_array.shape[0], band_array.shape[1]]) cloud_mask['img_values'].loc[( dict(band=band_id, time=cfg.PARAMS['date'][0]))] \ = band_array print("Cloud cover: ", cloud_cover) # Write Updated DataSet to file cloud_mask.to_netcdf(gdir.get_filepath('cloud_masked'), 'w') cloud_mask.close()
def convert_sentinel2(path_to_SAFE_folder, output_resolution=10, cloud_detection_resolution=60, n_threads=10, delete_safe_file=True): warnings.filterwarnings("ignore", category=DeprecationWarning) if path_to_SAFE_folder[-1] == '/': path_to_SAFE_folder = path_to_SAFE_folder[:-1] tile_name = path_to_SAFE_folder.split('/')[-1].replace('.SAFE', '') print('Processing:', tile_name) #Explore content in folder to find image-folder: input_dir = os.path.join(path_to_SAFE_folder, "GRANULE") sub_directories = [ name for name in os.listdir(input_dir) if os.path.isdir(os.path.join(input_dir, name)) ] image_dir = os.path.join(input_dir, sub_directories[0], "IMG_DATA") #Read all band scale_factor = 10000 band_data = {} t = time.time() time_left = 0 for img_dir, dirnames, filenames in os.walk(image_dir): for band_ind, band_name in enumerate(_all_bands): if band_name in _bands_10m: resolution = 10 elif band_name in _bands_20m: resolution = 20 elif band_name in _bands_60m: resolution = 60 for filename in filenames: #L2a products have multiple resolutions for each band. l2a_prod_res = filename.split('.jp2')[0].split('_')[-1] if l2a_prod_res in ['10m', '20m', '60m']: l2a_prod_res = int(l2a_prod_res.split('m')[0]) if l2a_prod_res > output_resolution and l2a_prod_res > resolution: continue if fnmatch.filter([filename], "*" + band_name.upper() + "*.jp2"): path_to_jp2 = os.path.join(img_dir, filename) # Assuming that os.walk loop through folders alphabetically, this ensure that the higest resolution is chosen if there exist multiple resolutions pr file if band_name not in band_data: print(' - Reading', band_name, '-', len(band_data) + 1, 'of', len(_all_bands), '-', 'Estimated time left:', str(datetime.timedelta(seconds=time_left))) with rasterio.open(path_to_jp2) as ds: band_data[band_name] = { 'data': ds.read() / scale_factor, 'transform': ds.transform, 'crs': ds.crs, 'res': resolution } time_left = (time.time() - t) / (len(band_data)) * ( len(_all_bands) - len(band_data)) #Collect sentral parameters for each resolution transforms = { 10: band_data[_bands_10m[0]]['transform'], 20: band_data[_bands_20m[0]]['transform'], 60: band_data[_bands_60m[0]]['transform'], } crss = { 10: band_data[_bands_10m[0]]['crs'], 20: band_data[_bands_20m[0]]['crs'], 60: band_data[_bands_60m[0]]['crs'], } shapes = { 10: band_data[_bands_10m[0]]['data'].shape, 20: band_data[_bands_20m[0]]['data'].shape, 60: band_data[_bands_60m[0]]['data'].shape, } target_transform = transforms[output_resolution] target_crs = crss[output_resolution] #Reprojections def reproject_to_resolution(data, res_from, res_to): if res_to == res_from: return data tmp_array = np.zeros(shapes[res_to]) reproject(data, tmp_array, src_transform=transforms[res_from], dst_transform=transforms[res_to], src_crs=crss[res_from], dst_crs=crss[res_to], num_threads=n_threads, resampling=Resampling.bilinear) return tmp_array #################################################################################################################### # Perform cloud detection using # https://github.com/sentinel-hub/sentinel2-cloud-detector/blob/master/examples/sentinel2-cloud-detector-example.ipynb #Convert all bands to 60m for cloud-detection print(" - Cloud detection running") bands_for_cloud_detection = [ reproject_to_resolution(band_data[band_name]['data'], band_data[band_name]['res'], cloud_detection_resolution) for band_name in _bands_used_in_cloud_detection ] bands_for_cloud_detection = np.concatenate( [np.expand_dims(np.squeeze(b), -1) for b in bands_for_cloud_detection], -1) bands_for_cloud_detection = np.expand_dims(bands_for_cloud_detection, 0) cloud_detector = S2PixelCloudDetector(threshold=0.4, all_bands=False, average_over=4, dilation_size=2) cloud_mask = cloud_detector.get_cloud_masks(bands_for_cloud_detection)[0] #Save to output resolution cloud_mask = reproject_to_resolution(cloud_mask, cloud_detection_resolution, output_resolution) print(" - Cloud detection finished") # Save memory maps print(" - Creating 10m resolution data-cube") bands = [ "b02", "b03", "b04", "b08", "b05", "b06", "b07", "b8a", "b11", "b12", "b01", "b09", "b10" ] bands_data = [] for band_name in bands: data_at_res = reproject_to_resolution(band_data[band_name]['data'], band_data[band_name]['res'], output_resolution) data_at_res[data_at_res == 0] = np.nan #Insert np.nan as nodata value bands_data.append(data_at_res.squeeze().astype('float16')) bands_data = [d[:, :, None] for d in bands_data] bands_data = np.concatenate(bands_data, -1) if delete_safe_file: print(' - Deleting SAFE file', path_to_SAFE_folder) shutil.rmtree(path_to_SAFE_folder) warnings.filterwarnings("default", category=DeprecationWarning) return bands_data, cloud_mask, target_transform, target_crs
def mask_s2(data, solar_za, solar_az, cloud_heights=None, nodata=None, scale_factor=1, num_workers=1, **kwargs): """ Masks Sentinel 2 data Args: data (DataArray): The Sentinel 2 data to mask. solar_za (DataArray): The solar zenith angle. solar_az (DataArray): The solar azimuth angle. cloud_heights (Optional[list]): A list of potential cloud heights. nodata (Optional[int or float]): A 'no data' value to fill NAs with. scale_factor (Optional[float]): A scale factor to apply to the data. num_workers (Optional[int]): The number of parallel compute workers. kwargs (Optional[dict]): Keyword arguments for ``s2cloudless.S2PixelCloudDetector``. Returns: ``xarray.DataArray``: Data range: 0 to 4, where 0=clear; 2=shadow; 4=cloud Example: >>> import geowombat as gw >>> from geowombat.radiometry import mask_s2 >>> >>> with gw.config.update(sensor='s2f', scale_factor=0.0001): >>> >>> with gw.open('image.tif') as src, \ >>> gw.open('solar_zenith.tif') as sza, \ >>> gw.open('solar_azimuth.tif') as saa: >>> >>> s2_mask = mask_s2(src, sza, saa) """ # from ..radiometry.mask import CloudShadowMasker # mask_s2 = CloudShadowMasker().mask_s2 # # mask = mask_s2(data, # sza, # saa, # scale_factor=0.0001, # nodata=0, # num_workers=num_threads) # # fnmask = Path(load_bands_names[0]).name.split('.')[0] # mask.gw.to_raster(f'/media/jcgr/data/projects/global_fields/data/grids/ms/test/000960/{fnmask}_mask.tif', # n_workers=1, n_threads=1) # # if bands_out: # data = _assign_attrs(data, attrs, bands_out) new_attrs = data.attrs.copy() if not cloud_heights: cloud_heights = list(range(500, 2000, 500)) if not isinstance(nodata, int) and not isinstance(nodata, float): nodata = data.gw.nodata if scale_factor == 1.0: scale_factor = data.gw.scale_factor if S2CLOUDLESS_INSTALLED: if not kwargs: kwargs = dict(threshold=0.4, average_over=4, dilation_size=5, all_bands=False) cloud_detector = S2PixelCloudDetector(**kwargs) # Get the S2Cloudless bands data_cloudless = data.sel(band=[ 'coastal', 'blue', 'red', 'nir1', 'nir', 'rededge', 'water', 'cirrus', 'swir1', 'swir2' ]) # Scale from 0-10000 to 0-1 if isinstance(nodata, int) or isinstance(nodata, float): data_cloudless = xr.where(data_cloudless != nodata, data_cloudless * scale_factor, nodata).clip(0, 1).astype('float64') else: data_cloudless = (data_cloudless * scale_factor).clip( 0, 1).astype('float64') # Reshape for predictions .. # from [bands x rows x columns] # to [images x rows x columns x bands] X = data_cloudless.transpose('y', 'x', 'band').data.compute( num_workers=num_workers)[np.newaxis, :, :, :] ################ # Predict clouds ################ # Convert from NumPy array to DataArray # clear=0, clouds=1 cloud_mask = ndarray_to_xarray(data, cloud_detector.get_cloud_masks(X), [1]) ################# # Predict shadows ################# # Scale the angles to degrees sza = solar_za * 0.01 sza.coords['band'] = [1] saa = solar_az * 0.01 saa.coords['band'] = [1] # Convert to radians rad_sza = xr.ufuncs.deg2rad(sza) rad_saa = xr.ufuncs.deg2rad(saa) # non-shadow=0, shadows=1 shadow_mask = estimate_shadows(data, cloud_mask, rad_sza, rad_saa, cloud_heights, nodata, scale_factor, num_workers) # Recode for final output mask = xr.where( cloud_mask.sel(band=1) == 1, 4, xr.where( shadow_mask.sel(band=1) == 1, 2, xr.where(data.max(dim='band') == nodata, 255, 0))).expand_dims(dim='band').astype('uint8') mask = mask.assign_coords(coords={'band': ['mask']}) new_attrs['nodatavals'] = (255) new_attrs['scales'] = (1.0) new_attrs['offsets'] = (0.0) new_attrs['pre-scaling'] = scale_factor new_attrs['sensor'] = 's2' new_attrs['clearval'] = (0) new_attrs['shadowval'] = (2) new_attrs['cloudval'] = (4) new_attrs['fillval'] = (255) mask = mask.assign_attrs(**new_attrs) else: logger.warning(' S2Cloudless is not installed.') mask = None return mask
def cloud_detection(input_path, cloudmask, cloudprobs): for file in sorted(os.listdir(input_path)): if file.endswith("B01.jp2"): ds = gdal.Open(os.path.join(input_path, file), gdal.GA_ReadOnly) with rasterio.open(os.path.join(input_path, file)) as scl: B01 = scl.read() tmparr = np.empty_like(B01) aff = scl.transform print(B01.shape) if file.endswith("B02.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B02 = scl.read() reproject( B02, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B02 = tmparr print(B02.shape) if file.endswith("B04.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B04 = scl.read() reproject( B04, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B04 = tmparr print(B04.shape) if file.endswith("B05.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B05 = scl.read() reproject( B05, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B05 = tmparr print(B05.shape) if file.endswith("B08.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B08 = scl.read() reproject( B08, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B08 = tmparr print(B08.shape) if file.endswith("B8A.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B8A = scl.read() reproject( B8A, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B8A = tmparr print(B8A.shape) if file.endswith("B09.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B09 = scl.read() if file.endswith("B10.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B10 = scl.read() if file.endswith("B11.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B11 = scl.read() reproject( B11, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B11 = tmparr print(B11.shape) if file.endswith("B12.jp2"): with rasterio.open(os.path.join(input_path, file)) as scl: B12 = scl.read() reproject( B12, tmparr, src_transform=scl.transform, dst_transform=aff, src_crs=scl.crs, dst_crs=scl.crs, resampling=Resampling.bilinear) B12 = tmparr print(B12.shape) print(B12.shape) bands = np.array([np.dstack((B01[0] / 10000.0, B02[0] / 10000.0, B04[0] / 10000.0, B05[0] / 10000.0, B08[0] / 10000.0, B8A[0] / 10000.0, B09[0] / 10000.0, B10[0] / 10000.0, B11[0] / 10000.0, B12[0] / 10000.0))]) print(bands.shape) cloud_detector = S2PixelCloudDetector(threshold=0.4, average_over=4, dilation_size=2) cloud_probs = cloud_detector.get_cloud_probability_maps(bands) mask = cloud_detector.get_cloud_masks(bands)#.astype(rasterio.uint8) # write the cloud mask and cloud probability map driver = gdal.GetDriverByName("GTiff") outdata = driver.Create(cloudmask, ds.RasterXSize, ds.RasterYSize, 1, gdal.GDT_UInt16) outdata.SetGeoTransform(ds.GetGeoTransform()) outdata.SetProjection(ds.GetProjection())##sets same projection as input outband = outdata.GetRasterBand(1) outband.WriteArray(mask[0]) outband.FlushCache() ##saves to disk!! del outband del outdata trim_processed(shapefile_link, cloudmask, cloudmask_clipped) os.remove(cloudmask) outdata = driver.Create(cloudprobs, ds.RasterXSize, ds.RasterYSize, 1, gdal.GDT_Float32) outdata.SetGeoTransform(ds.GetGeoTransform()) outdata.SetProjection(ds.GetProjection())##sets same projection as input outband = outdata.GetRasterBand(1) outband.WriteArray(cloud_probs[0]) outband.FlushCache() ##saves to disk!! del outband del outdata trim_processed(shapefile_link, cloudprobs, cloudprobs_clipped) os.remove(cloudprobs)
@pytest.fixture(name='config') def config_fixture(): config = SHConfig() for param in config.get_params(): env_variable = param.upper() if os.environ.get(env_variable): setattr(config, param, os.environ.get(env_variable)) return config @pytest.mark.parametrize('input_params,stats', [ (dict(cloud_detector=S2PixelCloudDetector( threshold=0.6, average_over=2, dilation_size=5, all_bands=True), bbox=BBOX1, time=('2017-12-01', '2017-12-31'), size=(60, 81)), dict(clm_min=0, clm_max=1, clm_mean=0.343827, clm_median=0, clp_min=0.00011, clp_max=0.99999, clp_mean=0.23959, clp_median=0.01897, mask_shape=(7, 81, 60))), (dict(cloud_detector=S2PixelCloudDetector(), bbox=BBOX2, time=[dt.datetime(2016, 7, 18, 7, 14, 4)],
def generate_cloud_masks(array: np.ndarray) -> np.ndarray: """ Generate binary cloud masks with s2cloudless """ cloud_detector = S2PixelCloudDetector(threshold=CLOUD_THRESHOLD, all_bands=True) return np.squeeze(cloud_detector.get_cloud_masks(array)).astype("byte")
def cloud_process(bounding_box, Date_Ini, Date_Fin, x_width, y_height, analysis_area, clouds_folder, lote_aoi, municipio, departamento): INSTANCE_ID = '3a63d637-11ad-493a-b921-91be7c4da68d' #From Sentinel HUB Python Instance ID /change to dynamic user input LAYER_NAME = 'TRUE-COLOR-S2-L1C' # e.g. TRUE-COLOR-S2-L1C #Obtener imagenes por fecha (dentro de rango) dentro de box de interés wms_true_color_request = WmsRequest( layer=LAYER_NAME, bbox=bounding_box, time=(Date_Ini, Date_Fin), #cambiar a fechas de interés width=x_width, height=y_height, image_format=MimeType.PNG, time_difference=datetime.timedelta(hours=2), instance_id=INSTANCE_ID) wms_true_color_imgs = wms_true_color_request.get_data() #Cloudless_tools.plot_previews(np.asarray(wms_true_color_imgs), wms_true_color_request.get_dates(), cols=4, figsize=(15, 10)) #count of 0's to know how empty is the image count_of_zeros = [] for n in range(0, len(wms_true_color_imgs)): # zeros / 4 channels * width * height (pixels) count_of_zeros.append( (np.count_nonzero(wms_true_color_imgs[n] == 0)) / (4 * wms_true_color_imgs[n][:, :, 0].shape[0] * wms_true_color_imgs[n][:, :, 0].shape[1])) #Calculo de probabilidades y obtención de mascaras de nubes bands_script = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]' wms_bands_request = WmsRequest( layer=LAYER_NAME, custom_url_params={ CustomUrlParam.EVALSCRIPT: bands_script, CustomUrlParam.ATMFILTER: 'NONE' }, bbox=bounding_box, time=(Date_Ini, Date_Fin), width=x_width, height=y_height, image_format=MimeType.TIFF_d32f, time_difference=datetime.timedelta(hours=2), instance_id=INSTANCE_ID) wms_bands = wms_bands_request.get_data() #wms_bands_request.get_filename_list() #wms_bands_request.get_url_list() #wms_bands_request.get_dates() cloud_detector = S2PixelCloudDetector( threshold=0.35, average_over=8, dilation_size=3) #change threshold to test #cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands)) cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands)) all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request, threshold=0.35) #cloud_masks = all_cloud_masks.get_cloud_masks() #Mostrar las probabilidades de nubes para cada imagen por fecha en el rango de analisis n_cols = 4 n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols)) fig = plt.figure(figsize=(n_cols * 4, n_rows * 3)) #, constrained_layout=False for idx, [prob, mask, data] in enumerate(all_cloud_masks): ax = fig.add_subplot(n_rows, n_cols, idx + 1) image = wms_true_color_imgs[idx] Cloudless_tools.overlay_cloud_mask(image, mask, factor=1, fig=fig) plt.tight_layout() plt.savefig(clouds_folder + analysis_area + '/real_and_cloud.png') #Mostrar las mascaras de nubes para cada imagen por fecha en el rango de analisis n_cols = 4 n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols)) fig = plt.figure(figsize=(n_cols * 4, n_rows * 3)) #each_cld_mask = all_cloud_masks.get_cloud_masks(threshold=0.35) cld_per_idx = [] for idx, cloud_mask in enumerate( all_cloud_masks.get_cloud_masks(threshold=0.35)): ax = fig.add_subplot(n_rows, n_cols, idx + 1) #correct mask, when no data is in the image, to mask non values cloud_mask[wms_true_color_imgs[idx][:, :, 0] == 0] = 1 Cloudless_tools.plot_cloud_mask(cloud_mask, fig=fig) n_cloud_mask = np.shape(np.concatenate(cloud_mask)) cloud_perc = sum(np.concatenate(cloud_mask) == 1) / n_cloud_mask cld_per_idx.append(cloud_perc.astype(float)) plt.tight_layout() plt.savefig(clouds_folder + analysis_area + '/cloud_masks.png') #Calculo y extracción de imagenes con cobertura de nubes menor a x% x = pd.DataFrame( cld_per_idx ) < 0.6 #Menor a 60% de cobertura de nubes // or lote is visible TO ADD all_dates = pd.DataFrame(all_cloud_masks.get_dates()) valid_dates = all_dates[x[0]] all_dates['year-month'] = all_dates[0].dt.to_period('M') all_dates['cld_percent'] = cld_per_idx all_dates['empty_percent'] = count_of_zeros all_dates = all_dates.rename(columns={0: 'dates'}) #summary ''' summary_clds = all_dates[['year-month','cld_percent','dates']].groupby('year-month').agg({'dates':lambda x: x.diff().mean(), 'cld_percent': ['count', lambda x: (x<0.6).sum(), lambda x: x.mean(), 'min']}) \ .reset_index() ''' def f_mi(x): d = [] d.append(x['dates'].diff().mean()) d.append(x['cld_percent'].count()) d.append((x['cld_percent'] < 0.6).sum()) d.append(x['cld_percent'].mean()) d.append(x['cld_percent'].min()) d.append(x[x['cld_percent'] < 0.6]['dates'].max()) d.append(x[x['cld_percent'] < 0.6]['dates'].min()) d.append(x['empty_percent'].max()) d.append(x['empty_percent'].min()) return pd.Series(d, index=[ 'time_between_pass', 'count_pass', 'clear_images', 'mean_cloud_cover', 'min_cloud_cover', 'last_good_date', 'first_good_date', 'max_empty_space', 'min_empty_space' ]) # summary_clds = all_dates.groupby('year-month').apply(f_mi) summary_clds['centroid_x'], summary_clds['centroid_y'], summary_clds[ 'terrain_name'], summary_clds['terrain_code'], summary_clds[ 'municipio'], summary_clds['departamento'] = lote_aoi['x'][ 0], lote_aoi['y'][0], lote_aoi['name'][ 0], analysis_area, municipio, departamento #export data summary_clds.to_csv(clouds_folder + analysis_area + '/Analisis_nubes.csv', index=True, header=True) #filter clouds dataframe with only valid dates clouds_data = cloud_masks[x[0]] minIndex = cld_per_idx.index(min(cld_per_idx)) best_date = valid_dates[valid_dates.index == minIndex] best_date = best_date.iloc[0, 0] #Mostrar las mascaras de nubes para cada imagen por fecha valida n_cols = 4 n_rows = int(np.ceil(len(clouds_data) / n_cols)) fig = plt.figure(figsize=(n_cols * 4, n_rows * 3)) for idx, cloud_mask in enumerate(clouds_data): ax = fig.add_subplot(n_rows, n_cols, idx + 1) Cloudless_tools.plot_cloud_mask(cloud_mask, fig=fig) plt.tight_layout() plt.savefig(clouds_folder + analysis_area + '/cloud_masks_valid.png') clear_pct = len(valid_dates) / len(cld_per_idx) number_cld_analysis = len(cld_per_idx) return best_date, valid_dates, clouds_data, clear_pct, number_cld_analysis