def setUpClass(cls):
super().setUpClass()
bbox1 = BBox([-90.9216499, 14.4190528, -90.8186531, 14.5520163],
crs=CRS.WGS84) # From examples
bbox2 = BBox([46.16, -16.15, 46.51, -15.58],
crs=CRS.WGS84) # From sentinelhub-py examples
cls.custom_url_params = {
CustomUrlParam.SHOWLOGO: True,
CustomUrlParam.TRANSPARENT: False,
CustomUrlParam.EVALSCRIPT: 'return [B01]',
CustomUrlParam.ATMFILTER: 'DOS1'
}
cls.wms_request = WmsRequest(layer='S2-DOS1',
bbox=bbox1,
time=('2017-12-01', '2017-12-31'),
width=60,
height=None,
image_format=MimeType.TIFF,
custom_url_params=cls.custom_url_params,
instance_id=cls.CONFIG.instance_id)
cls.wcs_request = WcsRequest(layer='S2-ATMCOR',
bbox=bbox2,
time='2016-07-18T07:14:04',
resx='100m',
resy='100m',
image_format=MimeType.PNG,
data_folder='.')
cls.test_cases = [
TestCaseContainer('WMS',
CloudMaskRequest(cls.wms_request,
threshold=0.6,
average_over=2,
dilation_size=5),
clm_min=0,
clm_max=1,
clm_mean=0.343827,
clm_median=0,
clp_min=0.00011,
clp_max=0.99999,
clp_mean=0.23959,
clp_median=0.01897,
mask_shape=(7, 81, 60)),
TestCaseContainer('WCS, partial no data',
CloudMaskRequest(cls.wcs_request,
all_bands=True),
clm_min=0,
clm_max=1,
clm_mean=0.04468,
clm_median=0,
clp_min=-50.0,
clp_max=0.999635,
clp_mean=-7.5472468,
clp_median=0.011568,
mask_shape=(1, 634, 374))
]
def get_cloud(lefttoplon, lefttoplat, rightbtmlon, rightbtmlat, time):
"""
Return cloud masks given a bounding box and time
"""
if abs(lefttoplon) > 180 or abs(rightbtmlon) > 180:
print("wrong longitude")
return None
if abs(lefttoplat) > 90 or abs(rightbtmlat) > 90:
print("wrong latitude")
return None
bands_script = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]'
desired_coords_wgs84 = [lefttoplon, lefttoplat, rightbtmlon, rightbtmlat]
desired_bbox = BBox(bbox=desired_coords_wgs84, crs=CRS.WGS84)
wms_bands_request = WmsRequest(
layer='TRUE_COLOR',
custom_url_params={CustomUrlParam.EVALSCRIPT: bands_script},
bbox=desired_bbox,
time=time,
width=100,
height=100,
image_format=MimeType.TIFF_d32f,
instance_id=INSTANCE_ID)
all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
threshold=0.4)
cloud_dates = all_cloud_masks.get_dates()
cloud_masks = all_cloud_masks.get_cloud_masks(threshold=0.4)
return cloud_masks, cloud_dates
def __init__(self, project_name, bbox, time_interval, instance_id, full_size=(1920, 1080), preview_size=(455, 256),
cloud_mask_res=('60m', '60m'), use_atmcor=True, layer='TRUE_COLOR',
time_difference=datetime.timedelta(seconds=-1)):
self.project_name = project_name
self.preview_request = WmsRequest(data_folder=project_name + '/previews', layer=layer, bbox=bbox,
time=time_interval, width=preview_size[0], height=preview_size[1],
maxcc=1.0, image_format=MimeType.PNG, instance_id=instance_id,
custom_url_params={CustomUrlParam.TRANSPARENT: True},
time_difference=time_difference)
self.fullres_request = WcsRequest(data_folder=project_name + '/fullres', layer=layer, bbox=bbox,
time=time_interval, resx='10m', resy='10m',
maxcc=1.0, image_format=MimeType.PNG, instance_id=instance_id,
custom_url_params={CustomUrlParam.TRANSPARENT: True,
CustomUrlParam.ATMFILTER: 'ATMCOR'} if use_atmcor else {CustomUrlParam.TRANSPARENT: True},
time_difference=time_difference)
wcs_request = WcsRequest(layer=layer, bbox=bbox, time=time_interval,
resx=cloud_mask_res[0], resy=cloud_mask_res[1], maxcc=1.0,
image_format=MimeType.TIFF_d32f, instance_id=instance_id,
time_difference=time_difference, custom_url_params={CustomUrlParam.EVALSCRIPT:
MODEL_EVALSCRIPT})
self.cloud_mask_request = CloudMaskRequest(wcs_request)
self.transparency_data = None
self.preview_transparency_data = None
self.invalid_coverage = None
self.dates = self.preview_request.get_dates()
if not self.dates:
raise ValueError('Input parameters are not valid. No Sentinel 2 image is found.')
if self.dates != self.fullres_request.get_dates():
raise ValueError('Lists of previews and full resolution images do not match.')
if self.dates != self.cloud_mask_request.get_dates():
raise ValueError('List of previews and cloud masks do not match.')
self.mask = np.zeros((len(self.dates),), dtype=np.uint8)
self.cloud_masks = None
self.cloud_coverage = None
self.full_res_data = None
self.previews = None
self.full_size = full_size
self.timelapse = None
LOGGER.info('Found %d images of %s between %s and %s.', len(self.dates), project_name,
time_interval[0], time_interval[1])
LOGGER.info('\nI suggest you start by downloading previews first to see,\n'
'if BBOX is OK, images are usefull, etc...\n'
'Execute get_previews() method on your object.\n')
def test_no_data_available_request(config):
""" Tests an exception raised by CloudMaskRequest
"""
cloud_detector = S2PixelCloudDetector()
with pytest.raises(NoDataAvailableException):
CloudMaskRequest(cloud_detector,
bbox=BBOX1,
time=('2021-01-01', '2021-01-10'),
size=(250, 250),
maxcc=0.01,
config=config)
def test_cloud_mask_request(input_params, stats, config, subtests):
""" Integration tests for CloudMasKRequest class that interacts with Sentinel Hub service
"""
request = CloudMaskRequest(config=config, **input_params)
masks = request.get_cloud_masks()
_test_numpy_data(subtests,
masks,
exp_shape=stats['mask_shape'],
exp_dtype=np.int8,
exp_min=stats['clm_min'],
exp_max=stats['clm_max'],
exp_mean=stats['clm_mean'],
exp_median=stats['clm_median'],
delta=1e-4)
prob_masks = request.get_probability_masks(non_valid_value=-50)
_test_numpy_data(subtests,
prob_masks,
exp_shape=stats['mask_shape'],
exp_dtype=np.float64,
exp_min=stats['clp_min'],
exp_max=stats['clp_max'],
exp_mean=stats['clp_mean'],
exp_median=stats['clp_median'],
delta=1e-4)
timestamps = request.get_timestamps()
assert isinstance(timestamps, list)
assert len(timestamps) == stats['mask_shape'][0]
assert all(isinstance(timestamp, dt.datetime) for timestamp in timestamps)
data = request.get_data()
band_num = 13 if request.cloud_detector.all_bands else 10
assert data.shape == stats['mask_shape'] + (band_num, )
assert data.dtype == np.float32
data_mask = request.get_data_mask()
assert data_mask.shape == stats['mask_shape']
assert data_mask.dtype == bool
def get_all_bands(bbox, layer, SRS="epsg:3912"):
minx, miny = bbox[0][0] #bbox[0:2]
maxx, maxy = bbox[0][2] #bbox[4:6]
right, bot = transform(Proj(init=SRS), Proj(init='epsg:4326'), maxx, miny)
left, top = transform(Proj(init=SRS), Proj(init='epsg:4326'), minx, maxy)
bounding_box = BBox([top, left, bot, right], crs=CRS.WGS84)
#lat, lon = top-bot, right-left
height, width = round(maxy - miny), round(maxx - minx)
bands_script = "return [B01,B02,B03,B04,B05,B08,B8A,B09,B10,B11,B12]"
wms_bands_request = WmsRequest(layer=layer,
custom_url_params={
CustomUrlParam.EVALSCRIPT: bands_script,
CustomUrlParam.ATMFILTER: 'NONE'
},
bbox=bounding_box,
time=('2017-01-01', '2018-12-01'),
width=width,
height=height,
image_format=MimeType.TIFF_d32f,
instance_id=api_key)
all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
threshold=0.1)
masks = []
wms_bands = []
for idx, [prob, mask, data] in enumerate(all_cloud_masks):
masks.append(mask)
wms_bands.append(data)
return wms_bands, masks, wms_bands_request.get_dates()
custom_url_params={
CustomUrlParam.EVALSCRIPT: bands_script,
CustomUrlParam.ATMFILTER: 'NONE'
},
bbox=bounding_box,
time=(Date_Ini, Date_Fin),
width=x_width,
height=y_height,
image_format=MimeType.TIFF_d32f,
instance_id=INSTANCE_ID)
wms_bands = wms_bands_request.get_data()
cloud_detector = S2PixelCloudDetector(
threshold=0.35, average_over=8, dilation_size=3) #change threshold to test
cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands))
cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands))
all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
threshold=0.1)
#folder de imagenes nubes
Path('output_clouds/' + analysis_area).mkdir(parents=True, exist_ok=True)
if not os.path.exists(analysis_area):
os.makedirs(analysis_area)
#Mostrar las probabilidades de nubes para cada imagen por fecha en el rango de analisis
fig = plt.figure(figsize=(15, 10))
n_cols = 4
n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))
for idx, [prob, mask, data] in enumerate(all_cloud_masks):
ax = fig.add_subplot(n_rows, n_cols, idx + 1)
image = wms_true_color_imgs[idx]
Cloudless_tools.overlay_cloud_mask(image, mask, factor=1, fig=fig)
plt.tight_layout()
def extract_surface_water_area_per_frame(dam_id, dam_poly, dam_bbox, date,
resx, resy):
"""
Run water detection algorithm for a single timestamp.
"""
measurement = get_new_measurement_entry(dam_id, date,
WaterDetectionSensor.S2_NDWI,
S2_WATER_DETECTOR_VERSION)
# initialise requests
try:
wcs_ndwi_request = WcsRequest(layer='NDWI',
bbox=dam_bbox,
time=date.strftime('%Y-%m-%d'),
maxcc=S2_MAX_CC,
resx=f'{resx}m',
resy=f'{resy}m',
image_format=MimeType.TIFF_d32f,
time_difference=timedelta(hours=2),
custom_url_params={
CustomUrlParam.SHOWLOGO: False,
CustomUrlParam.TRANSPARENT: True
})
cloudresx, cloudresy = get_optimal_cloud_resolution(resx, resy)
wcs_bands_request = WcsRequest(layer='NDWI',
bbox=dam_bbox,
time=date.strftime('%Y-%m-%d'),
maxcc=S2_MAX_CC,
resx=f'{cloudresx}m',
resy=f'{cloudresy}m',
image_format=MimeType.TIFF_d32f,
time_difference=timedelta(hours=2),
custom_url_params={
CustomUrlParam.EVALSCRIPT:
S2_CLOUD_BANDS_SCRIPT_V3
})
except (RuntimeError, DownloadFailedException):
set_measurement_status(measurement,
WaterDetectionStatus.SH_REQUEST_ERROR)
return measurement
# download NDWI
try:
ndwi = np.asarray(wcs_ndwi_request.get_data())
except (DownloadFailedException, ImageDecodingError):
set_measurement_status(measurement,
WaterDetectionStatus.SH_REQUEST_ERROR)
return measurement
if len(ndwi) == 0:
set_measurement_status(measurement, WaterDetectionStatus.SH_NO_DATA)
return measurement
# check that image has no INVALID PIXELS
valid_pxs_frac = np.count_nonzero(ndwi[..., 1]) / np.size(ndwi[..., 1])
if valid_pxs_frac < S2_MIN_VALID_FRACTION:
del ndwi
set_measurement_status(measurement, WaterDetectionStatus.INVALID_DATA)
return measurement
# run cloud detection
try:
all_cloud_masks = CloudMaskRequest(ogc_request=wcs_bands_request,
threshold=0.4)
cloud_mask = all_cloud_masks.get_cloud_masks()
except (DownloadFailedException, ImageDecodingError):
set_measurement_status(measurement,
WaterDetectionStatus.SH_REQUEST_ERROR)
return measurement
if len(ndwi) == 0:
set_measurement_status(measurement,
WaterDetectionStatus.SH_NO_CLOUD_DATA)
return measurement
# check cloud coverage
cloud_cov = np.count_nonzero(cloud_mask) / np.size(cloud_mask)
if cloud_cov > S2_MAX_CLOUD_COVERAGE:
del cloud_mask, all_cloud_masks
set_measurement_status(measurement, WaterDetectionStatus.TOO_CLOUDY)
return measurement
measurement.CLOUD_COVERAGE = cloud_cov
try:
# run water detction algorithm
result = get_water_level_optical(date,
ndwi[0, ..., 0],
dam_poly,
dam_bbox,
simplify=True)
set_measurement_status(measurement,
WaterDetectionStatus.MEASUREMENT_VALID)
measurement.SURF_WATER_LEVEL = result['water_level']
measurement.GEOMETRY = result['geometry'].wkt
measurement.ALG_STATUS = result['alg_status']
del result
except AttributeError:
set_measurement_status(measurement,
WaterDetectionStatus.INVALID_POLYGON)
del ndwi, cloud_mask, all_cloud_masks, wcs_ndwi_request, wcs_bands_request
return measurement
def cloud_process(bounding_box, Date_Ini, Date_Fin, x_width, y_height,
analysis_area, clouds_folder, lote_aoi, municipio,
departamento):
INSTANCE_ID = '3a63d637-11ad-493a-b921-91be7c4da68d' #From Sentinel HUB Python Instance ID /change to dynamic user input
LAYER_NAME = 'TRUE-COLOR-S2-L1C' # e.g. TRUE-COLOR-S2-L1C
#Obtener imagenes por fecha (dentro de rango) dentro de box de interés
wms_true_color_request = WmsRequest(
layer=LAYER_NAME,
bbox=bounding_box,
time=(Date_Ini, Date_Fin), #cambiar a fechas de interés
width=x_width,
height=y_height,
image_format=MimeType.PNG,
time_difference=datetime.timedelta(hours=2),
instance_id=INSTANCE_ID)
wms_true_color_imgs = wms_true_color_request.get_data()
#Cloudless_tools.plot_previews(np.asarray(wms_true_color_imgs), wms_true_color_request.get_dates(), cols=4, figsize=(15, 10))
#count of 0's to know how empty is the image
count_of_zeros = []
for n in range(0, len(wms_true_color_imgs)):
# zeros / 4 channels * width * height (pixels)
count_of_zeros.append(
(np.count_nonzero(wms_true_color_imgs[n] == 0)) /
(4 * wms_true_color_imgs[n][:, :, 0].shape[0] *
wms_true_color_imgs[n][:, :, 0].shape[1]))
#Calculo de probabilidades y obtención de mascaras de nubes
bands_script = 'return [B01,B02,B04,B05,B08,B8A,B09,B10,B11,B12]'
wms_bands_request = WmsRequest(
layer=LAYER_NAME,
custom_url_params={
CustomUrlParam.EVALSCRIPT: bands_script,
CustomUrlParam.ATMFILTER: 'NONE'
},
bbox=bounding_box,
time=(Date_Ini, Date_Fin),
width=x_width,
height=y_height,
image_format=MimeType.TIFF_d32f,
time_difference=datetime.timedelta(hours=2),
instance_id=INSTANCE_ID)
wms_bands = wms_bands_request.get_data()
#wms_bands_request.get_filename_list()
#wms_bands_request.get_url_list()
#wms_bands_request.get_dates()
cloud_detector = S2PixelCloudDetector(
threshold=0.35, average_over=8,
dilation_size=3) #change threshold to test
#cloud_probs = cloud_detector.get_cloud_probability_maps(np.array(wms_bands))
cloud_masks = cloud_detector.get_cloud_masks(np.array(wms_bands))
all_cloud_masks = CloudMaskRequest(ogc_request=wms_bands_request,
threshold=0.35)
#cloud_masks = all_cloud_masks.get_cloud_masks()
#Mostrar las probabilidades de nubes para cada imagen por fecha en el rango de analisis
n_cols = 4
n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))
fig = plt.figure(figsize=(n_cols * 4,
n_rows * 3)) #, constrained_layout=False
for idx, [prob, mask, data] in enumerate(all_cloud_masks):
ax = fig.add_subplot(n_rows, n_cols, idx + 1)
image = wms_true_color_imgs[idx]
Cloudless_tools.overlay_cloud_mask(image, mask, factor=1, fig=fig)
plt.tight_layout()
plt.savefig(clouds_folder + analysis_area + '/real_and_cloud.png')
#Mostrar las mascaras de nubes para cada imagen por fecha en el rango de analisis
n_cols = 4
n_rows = int(np.ceil(len(wms_true_color_imgs) / n_cols))
fig = plt.figure(figsize=(n_cols * 4, n_rows * 3))
#each_cld_mask = all_cloud_masks.get_cloud_masks(threshold=0.35)
cld_per_idx = []
for idx, cloud_mask in enumerate(
all_cloud_masks.get_cloud_masks(threshold=0.35)):
ax = fig.add_subplot(n_rows, n_cols, idx + 1)
#correct mask, when no data is in the image, to mask non values
cloud_mask[wms_true_color_imgs[idx][:, :, 0] == 0] = 1
Cloudless_tools.plot_cloud_mask(cloud_mask, fig=fig)
n_cloud_mask = np.shape(np.concatenate(cloud_mask))
cloud_perc = sum(np.concatenate(cloud_mask) == 1) / n_cloud_mask
cld_per_idx.append(cloud_perc.astype(float))
plt.tight_layout()
plt.savefig(clouds_folder + analysis_area + '/cloud_masks.png')
#Calculo y extracción de imagenes con cobertura de nubes menor a x%
x = pd.DataFrame(
cld_per_idx
) < 0.6 #Menor a 60% de cobertura de nubes // or lote is visible TO ADD
all_dates = pd.DataFrame(all_cloud_masks.get_dates())
valid_dates = all_dates[x[0]]
all_dates['year-month'] = all_dates[0].dt.to_period('M')
all_dates['cld_percent'] = cld_per_idx
all_dates['empty_percent'] = count_of_zeros
all_dates = all_dates.rename(columns={0: 'dates'})
#summary
'''
summary_clds = all_dates[['year-month','cld_percent','dates']].groupby('year-month').agg({'dates':lambda x: x.diff().mean(), 'cld_percent': ['count', lambda x: (x<0.6).sum(), lambda x: x.mean(), 'min']}) \
.reset_index()
'''
def f_mi(x):
d = []
d.append(x['dates'].diff().mean())
d.append(x['cld_percent'].count())
d.append((x['cld_percent'] < 0.6).sum())
d.append(x['cld_percent'].mean())
d.append(x['cld_percent'].min())
d.append(x[x['cld_percent'] < 0.6]['dates'].max())
d.append(x[x['cld_percent'] < 0.6]['dates'].min())
d.append(x['empty_percent'].max())
d.append(x['empty_percent'].min())
return pd.Series(d,
index=[
'time_between_pass', 'count_pass',
'clear_images', 'mean_cloud_cover',
'min_cloud_cover', 'last_good_date',
'first_good_date', 'max_empty_space',
'min_empty_space'
]) #
summary_clds = all_dates.groupby('year-month').apply(f_mi)
summary_clds['centroid_x'], summary_clds['centroid_y'], summary_clds[
'terrain_name'], summary_clds['terrain_code'], summary_clds[
'municipio'], summary_clds['departamento'] = lote_aoi['x'][
0], lote_aoi['y'][0], lote_aoi['name'][
0], analysis_area, municipio, departamento
#export data
summary_clds.to_csv(clouds_folder + analysis_area +
'/Analisis_nubes.csv',
index=True,
header=True)
#filter clouds dataframe with only valid dates
clouds_data = cloud_masks[x[0]]
minIndex = cld_per_idx.index(min(cld_per_idx))
best_date = valid_dates[valid_dates.index == minIndex]
best_date = best_date.iloc[0, 0]
#Mostrar las mascaras de nubes para cada imagen por fecha valida
n_cols = 4
n_rows = int(np.ceil(len(clouds_data) / n_cols))
fig = plt.figure(figsize=(n_cols * 4, n_rows * 3))
for idx, cloud_mask in enumerate(clouds_data):
ax = fig.add_subplot(n_rows, n_cols, idx + 1)
Cloudless_tools.plot_cloud_mask(cloud_mask, fig=fig)
plt.tight_layout()
plt.savefig(clouds_folder + analysis_area + '/cloud_masks_valid.png')
clear_pct = len(valid_dates) / len(cld_per_idx)
number_cld_analysis = len(cld_per_idx)
return best_date, valid_dates, clouds_data, clear_pct, number_cld_analysis