def test_cloud_coverage(self):
classifier = get_s2_pixel_cloud_detector(all_bands=True)
# Classifier is run on same resolution as data array
add_cm = AddCloudMaskTask(classifier,
'ALL_BANDS',
cmask_feature='clm',
cprobs_feature='clp')
eop_clm = add_cm(self.eop)
_, h, w, _ = eop_clm.mask['clm'].shape
cc = np.sum(eop_clm.mask['clm'][0]) / (w * h)
ps = np.sum(eop_clm.data['clp'][0]) / (w * h)
self.assertTrue(eop_clm.mask['clm'].ndim == 4)
self.assertAlmostEqual(cc, 0.687936507936508, places=4)
self.assertAlmostEqual(ps, 0.521114510213301, places=4)
del add_cm, eop_clm
# Classifier is run on downscaled version of data array
add_cm = AddCloudMaskTask(classifier,
'ALL_BANDS',
cm_size_y=50,
cmask_feature='clm',
cprobs_feature='clp')
eop_clm = add_cm(self.eop)
_, h, w, _ = eop_clm.mask['clm'].shape
cc = np.sum(eop_clm.mask['clm'][0]) / (w * h)
ps = np.sum(eop_clm.data['clp'][0]) / (w * h)
self.assertTrue(eop_clm.mask['clm'].ndim == 4)
self.assertAlmostEqual(cc, 0.710357142857142, places=4)
self.assertAlmostEqual(ps, 0.500692345333859, places=4)
def test_cloud_coverage(self):
classifier = get_s2_pixel_cloud_detector(all_bands=True)
# Classifier is run on same resolution as data array
add_cm = AddCloudMaskTask(classifier,
'ALL_DATA',
cmask_feature='CLM_TEST',
cprobs_feature='CLP_TEST')
eop_clm = add_cm(self.eop)
# Check shape and type
self._check_shape(eop_clm.mask['CLM_TEST'], eop_clm.data['ALL_DATA'])
self._check_shape(eop_clm.data['CLP_TEST'], eop_clm.data['ALL_DATA'])
self.assertTrue(eop_clm.mask['CLM_TEST'].dtype == np.bool)
self.assertTrue(eop_clm.data['CLP_TEST'].dtype == np.float32)
# Compare mean cloud coverage with provided reference
mean_clm_provided = np.mean(eop_clm.mask['CLM'])
mean_clp_provided = np.mean(eop_clm.data['CLP'])
self.assertAlmostEqual(np.mean(eop_clm.mask['CLM_TEST']),
mean_clm_provided,
places=1)
self.assertAlmostEqual(np.mean(eop_clm.data['CLP_TEST']),
mean_clp_provided,
places=2)
# Classifier is run on downscaled version of data array
add_cm = AddCloudMaskTask(classifier,
'ALL_DATA',
cm_size_y='20m',
cm_size_x='20m',
cmask_feature='CLM_TEST',
cprobs_feature='CLP_TEST')
eop_clm = add_cm(self.eop)
# Check shape and type
self._check_shape(eop_clm.mask['CLM_TEST'], eop_clm.data['ALL_DATA'])
self._check_shape(eop_clm.data['CLP_TEST'], eop_clm.data['ALL_DATA'])
self.assertTrue(eop_clm.mask['CLM_TEST'].dtype == np.bool)
self.assertTrue(eop_clm.data['CLP_TEST'].dtype == np.float32)
# Compare mean cloud coverage with provided reference
mean_clm_provided = np.mean(eop_clm.mask['CLM'])
mean_clp_provided = np.mean(eop_clm.data['CLP'])
self.assertAlmostEqual(np.mean(eop_clm.mask['CLM_TEST']),
mean_clm_provided,
places=2)
self.assertAlmostEqual(np.mean(eop_clm.data['CLP_TEST']),
mean_clp_provided,
places=2)
# Check if same times are flagged as cloudless
cloudless = np.mean(eop_clm.mask['CLM_TEST'], axis=(1, 2, 3)) == 0
self.assertTrue(
np.all(cloudless == eop_clm.label['IS_CLOUDLESS'][:, 0]))
def test_wms_request(self):
classifier = get_s2_pixel_cloud_detector(all_bands=False)
# Classifier is run on new request of data array
add_cm = AddCloudMaskTask(classifier,
'BANDS-S2-L1C',
cm_size_x='20m',
cm_size_y='20m',
cmask_feature='CLM_TEST',
cprobs_feature='CLP_TEST')
eop_clm = add_cm(self.eop)
# Check shape and type
self._check_shape(eop_clm.mask['CLM_TEST'], eop_clm.data['ALL_DATA'])
self._check_shape(eop_clm.data['CLP_TEST'], eop_clm.data['ALL_DATA'])
self.assertTrue(eop_clm.mask['CLM_TEST'].dtype == np.bool)
self.assertTrue(eop_clm.data['CLP_TEST'].dtype == np.float32)
# Compare mean cloud coverage with provided reference
mean_clm_provided = np.mean(eop_clm.mask['CLM'])
mean_clp_provided = np.mean(eop_clm.data['CLP'])
self.assertAlmostEqual(np.mean(eop_clm.mask['CLM_TEST']),
mean_clm_provided,
places=2)
self.assertAlmostEqual(np.mean(eop_clm.data['CLP_TEST']),
mean_clp_provided,
places=2)
def gather_data():
cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False)
add_clm = AddCloudMaskTask(cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='80m', cm_size_x='80m',
cmask_feature='CLM', # cloud mask name
cprobs_feature='CLP' # cloud prob. map name
)
ndvi = NormalizedDifferenceIndex('NDVI', 'BANDS/3', 'BANDS/2')
add_sh_valmask = AddValidDataMaskTask(SentinelHubValidData(),
'IS_VALID')
layer = 'BANDS-S2-L1C'
custom_script = 'return [B02, B03];'
input_task = S2L1CWCSInput(layer=layer,
feature=(FeatureType.DATA, 'BANDS'),
custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script},
resx='10m', resy='10m',
maxcc=.8)
add_ndvi = S2L1CWCSInput(layer='NDVI')
save = SaveToDisk('io_example', overwrite_permission=2)#compress_level=1
workflow = LinearWorkflow(
input_task,
add_clm,
add_ndvi,
add_sh_valmask,
)
time_interval = ('2017-01-01', '2017-12-31')
result = workflow.execute({input_task: {'bbox': roi_bbox, 'time_interval': time_interval},
save: {'eopatch_folder': 'eopatch'}})
return list(result.values())[0].data['NDVI'], list(result.values())[-1].mask['IS_VALID'], np.array(list(result.values())[0].timestamp)
def test_wms_request(self):
classifier = get_s2_pixel_cloud_detector(all_bands=False)
# Classifier is run on new request of data array
add_cm = AddCloudMaskTask(classifier,
'BANDS-S2-L1C',
cm_size_y=50,
cmask_feature='clm',
cprobs_feature='clp')
eop_clm = add_cm(self.eop)
_, h, w, _ = eop_clm.mask['clm'].shape
cc = np.sum(eop_clm.mask['clm'][0]) / (w * h)
ps = np.sum(eop_clm.data['clp'][0]) / (w * h)
self.assertTrue(eop_clm.mask['clm'].ndim == 4)
self.assertAlmostEqual(cc, 0.737738, places=4)
self.assertAlmostEqual(ps, 0.520182, places=4)
def cloud_classifier_task():
'''A convenience function that sets up the cloud detection task.
Configures an instance of the EOTask s2_pixel_cloud_detector and AddCloudMaskTask
'''
cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
dilation_size=1,
all_bands=False)
cloud_detection = AddCloudMaskTask(cloud_classifier,
'BANDS-S2CLOUDLESS',
cm_size_y='40m',
cm_size_x='40m',
cmask_feature='CLM',
cprobs_feature='CLP')
return cloud_detection
resx='10m', # resolution x
resy='10m', # resolution y
maxcc=0.8, # maximum allowed cloud cover of original ESA tiles
)
# TASK FOR CLOUD INFO
# cloud detection is performed at 80m resolution
# and the resulting cloud probability map and mask
# are scaled to EOPatch's resolution
cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
dilation_size=1,
all_bands=False)
add_clm = AddCloudMaskTask(
cloud_classifier,
'BANDS-S2CLOUDLESS',
cm_size_y='80m',
cm_size_x='80m',
cmask_feature='CLM', # cloud mask name
cprobs_feature='CLP' # cloud prob. map name
)
# TASKS FOR CALCULATING NEW FEATURES
# NDVI: (B08 - B04)/(B08 + B04)
# NDWI: (B03 - B08)/(B03 + B08)
# NORM: sqrt(B02^2 + B03^2 + B04^2 + B08^2 + B11^2 + B12^2)
ndvi = NormalizedDifferenceIndex('NDVI', 'BANDS/3', 'BANDS/2')
ndwi = NormalizedDifferenceIndex('NDWI', 'BANDS/1', 'BANDS/3')
norm = EuclideanNorm('NORM', 'BANDS')
# TASK FOR VALID MASK
# validate pixels using SentinelHub's cloud detection mask and region of acquisition
add_sh_valmask = AddValidDataMaskTask(
instance_id=WMS_INSTANCE)
add_ndwi = S2L1CWCSInput('NDWI', instance_id=WMS_INSTANCE)
gdf = gpd.GeoDataFrame(crs={'init': 'epsg:4326'}, geometry=[dam_nominal])
gdf.plot()
add_nominal_water = VectorToRaster(
(FeatureType.MASK_TIMELESS, 'NOMINAL_WATER'), gdf, 1,
(FeatureType.MASK, 'IS_DATA'), np.uint8)
cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
dilation_size=1,
all_bands=False)
cloud_det = AddCloudMaskTask(cloud_classifier,
'BANDS-S2CLOUDLESS',
cm_size_y='60m',
cm_size_x='60m',
cmask_feature='CLM',
cprobs_feature='CLP',
instance_id=WMS_INSTANCE)
class ValidDataPredicate:
def __call__(self, eopatch):
return np.logical_and(
eopatch.mask['IS_DATA'].astype(np.bool),
np.logical_not(eopatch.mask['CLM'].astype(np.bool)))
add_valmask = AddValidDataMaskTask(predicate=ValidDataPredicate())
def test_raises_errors(self):
classifier = get_s2_pixel_cloud_detector(all_bands=True)
add_cm = AddCloudMaskTask(classifier, 'bands', cmask_feature='clm')
self.assertRaises(ValueError, add_cm, self.eop)
def download_data(path_save,
coords_top,
coords_bot,
patch_n,
s_date,
e_date,
debug=False):
# before moving onto actual tasks, check setup
check_sentinel_cfg()
[lat_left_top, lon_left_top] = coords_top
[lat_right_bot, lon_right_bot] = coords_bot
# TASK FOR BAND DATA
# add a request for B(B02), G(B03), R(B04), NIR (B08), SWIR1(B11), SWIR2(B12)
# from default layer 'ALL_BANDS' at 10m resolution
# Here we also do a simple filter of cloudy scenes. A detailed cloud cover
# detection is performed in the next step
custom_script = "return [B02, B03, B04, B08, B11, B12];"
add_data = S2L1CWCSInput(
layer="BANDS-S2-L1C",
feature=(FeatureType.DATA, "BANDS"), # save under name 'BANDS'
# custom url for 6 specific bands
custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script},
resx="10m", # resolution x
resy="10m", # resolution y
maxcc=0.1, # maximum allowed cloud cover of original ESA tiles
)
# TASK FOR CLOUD INFO
# cloud detection is performed at 80m resolution
# and the resulting cloud probability map and mask
# are scaled to EOPatch's resolution
cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
dilation_size=1,
all_bands=False)
add_clm = AddCloudMaskTask(
cloud_classifier,
"BANDS-S2CLOUDLESS",
cm_size_y="80m",
cm_size_x="80m",
cmask_feature="CLM", # cloud mask name
cprobs_feature="CLP", # cloud prob. map name
)
# TASKS FOR CALCULATING NEW FEATURES
# NDVI: (B08 - B04)/(B08 + B04)
# NDWI: (B03 - B08)/(B03 + B08)
# NORM: sqrt(B02^2 + B03^2 + B04^2 + B08^2 + B11^2 + B12^2)
ndvi = NormalizedDifferenceIndex("NDVI", "BANDS/3", "BANDS/2")
ndwi = NormalizedDifferenceIndex("NDWI", "BANDS/1", "BANDS/3")
norm = EuclideanNorm("NORM", "BANDS")
# TASK FOR VALID MASK
# validate pixels using SentinelHub's cloud detection mask and region of acquisition
add_sh_valmask = AddValidDataMaskTask(
SentinelHubValidData(),
"IS_VALID" # name of output mask
)
# TASK FOR COUNTING VALID PIXELS
# count number of valid observations per pixel using valid data mask
count_val_sh = CountValid(
"IS_VALID",
"VALID_COUNT" # name of existing mask # name of output scalar
)
# TASK FOR SAVING TO OUTPUT (if needed)
path_save = Path(path_save)
path_save.mkdir(exist_ok=True)
# if not os.path.isdir(path_save):
# os.makedirs(path_save)
save = SaveToDisk(path_save,
overwrite_permission=OverwritePermission.OVERWRITE_PATCH)
# Define the workflow
workflow = LinearWorkflow(add_data, add_clm, ndvi, ndwi, norm,
add_sh_valmask, count_val_sh, save)
# Execute the workflow
# time interval for the SH request
# TODO: need to check if specified time interval is valid
time_interval = [s_date, e_date]
# define additional parameters of the workflow
execution_args = []
path_EOPatch = path_save / f"eopatch_{patch_n}"
execution_args.append({
add_data: {
"bbox":
BBox(
((lon_left_top, lat_left_top), (lon_right_bot, lat_right_bot)),
crs=CRS.WGS84,
),
"time_interval":
time_interval,
},
save: {
"eopatch_folder": path_EOPatch.stem
},
})
executor = EOExecutor(workflow, execution_args, save_logs=True)
if debug:
print("Downloading Satellite data ...")
executor.run(workers=2, multiprocess=False)
if executor.get_failed_executions():
raise RuntimeError("EOExecutor failed in finishing tasks!")
if debug:
executor.make_report()
if debug:
print("Satellite data is downloaded")
return path_EOPatch
def get_add_l2a_data_workflow(data):
"""
Creates an workflow that:
1. loads existing EOPatch
2. adds sen2cor scene classification map
3. adds L2A data (all 12 bands)
4. adds s2cloudless cloud masks
5. determines `L2A_VALID` - map of valid observations (t,h,w,1) based on L2A SCL map
* pixels are marked as valid, if they're tagged as
`[DARK_AREA_PIXELS, VEGETATION, NOT_VEGETATED, WATER, UNCLASSIFIED]`
* performs opening with disk with radius 11 on `L2A_VALID`
6. determines `L1C_VALID` - map of valid observations (t,h,w,1) based on s2cloudless cloud map
* pixels are marked as valid, if they're tagged as not cloud
7. saves EOPatch to disk
"""
# 1. loads existing EOPatch
load = LoadFromDisk(str(data))
# 2. add L2A
add_l2a = S2L2AWCSInput(layer='BANDS-S2-L2A',
resx='10m',
resy='10m',
maxcc=0.8,
time_difference=timedelta(hours=2),
raise_download_errors=False)
# 3. add sen2cor's scene classification map and snow probability map
add_scl = AddSen2CorClassificationFeature(sen2cor_classification='SCL',
layer='TRUE-COLOR-S2-L2A',
image_format=MimeType.TIFF_d32f,
raise_download_errors=False)
# 4. add s2cloudless cloud mask
cloud_classifier = get_s2_pixel_cloud_detector(average_over=2,
dilation_size=1,
all_bands=False)
add_clm = AddCloudMaskTask(cloud_classifier,
'BANDS-S2CLOUDLESS',
cm_size_y='160m',
cm_size_x='160m',
cmask_feature='CLM')
# create valid data masks
scl_valid_classes = [2, 4, 5, 6, 7]
# 5. and 6. add L2A and L1C valid data masks
add_l1c_valmask = AddValidDataMaskTask(SentinelHubValidData(), 'L1C_VALID')
add_l2a_valmask = AddValidDataMaskTask(
Sen2CorValidData(scl_valid_classes, 6, 22), 'L2A_VALID')
add_valmask = AddValidDataMaskTask(MergeMasks('L1C_VALID', 'L2A_VALID'),
'VALID_DATA')
# 3. keep only frames with valid data fraction over 70%
valid_data_predicate = ValidDataFractionPredicate(0.7)
filter_task = SimpleFilterTask((FeatureType.MASK, 'VALID_DATA'),
valid_data_predicate)
# save
save = SaveToDisk(str(data),
compress_level=1,
overwrite_permission=OverwritePermission.OVERWRITE_PATCH)
workflow = LinearWorkflow(load,
add_l2a,
add_scl,
add_clm,
add_l1c_valmask,
add_l2a_valmask,
add_valmask,
filter_task,
save,
task_names={
load: 'load',
add_l2a: 'add_L2A',
add_scl: 'add_SCL',
add_clm: 'add_clm',
add_l1c_valmask: 'add_L1C_valmask',
add_l2a_valmask: 'add_L2A_valmask',
add_valmask: 'add_valmask',
filter_task: ' filter_task',
save: 'save'
})
return workflow