def test_cloud_coverage(self): classifier = get_s2_pixel_cloud_detector(all_bands=True) # Classifier is run on same resolution as data array add_cm = AddCloudMaskTask(classifier, 'ALL_BANDS', cmask_feature='clm', cprobs_feature='clp') eop_clm = add_cm(self.eop) _, h, w, _ = eop_clm.mask['clm'].shape cc = np.sum(eop_clm.mask['clm'][0]) / (w * h) ps = np.sum(eop_clm.data['clp'][0]) / (w * h) self.assertTrue(eop_clm.mask['clm'].ndim == 4) self.assertAlmostEqual(cc, 0.687936507936508, places=4) self.assertAlmostEqual(ps, 0.521114510213301, places=4) del add_cm, eop_clm # Classifier is run on downscaled version of data array add_cm = AddCloudMaskTask(classifier, 'ALL_BANDS', cm_size_y=50, cmask_feature='clm', cprobs_feature='clp') eop_clm = add_cm(self.eop) _, h, w, _ = eop_clm.mask['clm'].shape cc = np.sum(eop_clm.mask['clm'][0]) / (w * h) ps = np.sum(eop_clm.data['clp'][0]) / (w * h) self.assertTrue(eop_clm.mask['clm'].ndim == 4) self.assertAlmostEqual(cc, 0.710357142857142, places=4) self.assertAlmostEqual(ps, 0.500692345333859, places=4)
def test_cloud_coverage(self): classifier = get_s2_pixel_cloud_detector(all_bands=True) # Classifier is run on same resolution as data array add_cm = AddCloudMaskTask(classifier, 'ALL_DATA', cmask_feature='CLM_TEST', cprobs_feature='CLP_TEST') eop_clm = add_cm(self.eop) # Check shape and type self._check_shape(eop_clm.mask['CLM_TEST'], eop_clm.data['ALL_DATA']) self._check_shape(eop_clm.data['CLP_TEST'], eop_clm.data['ALL_DATA']) self.assertTrue(eop_clm.mask['CLM_TEST'].dtype == np.bool) self.assertTrue(eop_clm.data['CLP_TEST'].dtype == np.float32) # Compare mean cloud coverage with provided reference mean_clm_provided = np.mean(eop_clm.mask['CLM']) mean_clp_provided = np.mean(eop_clm.data['CLP']) self.assertAlmostEqual(np.mean(eop_clm.mask['CLM_TEST']), mean_clm_provided, places=1) self.assertAlmostEqual(np.mean(eop_clm.data['CLP_TEST']), mean_clp_provided, places=2) # Classifier is run on downscaled version of data array add_cm = AddCloudMaskTask(classifier, 'ALL_DATA', cm_size_y='20m', cm_size_x='20m', cmask_feature='CLM_TEST', cprobs_feature='CLP_TEST') eop_clm = add_cm(self.eop) # Check shape and type self._check_shape(eop_clm.mask['CLM_TEST'], eop_clm.data['ALL_DATA']) self._check_shape(eop_clm.data['CLP_TEST'], eop_clm.data['ALL_DATA']) self.assertTrue(eop_clm.mask['CLM_TEST'].dtype == np.bool) self.assertTrue(eop_clm.data['CLP_TEST'].dtype == np.float32) # Compare mean cloud coverage with provided reference mean_clm_provided = np.mean(eop_clm.mask['CLM']) mean_clp_provided = np.mean(eop_clm.data['CLP']) self.assertAlmostEqual(np.mean(eop_clm.mask['CLM_TEST']), mean_clm_provided, places=2) self.assertAlmostEqual(np.mean(eop_clm.data['CLP_TEST']), mean_clp_provided, places=2) # Check if same times are flagged as cloudless cloudless = np.mean(eop_clm.mask['CLM_TEST'], axis=(1, 2, 3)) == 0 self.assertTrue( np.all(cloudless == eop_clm.label['IS_CLOUDLESS'][:, 0]))
def test_wms_request(self): classifier = get_s2_pixel_cloud_detector(all_bands=False) # Classifier is run on new request of data array add_cm = AddCloudMaskTask(classifier, 'BANDS-S2-L1C', cm_size_x='20m', cm_size_y='20m', cmask_feature='CLM_TEST', cprobs_feature='CLP_TEST') eop_clm = add_cm(self.eop) # Check shape and type self._check_shape(eop_clm.mask['CLM_TEST'], eop_clm.data['ALL_DATA']) self._check_shape(eop_clm.data['CLP_TEST'], eop_clm.data['ALL_DATA']) self.assertTrue(eop_clm.mask['CLM_TEST'].dtype == np.bool) self.assertTrue(eop_clm.data['CLP_TEST'].dtype == np.float32) # Compare mean cloud coverage with provided reference mean_clm_provided = np.mean(eop_clm.mask['CLM']) mean_clp_provided = np.mean(eop_clm.data['CLP']) self.assertAlmostEqual(np.mean(eop_clm.mask['CLM_TEST']), mean_clm_provided, places=2) self.assertAlmostEqual(np.mean(eop_clm.data['CLP_TEST']), mean_clp_provided, places=2)
def gather_data(): cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) add_clm = AddCloudMaskTask(cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='80m', cm_size_x='80m', cmask_feature='CLM', # cloud mask name cprobs_feature='CLP' # cloud prob. map name ) ndvi = NormalizedDifferenceIndex('NDVI', 'BANDS/3', 'BANDS/2') add_sh_valmask = AddValidDataMaskTask(SentinelHubValidData(), 'IS_VALID') layer = 'BANDS-S2-L1C' custom_script = 'return [B02, B03];' input_task = S2L1CWCSInput(layer=layer, feature=(FeatureType.DATA, 'BANDS'), custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script}, resx='10m', resy='10m', maxcc=.8) add_ndvi = S2L1CWCSInput(layer='NDVI') save = SaveToDisk('io_example', overwrite_permission=2)#compress_level=1 workflow = LinearWorkflow( input_task, add_clm, add_ndvi, add_sh_valmask, ) time_interval = ('2017-01-01', '2017-12-31') result = workflow.execute({input_task: {'bbox': roi_bbox, 'time_interval': time_interval}, save: {'eopatch_folder': 'eopatch'}}) return list(result.values())[0].data['NDVI'], list(result.values())[-1].mask['IS_VALID'], np.array(list(result.values())[0].timestamp)
def test_wms_request(self): classifier = get_s2_pixel_cloud_detector(all_bands=False) # Classifier is run on new request of data array add_cm = AddCloudMaskTask(classifier, 'BANDS-S2-L1C', cm_size_y=50, cmask_feature='clm', cprobs_feature='clp') eop_clm = add_cm(self.eop) _, h, w, _ = eop_clm.mask['clm'].shape cc = np.sum(eop_clm.mask['clm'][0]) / (w * h) ps = np.sum(eop_clm.data['clp'][0]) / (w * h) self.assertTrue(eop_clm.mask['clm'].ndim == 4) self.assertAlmostEqual(cc, 0.737738, places=4) self.assertAlmostEqual(ps, 0.520182, places=4)
def cloud_classifier_task(): '''A convenience function that sets up the cloud detection task. Configures an instance of the EOTask s2_pixel_cloud_detector and AddCloudMaskTask ''' cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) cloud_detection = AddCloudMaskTask(cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='40m', cm_size_x='40m', cmask_feature='CLM', cprobs_feature='CLP') return cloud_detection
resx='10m', # resolution x resy='10m', # resolution y maxcc=0.8, # maximum allowed cloud cover of original ESA tiles ) # TASK FOR CLOUD INFO # cloud detection is performed at 80m resolution # and the resulting cloud probability map and mask # are scaled to EOPatch's resolution cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) add_clm = AddCloudMaskTask( cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='80m', cm_size_x='80m', cmask_feature='CLM', # cloud mask name cprobs_feature='CLP' # cloud prob. map name ) # TASKS FOR CALCULATING NEW FEATURES # NDVI: (B08 - B04)/(B08 + B04) # NDWI: (B03 - B08)/(B03 + B08) # NORM: sqrt(B02^2 + B03^2 + B04^2 + B08^2 + B11^2 + B12^2) ndvi = NormalizedDifferenceIndex('NDVI', 'BANDS/3', 'BANDS/2') ndwi = NormalizedDifferenceIndex('NDWI', 'BANDS/1', 'BANDS/3') norm = EuclideanNorm('NORM', 'BANDS') # TASK FOR VALID MASK # validate pixels using SentinelHub's cloud detection mask and region of acquisition add_sh_valmask = AddValidDataMaskTask(
instance_id=WMS_INSTANCE) add_ndwi = S2L1CWCSInput('NDWI', instance_id=WMS_INSTANCE) gdf = gpd.GeoDataFrame(crs={'init': 'epsg:4326'}, geometry=[dam_nominal]) gdf.plot() add_nominal_water = VectorToRaster( (FeatureType.MASK_TIMELESS, 'NOMINAL_WATER'), gdf, 1, (FeatureType.MASK, 'IS_DATA'), np.uint8) cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) cloud_det = AddCloudMaskTask(cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='60m', cm_size_x='60m', cmask_feature='CLM', cprobs_feature='CLP', instance_id=WMS_INSTANCE) class ValidDataPredicate: def __call__(self, eopatch): return np.logical_and( eopatch.mask['IS_DATA'].astype(np.bool), np.logical_not(eopatch.mask['CLM'].astype(np.bool))) add_valmask = AddValidDataMaskTask(predicate=ValidDataPredicate())
def test_raises_errors(self): classifier = get_s2_pixel_cloud_detector(all_bands=True) add_cm = AddCloudMaskTask(classifier, 'bands', cmask_feature='clm') self.assertRaises(ValueError, add_cm, self.eop)
def download_data(path_save, coords_top, coords_bot, patch_n, s_date, e_date, debug=False): # before moving onto actual tasks, check setup check_sentinel_cfg() [lat_left_top, lon_left_top] = coords_top [lat_right_bot, lon_right_bot] = coords_bot # TASK FOR BAND DATA # add a request for B(B02), G(B03), R(B04), NIR (B08), SWIR1(B11), SWIR2(B12) # from default layer 'ALL_BANDS' at 10m resolution # Here we also do a simple filter of cloudy scenes. A detailed cloud cover # detection is performed in the next step custom_script = "return [B02, B03, B04, B08, B11, B12];" add_data = S2L1CWCSInput( layer="BANDS-S2-L1C", feature=(FeatureType.DATA, "BANDS"), # save under name 'BANDS' # custom url for 6 specific bands custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script}, resx="10m", # resolution x resy="10m", # resolution y maxcc=0.1, # maximum allowed cloud cover of original ESA tiles ) # TASK FOR CLOUD INFO # cloud detection is performed at 80m resolution # and the resulting cloud probability map and mask # are scaled to EOPatch's resolution cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) add_clm = AddCloudMaskTask( cloud_classifier, "BANDS-S2CLOUDLESS", cm_size_y="80m", cm_size_x="80m", cmask_feature="CLM", # cloud mask name cprobs_feature="CLP", # cloud prob. map name ) # TASKS FOR CALCULATING NEW FEATURES # NDVI: (B08 - B04)/(B08 + B04) # NDWI: (B03 - B08)/(B03 + B08) # NORM: sqrt(B02^2 + B03^2 + B04^2 + B08^2 + B11^2 + B12^2) ndvi = NormalizedDifferenceIndex("NDVI", "BANDS/3", "BANDS/2") ndwi = NormalizedDifferenceIndex("NDWI", "BANDS/1", "BANDS/3") norm = EuclideanNorm("NORM", "BANDS") # TASK FOR VALID MASK # validate pixels using SentinelHub's cloud detection mask and region of acquisition add_sh_valmask = AddValidDataMaskTask( SentinelHubValidData(), "IS_VALID" # name of output mask ) # TASK FOR COUNTING VALID PIXELS # count number of valid observations per pixel using valid data mask count_val_sh = CountValid( "IS_VALID", "VALID_COUNT" # name of existing mask # name of output scalar ) # TASK FOR SAVING TO OUTPUT (if needed) path_save = Path(path_save) path_save.mkdir(exist_ok=True) # if not os.path.isdir(path_save): # os.makedirs(path_save) save = SaveToDisk(path_save, overwrite_permission=OverwritePermission.OVERWRITE_PATCH) # Define the workflow workflow = LinearWorkflow(add_data, add_clm, ndvi, ndwi, norm, add_sh_valmask, count_val_sh, save) # Execute the workflow # time interval for the SH request # TODO: need to check if specified time interval is valid time_interval = [s_date, e_date] # define additional parameters of the workflow execution_args = [] path_EOPatch = path_save / f"eopatch_{patch_n}" execution_args.append({ add_data: { "bbox": BBox( ((lon_left_top, lat_left_top), (lon_right_bot, lat_right_bot)), crs=CRS.WGS84, ), "time_interval": time_interval, }, save: { "eopatch_folder": path_EOPatch.stem }, }) executor = EOExecutor(workflow, execution_args, save_logs=True) if debug: print("Downloading Satellite data ...") executor.run(workers=2, multiprocess=False) if executor.get_failed_executions(): raise RuntimeError("EOExecutor failed in finishing tasks!") if debug: executor.make_report() if debug: print("Satellite data is downloaded") return path_EOPatch
def get_add_l2a_data_workflow(data): """ Creates an workflow that: 1. loads existing EOPatch 2. adds sen2cor scene classification map 3. adds L2A data (all 12 bands) 4. adds s2cloudless cloud masks 5. determines `L2A_VALID` - map of valid observations (t,h,w,1) based on L2A SCL map * pixels are marked as valid, if they're tagged as `[DARK_AREA_PIXELS, VEGETATION, NOT_VEGETATED, WATER, UNCLASSIFIED]` * performs opening with disk with radius 11 on `L2A_VALID` 6. determines `L1C_VALID` - map of valid observations (t,h,w,1) based on s2cloudless cloud map * pixels are marked as valid, if they're tagged as not cloud 7. saves EOPatch to disk """ # 1. loads existing EOPatch load = LoadFromDisk(str(data)) # 2. add L2A add_l2a = S2L2AWCSInput(layer='BANDS-S2-L2A', resx='10m', resy='10m', maxcc=0.8, time_difference=timedelta(hours=2), raise_download_errors=False) # 3. add sen2cor's scene classification map and snow probability map add_scl = AddSen2CorClassificationFeature(sen2cor_classification='SCL', layer='TRUE-COLOR-S2-L2A', image_format=MimeType.TIFF_d32f, raise_download_errors=False) # 4. add s2cloudless cloud mask cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) add_clm = AddCloudMaskTask(cloud_classifier, 'BANDS-S2CLOUDLESS', cm_size_y='160m', cm_size_x='160m', cmask_feature='CLM') # create valid data masks scl_valid_classes = [2, 4, 5, 6, 7] # 5. and 6. add L2A and L1C valid data masks add_l1c_valmask = AddValidDataMaskTask(SentinelHubValidData(), 'L1C_VALID') add_l2a_valmask = AddValidDataMaskTask( Sen2CorValidData(scl_valid_classes, 6, 22), 'L2A_VALID') add_valmask = AddValidDataMaskTask(MergeMasks('L1C_VALID', 'L2A_VALID'), 'VALID_DATA') # 3. keep only frames with valid data fraction over 70% valid_data_predicate = ValidDataFractionPredicate(0.7) filter_task = SimpleFilterTask((FeatureType.MASK, 'VALID_DATA'), valid_data_predicate) # save save = SaveToDisk(str(data), compress_level=1, overwrite_permission=OverwritePermission.OVERWRITE_PATCH) workflow = LinearWorkflow(load, add_l2a, add_scl, add_clm, add_l1c_valmask, add_l2a_valmask, add_valmask, filter_task, save, task_names={ load: 'load', add_l2a: 'add_L2A', add_scl: 'add_SCL', add_clm: 'add_clm', add_l1c_valmask: 'add_L1C_valmask', add_l2a_valmask: 'add_L2A_valmask', add_valmask: 'add_valmask', filter_task: ' filter_task', save: 'save' }) return workflow