def test_execution_errors(multiprocess, workflow, execution_kwargs): with tempfile.TemporaryDirectory() as tmp_dir_name: executor = EOExecutor(workflow, execution_kwargs, logs_folder=tmp_dir_name) executor.run(workers=5, multiprocess=multiprocess) for idx, results in enumerate(executor.execution_results): if idx == 3: assert results.workflow_failed() else: assert not results.workflow_failed() assert executor.get_successful_executions() == [0, 1, 2] assert executor.get_failed_executions() == [3]
def test_execution_errors(self): for multiprocess in [True, False]: with tempfile.TemporaryDirectory() as tmp_dir_name: executor = EOExecutor(self.workflow, self.execution_args, logs_folder=tmp_dir_name) executor.run(workers=5, multiprocess=multiprocess) for idx, stats in enumerate(executor.execution_stats): if idx != 3: self.assertFalse('error' in stats, 'Workflow {} should be executed without errors'.format(idx)) else: self.assertTrue('error' in stats and stats['error'], 'This workflow should be executed with an error') self.assertEqual(executor.get_successful_executions(), [0, 1, 2]) self.assertEqual(executor.get_failed_executions(), [3])
def download_data(path_save, coords_top, coords_bot, patch_n, s_date, e_date, debug=False): # before moving onto actual tasks, check setup check_sentinel_cfg() [lat_left_top, lon_left_top] = coords_top [lat_right_bot, lon_right_bot] = coords_bot # TASK FOR BAND DATA # add a request for B(B02), G(B03), R(B04), NIR (B08), SWIR1(B11), SWIR2(B12) # from default layer 'ALL_BANDS' at 10m resolution # Here we also do a simple filter of cloudy scenes. A detailed cloud cover # detection is performed in the next step custom_script = "return [B02, B03, B04, B08, B11, B12];" add_data = S2L1CWCSInput( layer="BANDS-S2-L1C", feature=(FeatureType.DATA, "BANDS"), # save under name 'BANDS' # custom url for 6 specific bands custom_url_params={CustomUrlParam.EVALSCRIPT: custom_script}, resx="10m", # resolution x resy="10m", # resolution y maxcc=0.1, # maximum allowed cloud cover of original ESA tiles ) # TASK FOR CLOUD INFO # cloud detection is performed at 80m resolution # and the resulting cloud probability map and mask # are scaled to EOPatch's resolution cloud_classifier = get_s2_pixel_cloud_detector(average_over=2, dilation_size=1, all_bands=False) add_clm = AddCloudMaskTask( cloud_classifier, "BANDS-S2CLOUDLESS", cm_size_y="80m", cm_size_x="80m", cmask_feature="CLM", # cloud mask name cprobs_feature="CLP", # cloud prob. map name ) # TASKS FOR CALCULATING NEW FEATURES # NDVI: (B08 - B04)/(B08 + B04) # NDWI: (B03 - B08)/(B03 + B08) # NORM: sqrt(B02^2 + B03^2 + B04^2 + B08^2 + B11^2 + B12^2) ndvi = NormalizedDifferenceIndex("NDVI", "BANDS/3", "BANDS/2") ndwi = NormalizedDifferenceIndex("NDWI", "BANDS/1", "BANDS/3") norm = EuclideanNorm("NORM", "BANDS") # TASK FOR VALID MASK # validate pixels using SentinelHub's cloud detection mask and region of acquisition add_sh_valmask = AddValidDataMaskTask( SentinelHubValidData(), "IS_VALID" # name of output mask ) # TASK FOR COUNTING VALID PIXELS # count number of valid observations per pixel using valid data mask count_val_sh = CountValid( "IS_VALID", "VALID_COUNT" # name of existing mask # name of output scalar ) # TASK FOR SAVING TO OUTPUT (if needed) path_save = Path(path_save) path_save.mkdir(exist_ok=True) # if not os.path.isdir(path_save): # os.makedirs(path_save) save = SaveToDisk(path_save, overwrite_permission=OverwritePermission.OVERWRITE_PATCH) # Define the workflow workflow = LinearWorkflow(add_data, add_clm, ndvi, ndwi, norm, add_sh_valmask, count_val_sh, save) # Execute the workflow # time interval for the SH request # TODO: need to check if specified time interval is valid time_interval = [s_date, e_date] # define additional parameters of the workflow execution_args = [] path_EOPatch = path_save / f"eopatch_{patch_n}" execution_args.append({ add_data: { "bbox": BBox( ((lon_left_top, lat_left_top), (lon_right_bot, lat_right_bot)), crs=CRS.WGS84, ), "time_interval": time_interval, }, save: { "eopatch_folder": path_EOPatch.stem }, }) executor = EOExecutor(workflow, execution_args, save_logs=True) if debug: print("Downloading Satellite data ...") executor.run(workers=2, multiprocess=False) if executor.get_failed_executions(): raise RuntimeError("EOExecutor failed in finishing tasks!") if debug: executor.make_report() if debug: print("Satellite data is downloaded") return path_EOPatch