def test_evaluate_predefined_property(backend_implementation): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "TERRASCOPE_S2_FAPAR_V2"}, "result": True}, } env = EvalEnv(dict(backend_implementation=backend_implementation)) evaluate(pg, do_dry_run=True, env=env)
def test_CropSAR_aggregate_spatial_constraint(dry_run_env, dry_run_tracer): cropsar_process = load_json("pg/1.0/cropsar_graph.json") custom_process_from_process_graph(cropsar_process, namespace="test") try: geometry_path = str(get_path("geojson/thaipolys_ad.geojson")) pg = { "CropSAR1": { "process_id": "CropSAR", "arguments": { "file_polygons": geometry_path, "time_range": [ "2019-07-01", "2019-08-31" ] }, "namespace": "test", "result": True } } evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) > 0 for _, constraints in source_constraints: assert constraints['aggregate_spatial']['geometries'].path == geometry_path finally: del process_registry_100._processes['test', 'CropSAR']
def test_aggregate_spatial_read_vector(dry_run_env, dry_run_tracer): geometry_path = str(get_path("geojson/GeometryCollection01.json")) pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "vector": {"process_id": "read_vector", "arguments": {"filename": geometry_path}}, "agg": { "process_id": "aggregate_spatial", "arguments": { "data": {"from_node": "lc"}, "geometries": {"from_node": "vector"}, "reducer": { "process_graph": { "mean": { "process_id": "mean", "arguments": {"data": {"from_parameter": "data"}}, "result": True } } } }, "result": True, }, } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "spatial_extent": {"west": 5.05, "south": 51.21, "east": 5.15, "north": 51.3, "crs": "EPSG:4326"}, "aggregate_spatial": {"geometries": DelayedVector(geometry_path)}, } geometries, = dry_run_tracer.get_geometries() assert isinstance(geometries, DelayedVector)
def test_evaluate_basic_no_load_collection(dry_run_env, dry_run_tracer): pg = { "add": {"process_id": "add", "arguments": {"x": 1, "y": 2}, "result": True}, } res = evaluate(pg, env=dry_run_env) assert res == 3 source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert source_constraints == []
def point(): # TODO: deprecated? do we still need this endpoint? #35 x = float(request.args.get('x', '')) y = float(request.args.get('y', '')) srs = request.args.get('srs', None) process_graph = _extract_process_graph(request.json) image_collection = evaluate(process_graph, viewingParameters={'version': g.api_version}) return jsonify(image_collection.timeseries(x, y, srs))
def test_evaluate_merge_collections(dry_run_env, dry_run_tracer): pg = { "load": { "process_id": "load_collection", "arguments": { "id": "S2_FOOBAR", "spatial_extent": {"west": 0, "south": 50, "east": 5, "north": 55}, "temporal_extent": ["2020-01-01", "2020-10-10"], "bands": ["red", "green", "blue"] }, }, "load_s1": { "process_id": "load_collection", "arguments": { "id": "S2_FAPAR_CLOUDCOVER", "spatial_extent": {"west": -1, "south": 50, "east": 5, "north": 55}, "temporal_extent": ["2020-01-01", "2020-10-10"], "bands": ["VV"] }, }, "merge": { "process_id": "merge_cubes", "arguments": { "cube1": {"from_node": "load"}, "cube2": {"from_node": "load_s1"} }, "result": True } } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 2 source, constraints = source_constraints[0] assert source == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "temporal_extent": ("2020-01-01", "2020-10-10"), "spatial_extent": {"west": 0, "south": 50, "east": 5, "north": 55, "crs": "EPSG:4326"}, "bands": ["red", "green", "blue"], } source, constraints = source_constraints[1] assert source == ("load_collection", ("S2_FAPAR_CLOUDCOVER", ())) assert constraints == { "temporal_extent": ("2020-01-01", "2020-10-10"), "spatial_extent": {"west": -1, "south": 50, "east": 5, "north": 55, "crs": "EPSG:4326"}, "bands": ["VV"], } dry_run_env = dry_run_env.push({ENV_SOURCE_CONSTRAINTS: source_constraints}) loadparams = _extract_load_parameters(dry_run_env, ("load_collection", ("S2_FOOBAR", ()))) assert {"west": -1, "south": 50, "east": 5, "north": 55, "crs": "EPSG:4326"} == loadparams.global_extent
def test_aggregate_spatial_get_geometries_feature_collection(dry_run_env, dry_run_tracer): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "vector": {"process_id": "get_geometries", "arguments": {"feature_collection": { "type": "FeatureCollection", "name": "fields", "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, "features": [ { "type": "Feature", "geometry": { "type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]] }, "properties": { "CODE_OBJ": "0000000000000001" } } ] }}}, "agg": { "process_id": "aggregate_spatial", "arguments": { "data": {"from_node": "lc"}, "geometries": {"from_node": "vector"}, "reducer": { "process_graph": { "mean": { "process_id": "mean", "arguments": {"data": {"from_parameter": "data"}}, "result": True } } } }, "result": True, }, } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) expected_geometry_collection = shapely.geometry.GeometryCollection( [shapely.geometry.shape({"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]})] ) assert constraints == { "spatial_extent": {'west': 0.0, 'south': 0.0, 'east': 8.0, 'north': 5.0, 'crs': 'EPSG:4326'}, "aggregate_spatial": {"geometries": expected_geometry_collection} } geometries, = dry_run_tracer.get_geometries() assert isinstance(geometries, shapely.geometry.GeometryCollection)
def test_evaluate_basic_load_collection(dry_run_env, dry_run_tracer): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}, "result": True}, } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert source_constraints == [ (("load_collection", ("S2_FOOBAR", ())), {}) ]
def test_sources_are_subject_to_correct_constraints(dry_run_env, dry_run_tracer): pg = { 'loadcollection1': {'process_id': 'load_collection', 'arguments': {'bands': ['VV', 'VH'], 'id': 'S2_FOOBAR', 'spatial_extent': {'west': 11.465226, 'east': 11.465435, 'south': 46.343118, 'north': 46.343281, 'crs': 'EPSG:4326'}, 'temporal_extent': ['2018-01-01', '2018-01-01']}}, 'sarbackscatter1': {'process_id': 'sar_backscatter', 'arguments': {'coefficient': 'sigma0-ellipsoid', 'contributing_area': False, 'data': {'from_node': 'loadcollection1'}, 'elevation_model': None, 'ellipsoid_incidence_angle': False, 'local_incidence_angle': False, 'mask': False, 'noise_removal': True}}, 'renamelabels1': {'process_id': 'rename_labels', 'arguments': {'data': {'from_node': 'sarbackscatter1'}, 'dimension': 'bands', 'source': ['VV', 'VH'], 'target': ['VV_sigma0', 'VH_sigma0']}}, 'sarbackscatter2': {'process_id': 'sar_backscatter', 'arguments': {'coefficient': 'gamma0-terrain', 'contributing_area': False, 'data': {'from_node': 'loadcollection1'}, 'elevation_model': None, 'ellipsoid_incidence_angle': False, 'local_incidence_angle': False, 'mask': False, 'noise_removal': True}}, 'renamelabels2': {'process_id': 'rename_labels', 'arguments': {'data': {'from_node': 'sarbackscatter2'}, 'dimension': 'bands', 'source': ['VV', 'VH'], 'target': ['VV_gamma0', 'VH_gamma0']}}, 'mergecubes1': {'process_id': 'merge_cubes', 'arguments': {'cube1': {'from_node': 'renamelabels1'}, 'cube2': {'from_node': 'renamelabels2'}}, 'result': True} } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 2 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "temporal_extent": ('2018-01-01', '2018-01-01'), "spatial_extent": {'west': 11.465226, 'east': 11.465435, 'south': 46.343118, 'north': 46.343281, 'crs': 'EPSG:4326'}, 'bands': ['VV', 'VH'], 'sar_backscatter': SarBackscatterArgs(coefficient='sigma0-ellipsoid', elevation_model=None, mask=False, contributing_area=False, local_incidence_angle=False, ellipsoid_incidence_angle=False, noise_removal=True, options={}) } src, constraints = source_constraints[1] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "temporal_extent": ('2018-01-01', '2018-01-01'), "spatial_extent": {'west': 11.465226, 'east': 11.465435, 'south': 46.343118, 'north': 46.343281, 'crs': 'EPSG:4326'}, 'bands': ['VV', 'VH'], 'sar_backscatter': SarBackscatterArgs(coefficient='gamma0-terrain', elevation_model=None, mask=False, contributing_area=False, local_incidence_angle=False, ellipsoid_incidence_angle=False, noise_removal=True, options={}) }
def test_evaluate_load_collection_and_filter_extents(dry_run_env, dry_run_tracer): """temporal/bbox/band extents in load_collection *and* filter_ processes""" pg = { "load": { "process_id": "load_collection", "arguments": { "id": "S2_FOOBAR", "spatial_extent": {"west": 0, "south": 50, "east": 5, "north": 55}, "temporal_extent": ["2020-01-01", "2020-10-10"], "bands": ["red", "green", "blue"] }, }, "filter_temporal": { "process_id": "filter_temporal", "arguments": {"data": {"from_node": "load"}, "extent": ["2020-02-02", "2020-03-03"]}, }, "filter_bbox": { "process_id": "filter_bbox", "arguments": { "data": {"from_node": "filter_temporal"}, "extent": {"west": 1, "south": 51, "east": 3, "north": 53} }, }, "filter_bands": { "process_id": "filter_bands", "arguments": {"data": {"from_node": "filter_bbox"}, "bands": ["red"]}, "result": True, } } cube = evaluate(pg, env=dry_run_env) """ source_constraints = dry_run_tracer.get_source_constraints(merge=False) assert len(source_constraints) == 1 src, constraints = source_constraints.popitem() assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == [{ "temporal_extent": [("2020-01-01", "2020-10-10"), ("2020-02-02", "2020-03-03"), ], "spatial_extent": [ {"west": 0, "south": 50, "east": 5, "north": 55, "crs": "EPSG:4326"}, {"west": 1, "south": 51, "east": 3, "north": 53, "crs": "EPSG:4326"} ], "bands": [["red", "green", "blue"], ["red"]], }] """ source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "temporal_extent": ("2020-01-01", "2020-10-10"), "spatial_extent": {"west": 0, "south": 50, "east": 5, "north": 55, "crs": "EPSG:4326"}, "bands": ["red", "green", "blue"], }
def test_evaluate_drop_dimension(dry_run_env, dry_run_tracer, dimension_name, expected): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "drop": { "process_id": "drop_dimension", "arguments": {"data": {"from_node": "lc"}, "name": dimension_name}, "result": True, }, } cube = evaluate(pg, env=dry_run_env) assert cube.metadata.dimension_names() == expected
def download(): # TODO: deprecated? if request.method == 'POST': outputformat = request.args.get('outputformat', 'geotiff') process_graph = request.get_json() image_collection = evaluate(process_graph) # TODO Unify with execute? filename = image_collection.download(None,outputformat=outputformat) return send_from_directory(os.path.dirname(filename),os.path.basename(filename)) else: return 'Usage: Download image using POST.'
def test_evaluate_graph_diamond(dry_run_env, dry_run_tracer): """ Diamond graph: load -> band red -> mask -> bbox `-> band grass -^ """ pg = { "load": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "band_red": { "process_id": "filter_bands", "arguments": {"data": {"from_node": "load"}, "bands": ["red"]}, }, "band_grass": { "process_id": "filter_bands", "arguments": {"data": {"from_node": "load"}, "bands": ["grass"]}, }, "mask": { "process_id": "mask", "arguments": {"data": {"from_node": "band_red"}, "mask": {"from_node": "band_grass"}}, }, "bbox": { "process_id": "filter_bbox", "arguments": {"data": {"from_node": "mask"}, "extent": {"west": 1, "east": 2, "south": 51, "north": 52}}, "result": True, } } cube = evaluate(pg, env=dry_run_env) """ source_constraints = dry_run_tracer.get_source_constraints(merge=False) assert len(source_constraints) == 1 src, constraints = source_constraints.popitem() assert src == ("load_collection", ("S2_FOOBAR", ())) assert sorted(constraints, key=str) == [ """ source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 2 assert source_constraints == [( ("load_collection", ("S2_FOOBAR", ())), { "bands": ["grass"], "spatial_extent": {"west": 1, "east": 2, "south": 51, "north": 52, "crs": "EPSG:4326"} }), ( ("load_collection", ("S2_FOOBAR", ())), { "bands": ["red"], "spatial_extent": {"west": 1, "east": 2, "south": 51, "north": 52, "crs": "EPSG:4326"} }), ]
def test_evaluate_sar_backscatter(dry_run_env, dry_run_tracer, arguments, expected): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "sar": { "process_id": "sar_backscatter", "arguments": dict(data={"from_node": "lc"}, **arguments), "result": True, }, } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == {"sar_backscatter": expected}
def test_evaluate_atmospheric_correction(dry_run_env, dry_run_tracer, arguments, expected): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "sar": { "process_id": "atmospheric_correction", "arguments": dict(data={"from_node": "lc"}, **arguments), "result": True, }, } cube = evaluate(pg, env=dry_run_env) metadata_links = dry_run_tracer.get_metadata_links() assert len(metadata_links) == 1 src, links = metadata_links.popitem() assert src == ("load_collection", ("S2_FOOBAR", ())) assert links == expected
def test_evaluate_temporal_extent_dynamic_item(dry_run_env, dry_run_tracer): pg = { "load": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "start": {"process_id": "constant", "arguments": {"x": "2020-01-01"}}, "filtertemporal": { "process_id": "filter_temporal", "arguments": {"data": {"from_node": "load"}, "extent": [{"from_node": "start"}, "2020-02-02"]}, "result": True, }, } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints() assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == {"temporal_extent": ("2020-01-01", "2020-02-02")}
def test_mask_polygon_and_load_collection_spatial_extent(dry_run_env, dry_run_tracer): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} cube = DataCube(PGNode( "load_collection", id="S2_FOOBAR", spatial_extent={"west": -1, "south": -1, "east": 10, "north": 10} ), connection=None) cube = cube.mask_polygon(mask=polygon) pg = cube.flat_graph() res = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "spatial_extent": {"west": -1, "south": -1, "east": 10, "north": 10, "crs": "EPSG:4326"} }
def main(argv=None): logging.basicConfig(level=logging.INFO) process_graph, args = handle_cli(argv) _log.info( "Evaluating process graph: {pg}".format(pg=safe_repr(process_graph))) _setup_local_spark(print=_log.info) # Local imports to workaround the pyspark import issues. from openeo_driver.ProcessGraphDeserializer import evaluate from openeogeotrellis.backend import GeoPySparkBackendImplementation env = EvalEnv({ "version": args.api_version, "pyramid_levels": "highest", "user": None, # TODO "require_bounds": True, "correlation_id": f"cli-pid{os.getpid()}", "backend_implementation": GeoPySparkBackendImplementation(use_zookeeper=False), }) with TimingLogger(title="Evaluate process graph", logger=_log): result = evaluate(process_graph, env=env) if isinstance(result, ImageCollectionResult): filename = args.output or f"result.{result.format}" with TimingLogger(title=f"Saving result to {filename!r}", logger=_log): result.save_result(filename) elif isinstance(result, JSONResult): if args.output: with open(args.output, "w") as f: json.dump(result.prepare_for_json(), f) else: print(result.prepare_for_json()) elif isinstance(result, dict): # TODO: support storing JSON result to file print(result) else: # TODO: support more result types raise ValueError(result)
def execute(): # TODO: This is not an official endpoint, does this "/execute" still have to be exposed as route? post_data = request.get_json() process_graph = _extract_process_graph(post_data) result = evaluate(process_graph, viewingParameters={'version': g.api_version}) # TODO unify all this output handling within SaveResult logic? if isinstance(result, ImageCollection): format_options = post_data.get('output', {}) filename = result.download(None, bbox="", time="", **format_options) return send_from_directory(os.path.dirname(filename), os.path.basename(filename)) elif result is None: abort(500, "Process graph evaluation gave no result") elif isinstance(result, SaveResult): return result.create_flask_response() else: return jsonify(replace_nan_values(result))
def test_mask_polygon_only(dry_run_env, dry_run_tracer, inside, replacement, expect_spatial_extent): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} cube = DataCube(PGNode("load_collection", id="S2_FOOBAR"), connection=None) cube = cube.mask_polygon(mask=polygon, inside=inside, replacement=replacement) pg = cube.flat_graph() res = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) if expect_spatial_extent: expected = { "spatial_extent": {"west": 0.0, "south": 0.0, "east": 8.0, "north": 5.0, "crs": "EPSG:4326"} } else: expected = {} assert constraints == expected
def create_service(self, process_graph: dict, service_type: str, api_version: str, post_data: dict) -> ServiceMetadata: """ https://open-eo.github.io/openeo-api/apireference/#tag/Secondary-Services-Management/paths/~1services/post :return: (location, openeo_identifier) """ from openeo_driver.ProcessGraphDeserializer import evaluate # TODO require auth/user handle? if service_type.lower() not in set(st.lower() for st in self.service_types()): raise ServiceUnsupportedException( message="Secondary service type {t!r} is not supported.".format(t=service_type), ) image_collection = evaluate(process_graph, viewingParameters={'version': api_version}) service_metadata = image_collection.tiled_viewing_service( service_type=service_type, process_graph=process_graph, post_data=post_data ) return service_metadata
def test_mask_polygon_and_filter_bbox(dry_run_env, dry_run_tracer, bbox_first): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} bbox = {"west": -1, "south": -1, "east": 9, "north": 9, "crs": "EPSG:4326"} # Use client lib to build process graph in flexible way cube = DataCube(PGNode("load_collection", id="S2_FOOBAR"), connection=None) if bbox_first: cube = cube.filter_bbox(bbox=bbox).mask_polygon(mask=polygon) else: cube = cube.mask_polygon(mask=polygon).filter_bbox(bbox=bbox) pg = cube.flat_graph() res = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "spatial_extent": {"west": -1, "south": -1, "east": 9, "north": 9, "crs": "EPSG:4326"} }
def test_load_collection_properties(dry_run_env, dry_run_tracer): def get_props(direction="DESCENDING"): return { "orbitDirection": { "process_graph": { "od": { "process_id": "eq", "arguments": { "x": {"from_parameter": "value"}, "y": direction }, "result": True } } } } properties = get_props() asc_props = get_props("ASCENDING") pg = { "lc": { "process_id": "load_collection", "arguments": {"id": "S2_FOOBAR", "properties": properties}, }, "lc2": { "process_id": "load_collection", "arguments": {"id": "S2_FOOBAR", "properties": asc_props}, }, "merge": { "process_id": "merge_cubes", "arguments": {"cube1": {"from_node": "lc"}, "cube2": {"from_node": "lc2"}}, "result": True } } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert source_constraints == [ (("load_collection", ("S2_FOOBAR", (("orbitDirection", (("eq", "DESCENDING"),),),))), {"properties": properties}), (("load_collection", ("S2_FOOBAR", (("orbitDirection", (("eq", "ASCENDING"),)),),)), {"properties": asc_props}) ]
def test_aggregate_spatial_and_filter_bbox(dry_run_env, dry_run_tracer): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} bbox = {"west": -1, "south": -1, "east": 9, "north": 9, "crs": "EPSG:4326"} cube = DataCube(PGNode("load_collection", id="S2_FOOBAR"), connection=None) cube = cube.filter_bbox(bbox=bbox) cube = cube.aggregate_spatial(geometries=polygon, reducer="mean") pg = cube.flat_graph() res = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "spatial_extent": bbox, "aggregate_spatial": {"geometries": shapely.geometry.shape(polygon)}, } geometries, = dry_run_tracer.get_geometries() assert isinstance(geometries, shapely.geometry.Polygon) assert shapely.geometry.mapping(geometries) == { "type": "Polygon", "coordinates": (((0.0, 0.0), (3.0, 5.0), (8.0, 2.0), (0.0, 0.0)),) }
def test_evaluate_basic_filter_temporal(dry_run_env, dry_run_tracer): pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "ft": { "process_id": "filter_temporal", "arguments": {"data": {"from_node": "lc"}, "extent": ["2020-02-02", "2020-03-03"]}, "result": True, }, } cube = evaluate(pg, env=dry_run_env) """ source_constraints = dry_run_tracer.get_source_constraints(merge=False) assert len(source_constraints) == 1 src, constraints = source_constraints.popitem() assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == [{"temporal_extent": [("2020-02-02", "2020-03-03")]}] """ source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == {"temporal_extent": ("2020-02-02", "2020-03-03")}
def test_aggregate_spatial_only(dry_run_env, dry_run_tracer): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} pg = { "lc": {"process_id": "load_collection", "arguments": {"id": "S2_FOOBAR"}}, "agg": { "process_id": "aggregate_spatial", "arguments": { "data": {"from_node": "lc"}, "geometries": polygon, "reducer": { "process_graph": { "mean": { "process_id": "mean", "arguments": {"data": {"from_parameter": "data"}}, "result": True } } } }, "result": True, }, } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "spatial_extent": {"west": 0.0, "south": 0.0, "east": 8.0, "north": 5.0, "crs": "EPSG:4326"}, "aggregate_spatial": {"geometries": shapely.geometry.shape(polygon)}, } geometries, = dry_run_tracer.get_geometries() assert isinstance(geometries, shapely.geometry.Polygon) assert shapely.geometry.mapping(geometries) == { "type": "Polygon", "coordinates": (((0.0, 0.0), (3.0, 5.0), (8.0, 2.0), (0.0, 0.0)),) }
def test_resample_filter_spatial(dry_run_env, dry_run_tracer): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} cube = DataCube(PGNode("load_collection", id="S2_FOOBAR"), connection=None) cube = cube.filter_spatial(geometries = polygon) cube = cube.resample_spatial(projection=4326, resolution=0.25) pg = cube.flat_graph() res = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) geometries, = dry_run_tracer.get_geometries(operation="filter_spatial") assert constraints == { "spatial_extent": {'crs': 'EPSG:4326','east': 8.0,'north': 5.0,'south': 0.0,'west': 0.0}, "filter_spatial": {"geometries": shapely.geometry.shape(polygon)}, "resample": {'method': 'near', 'resolution': [0.25, 0.25], 'target_crs': 4326}, } assert isinstance(geometries, shapely.geometry.Polygon) assert shapely.geometry.mapping(geometries) == { "type": "Polygon", "coordinates": (((0.0, 0.0), (3.0, 5.0), (8.0, 2.0), (0.0, 0.0)),) }
def test_filter_after_merge_cubes(dry_run_env, dry_run_tracer): """based on use case of https://jira.vito.be/browse/EP-3747""" pg = { "loadcollection1": { "process_id": "load_collection", "arguments": {"id": "S2_FOOBAR", "bands": ["B04", "B08"]} }, "reducedimension1": { "process_id": "reduce_dimension", "arguments": { "data": {"from_node": "loadcollection1"}, "dimension": "bands", "reducer": { "process_graph": { "arrayelement1": { "process_id": "array_element", "arguments": {"data": {"from_parameter": "data"}, "index": 1} }, "arrayelement2": { "process_id": "array_element", "arguments": {"data": {"from_parameter": "data"}, "index": 0} }, "subtract1": { "process_id": "subtract", "arguments": {"x": {"from_node": "arrayelement1"}, "y": {"from_node": "arrayelement2"}} }, "add1": { "process_id": "add", "arguments": {"x": {"from_node": "arrayelement1"}, "y": {"from_node": "arrayelement2"}} }, "divide1": { "process_id": "divide", "arguments": {"x": {"from_node": "subtract1"}, "y": {"from_node": "add1"}}, "result": True } } } } }, "adddimension1": { "process_id": "add_dimension", "arguments": { "data": {"from_node": "reducedimension1"}, "label": "s2_ndvi", "name": "bands", "type": "bands"} }, "loadcollection2": { "process_id": "load_collection", "arguments": {"id": "PROBAV_L3_S10_TOC_NDVI_333M_V2", "bands": ["ndvi"], } }, "resamplecubespatial1": { "process_id": "resample_cube_spatial", "arguments": { "data": {"from_node": "loadcollection2"}, "method": "near", "target": {"from_node": "adddimension1"} } }, "maskpolygon1": { "process_id": "mask_polygon", "arguments": { "data": {"from_node": "resamplecubespatial1"}, "mask": { "type": "Polygon", "coordinates": [[ [5.03536, 51.219], [5.03586, 51.230], [5.01754, 51.231], [5.01704, 51.219], [5.03536, 51.219] ]] } } }, "mergecubes1": { "process_id": "merge_cubes", "arguments": { "cube1": {"from_node": "adddimension1"}, "cube2": {"from_node": "maskpolygon1"} } }, "filtertemporal1": { "process_id": "filter_temporal", "arguments": { "data": {"from_node": "mergecubes1"}, "extent": ["2019-03-01", "2019-04-01"] } }, "filterbbox1": { "process_id": "filter_bbox", "arguments": { "data": {"from_node": "filtertemporal1"}, "extent": { "west": 640860.0, "east": 642140.0, "north": 5677450.0, "south": 5676170.0, "crs": "EPSG:32631" } }, "result": True } } cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert source_constraints == [ ( ('load_collection', ('S2_FOOBAR', ())), { 'bands': ['B04', 'B08'], 'spatial_extent': { 'crs': 'EPSG:32631', 'east': 642140.0, 'north': 5677450.0, 'south': 5676170.0, 'west': 640860.0, }, 'temporal_extent': ('2019-03-01', '2019-04-01')} ), ( ('load_collection', ('PROBAV_L3_S10_TOC_NDVI_333M_V2', ())), { 'bands': ['ndvi'], 'process_type': [ProcessType.FOCAL_SPACE], 'resample': {'resolution': [10, 10], 'target_crs': 'AUTO:42001'}, 'spatial_extent': { 'crs': 'EPSG:32631', 'east': 642140.0, 'north': 5677450.0, 'south': 5676170.0, 'west': 640860.0, }, 'temporal_extent': ('2019-03-01', '2019-04-01')} ), ( ('load_collection', ('S2_FOOBAR', ())), { 'bands': ['B04', 'B08'], 'spatial_extent': { 'crs': 'EPSG:32631', 'east': 642140.0, 'north': 5677450.0, 'south': 5676170.0, 'west': 640860.0, }, 'temporal_extent': ('2019-03-01', '2019-04-01')} ) ]
def test_aggregate_spatial_apply_dimension(dry_run_env, dry_run_tracer): polygon = {"type": "Polygon", "coordinates": [[(0, 0), (3, 5), (8, 2), (0, 0)]]} pg = {'loadcollection1': {'process_id': 'load_collection', 'arguments': {'bands': ['B04', 'B08', 'B11', 'SCL'], 'id': 'S2_FOOBAR', 'spatial_extent': None, 'temporal_extent': ['2018-11-01', '2020-02-01']}}, 'maskscldilation1': {'process_id': 'mask_scl_dilation', 'arguments': {'data': {'from_node': 'loadcollection1'}, 'scl_band_name': 'SCL'}}, 'aggregatetemporalperiod1': {'process_id': 'aggregate_temporal_period', 'arguments': {'data': {'from_node': 'maskscldilation1'}, 'period': 'month', 'reducer': {'process_graph': {'mean1': {'process_id': 'mean', 'arguments': {'data': { 'from_parameter': 'data'}}, 'result': True}}}}}, 'applydimension1': {'process_id': 'apply_dimension', 'arguments': {'data': {'from_node': 'aggregatetemporalperiod1'}, 'dimension': 't', 'process': {'process_graph': { 'arrayinterpolatelinear1': {'process_id': 'array_interpolate_linear', 'arguments': { 'data': {'from_parameter': 'data'}}, 'result': True}}}}}, 'filtertemporal1': {'process_id': 'filter_temporal', 'arguments': {'data': {'from_node': 'applydimension1'}, 'extent': ['2019-01-01', '2020-01-01']}}, 'applydimension2': {'process_id': 'apply_dimension', 'arguments': {'data': {'from_node': 'filtertemporal1'}, 'dimension': 'bands', 'process': { 'process_graph': {'arrayelement1': {'process_id': 'array_element', 'arguments': {'data': {'from_parameter': 'data'}, 'index': 1}}, 'arrayelement2': {'process_id': 'array_element', 'arguments': {'data': {'from_parameter': 'data'}, 'index': 0}}, 'normalizeddifference1': {'process_id': 'normalized_difference', 'arguments': { 'x': {'from_node': 'arrayelement1'}, 'y': {'from_node': 'arrayelement2'}}}, 'arraymodify1': {'process_id': 'array_modify', 'arguments': {'data': {'from_parameter': 'data'}, 'index': 0, 'values': { 'from_node': 'normalizeddifference1'}}, 'result': True}}}}}, 'renamelabels1': {'process_id': 'rename_labels', 'arguments': {'data': {'from_node': 'applydimension2'}, 'dimension': 'bands', 'target': ['NDVI', 'B04', 'B08']}}, 'aggregatespatial1': {'process_id': 'aggregate_spatial', 'arguments': {'data': {'from_node': 'renamelabels1'}, 'geometries': polygon, 'reducer': {'process_graph': {'mean2': {'process_id': 'mean', 'arguments': {'data': { 'from_parameter': 'data'}}, 'result': True}}}}, 'result': True}} cube = evaluate(pg, env=dry_run_env) source_constraints = dry_run_tracer.get_source_constraints(merge=True) assert len(source_constraints) == 1 src, constraints = source_constraints[0] assert src == ("load_collection", ("S2_FOOBAR", ())) assert constraints == { "spatial_extent": {"west": 0.0, "south": 0.0, "east": 8.0, "north": 5.0, "crs": "EPSG:4326"}, "process_type": [ProcessType.GLOBAL_TIME], "bands": ["B04", "B08", "B11", "SCL"], "custom_cloud_mask": {"method": "mask_scl_dilation", 'scl_band_name': 'SCL'}, "aggregate_spatial": {"geometries": shapely.geometry.shape(polygon)}, "temporal_extent": ("2018-11-01", "2020-02-01") } geometries, = dry_run_tracer.get_geometries() assert isinstance(geometries, shapely.geometry.Polygon) assert shapely.geometry.mapping(geometries) == { "type": "Polygon", "coordinates": (((0.0, 0.0), (3.0, 5.0), (8.0, 2.0), (0.0, 0.0)),) }