def test_multi_source_layer_no_calc(): layer_dict = { **minimal_layer_dict, "source_uri": [ f"s3://{BUCKET}/{GEOJSON_NAME}", f"s3://{BUCKET}/{GEOJSON_2_NAME}", ], } with pytest.raises(ValidationError): layers.layer_factory(LayerModel.parse_obj(layer_dict))
def test_vector_layer(): layer_dict = { **minimal_layer_dict, "source_type": "vector", "no_data": 0, "nbits": 2, "data_type": "uint8", "order": "desc", } layer_dict.pop("source_uri") layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) assert isinstance(layer, layers.VectorSrcLayer) assert layer.__class__.__name__ == "VectorSrcLayer" assert layer.dst_profile["dtype"] == "uint8" assert layer.dst_profile["compress"] == "DEFLATE" assert layer.dst_profile["tiled"] is True assert layer.dst_profile["blockxsize"] == 400 assert layer.dst_profile["blockysize"] == 400 assert layer.dst_profile["pixeltype"] == "DEFAULT" assert layer.dst_profile["nodata"] == 0 assert layer.dst_profile["nbits"] == 2 assert layer.calc == "Mg_ha-1" assert layer.resampling == Resampling.nearest assert layer.rasterize_method is None assert layer.order == "desc"
def test_raster_calc_layer(): layer_dict = { **minimal_layer_dict, "data_type": "uint8", "nbits": 3, "no_data": 0, "calc": "1*(A>10)+1*(A>15)+1*(A>20)+1*(A>25)+1*(A>30)+1*(A>50)+1*(A>75)", "resampling": "nearest", } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) assert isinstance(layer, layers.RasterSrcLayer) assert layer.__class__.__name__ == "RasterSrcLayer" assert layer.dst_profile["dtype"] == "uint8" assert layer.dst_profile["compress"] == "DEFLATE" assert layer.dst_profile["tiled"] is True assert layer.dst_profile["blockxsize"] == 400 assert layer.dst_profile["blockysize"] == 400 assert layer.dst_profile["pixeltype"] == "DEFAULT" assert layer.dst_profile["nodata"] == 0 assert layer.dst_profile["nbits"] == 3 assert layer.resampling == Resampling.nearest assert (layer.calc == "1*(A>10)+1*(A>15)+1*(A>20)+1*(A>25)+1*(A>30)+1*(A>50)+1*(A>75)") assert layer.rasterize_method is None assert layer.order is None
def test_multi_source_layer(): layer_dict = { **minimal_layer_dict, "source_uri": [ f"s3://{BUCKET}/{GEOJSON_NAME}", f"s3://{BUCKET}/{GEOJSON_2_NAME}", ], "calc": "A + B", } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) assert isinstance(layer, layers.RasterSrcLayer) assert layer.__class__.__name__ == "RasterSrcLayer" assert layer.dst_profile["dtype"] == "uint16" assert layer.dst_profile["compress"] == "DEFLATE" assert layer.dst_profile["tiled"] is True assert layer.dst_profile["blockxsize"] == 400 assert layer.dst_profile["blockysize"] == 400 assert layer.dst_profile["pixeltype"] == "DEFAULT" assert layer.resampling == Resampling.nearest assert layer.calc == "A + B" assert layer.rasterize_method is None assert layer.order is None assert layer.geom == MultiPolygon([ Polygon([[10.0, 10.0], [20.0, 10.0], [20.0, 0.0], [10.0, 0.0], [10.0, 10.0]]), Polygon([[-10.0, 10.0], [0.0, 10.0], [0.0, 0.0], [-10.0, 0.0], [-10.0, 10.0]]), ])
def LAYER_MULTI(): layer_dict_multi = deepcopy(LAYER_DICT) layer_dict_multi["source_uri"] = [ f"s3://{BUCKET}/{GEOJSON_NAME}", f"s3://{BUCKET}/{GEOJSON_NAME}", ] layer_dict_multi["calc"] = "A + B" yield layer_factory(LayerModel(**layer_dict_multi))
def test_folder_source_uri(): layer_dict = { **minimal_layer_dict, "source_uri": [f"s3://{BUCKET}/folder"], } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) assert isinstance(layer, layers.RasterSrcLayer) assert layer.geom == MultiPolygon([ Polygon([[10.0, 10.0], [20.0, 10.0], [20.0, 0.0], [10.0, 0.0], [10.0, 10.0]]), Polygon([[-10.0, 10.0], [0.0, 10.0], [0.0, 0.0], [-10.0, 0.0], [-10.0, 10.0]]), ])
def test_get_grid_tiles(): # message = "" # try: # len(PIPE.get_grid_tiles()) # except NotImplementedError: # message = "not implemented" # assert message == "not implemented" layer_dict = { **LAYER_DICT, "grid": "10/40000", } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) pipe = RasterPipe(layer) assert len(pipe.get_grid_tiles()) == 648
def test_pipe_factory_raster_src_layer(): layer_dict = { **minimal_layer_dict, "dataset": "aqueduct_erosion_risk", "version": "v201911", "pixel_meaning": "level", "no_data": 0, } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) pipe = pipe_factory(layer) assert isinstance(pipe, RasterPipe) assert pipe.__class__.__name__ == "RasterPipe" pipe = pipe_factory(layer, SUBSET) assert isinstance(pipe, RasterPipe) assert pipe.__class__.__name__ == "RasterPipe"
def test_raster_layer_uri(): layer_dict = { **minimal_layer_dict, "no_data": 0, } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) assert isinstance(layer, layers.RasterSrcLayer) assert layer.__class__.__name__ == "RasterSrcLayer" assert layer.dst_profile["dtype"] == "uint16" assert layer.dst_profile["compress"] == "DEFLATE" assert layer.dst_profile["tiled"] is True assert layer.dst_profile["blockxsize"] == 400 assert layer.dst_profile["blockysize"] == 400 assert layer.dst_profile["pixeltype"] == "DEFAULT" assert layer.dst_profile["nodata"] == 0 assert layer.resampling == Resampling.nearest assert layer.calc is None assert layer.rasterize_method is None assert layer.order is None
def pixetl( layer_def: LayerModel, subset: Optional[List[str]] = None, overwrite: bool = False, ) -> Tuple[List[Tile], List[Tile], List[Tile], List[Tile]]: click.echo(logo) LOGGER.info( f"Start tile preparation for dataset {layer_def.dataset}, " f"version {layer_def.version}, grid {layer_def.grid}, " f"source type {layer_def.source_type}, field {layer_def.pixel_meaning}, " f"with overwrite set to {overwrite}.") LOGGER.debug(f"Full layer_def: {layer_def.json()}") old_cwd = os.getcwd() cwd = set_cwd() # set available memory here before any major process is running # utils.set_available_memory() try: if subset: LOGGER.info("Running on subset: {}".format(subset)) else: LOGGER.info("Running on full extent") layer: Layer = layer_factory(layer_def) pipe: Pipe = pipe_factory(layer, subset) tiles, skipped_tiles, failed_tiles, existing_tiles = pipe.create_tiles( overwrite) remove_work_directory(old_cwd, cwd) return tiles, skipped_tiles, failed_tiles, existing_tiles except Exception as e: remove_work_directory(old_cwd, cwd) LOGGER.exception(e) raise
def test_transform_final_wm(): layer_dict_wm = deepcopy(LAYER_DICT) layer_dict_wm["grid"] = "zoom_0" layer_dict_wm["source_uri"] = [f"s3://{BUCKET}/{GEOJSON_2_NAME}"] layer_wm = layers.layer_factory(LayerModel(**layer_dict_wm)) assert isinstance(layer_wm, layers.RasterSrcLayer) tile = RasterSrcTile("000R_000C", layer_wm.grid, layer_wm) assert tile.dst[tile.default_format].crs.is_valid tile.transform() LOGGER.debug(tile.local_dst[tile.default_format].uri) with rasterio.Env(**GDAL_ENV), rasterio.open( tile.local_dst[tile.default_format].uri) as src: src_profile = src.profile output = src.read(1) LOGGER.debug(src_profile) assert output.shape == (256, 256) assert src_profile["blockxsize"] == layer_wm.grid.blockxsize assert src_profile["blockysize"] == layer_wm.grid.blockysize assert src_profile["compress"].lower( ) == layer_wm.dst_profile["compress"].lower() assert src_profile["count"] == 1 assert src_profile["crs"] == {"init": layer_wm.grid.crs.srs} assert src_profile["crs"].is_valid assert src_profile["driver"] == "GTiff" assert src_profile["dtype"] == layer_wm.dst_profile["dtype"] assert src_profile["height"] == layer_wm.grid.cols assert src_profile["interleave"] == "band" assert src_profile["nodata"] == layer_wm.dst_profile["nodata"] assert src_profile["tiled"] is True assert src_profile["width"] == layer_wm.grid.rows # assert src_profile["nbits"] == nbits # Not exposed in rasterio API assert not hasattr(src_profile, "compress") os.remove(tile.local_dst[tile.default_format].uri)
def _get_subset_tiles() -> Set[RasterSrcTile]: layer_dict = { **minimal_layer_dict, "dataset": "aqueduct_erosion_risk", "version": "v201911", "pixel_meaning": "level", "no_data": 0, "grid": "1/4000", } layer = layers.layer_factory(LayerModel.parse_obj(layer_dict)) assert isinstance(layer, layers.RasterSrcLayer) pipe = RasterPipe(layer) tiles = set() for i in range(10, 12): for j in range(10, 12): assert isinstance(pipe.grid, LatLngGrid) tile_id = pipe.grid.xy_to_tile_id(j, i) tiles.add( RasterSrcTile(tile_id=tile_id, grid=pipe.grid, layer=layer)) return tiles
def LAYER_WM(): layer_dict_wm = deepcopy(LAYER_DICT) layer_dict_wm["grid"] = "zoom_14" yield layer_factory(LayerModel(**layer_dict_wm))
def LAYER(): layer_def = LayerModel.parse_obj(LAYER_DICT) yield layer_factory(layer_def)