def test_multi_source_layer_no_calc():
layer_dict = {
**minimal_layer_dict,
"source_uri": [
f"s3://{BUCKET}/{GEOJSON_NAME}",
f"s3://{BUCKET}/{GEOJSON_2_NAME}",
],
}
with pytest.raises(ValidationError):
layers.layer_factory(LayerModel.parse_obj(layer_dict))
def test_vector_layer():
layer_dict = {
**minimal_layer_dict,
"source_type": "vector",
"no_data": 0,
"nbits": 2,
"data_type": "uint8",
"order": "desc",
}
layer_dict.pop("source_uri")
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.VectorSrcLayer)
assert layer.__class__.__name__ == "VectorSrcLayer"
assert layer.dst_profile["dtype"] == "uint8"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.dst_profile["nodata"] == 0
assert layer.dst_profile["nbits"] == 2
assert layer.calc == "Mg_ha-1"
assert layer.resampling == Resampling.nearest
assert layer.rasterize_method is None
assert layer.order == "desc"
def test_raster_calc_layer():
layer_dict = {
**minimal_layer_dict,
"data_type": "uint8",
"nbits": 3,
"no_data": 0,
"calc":
"1*(A>10)+1*(A>15)+1*(A>20)+1*(A>25)+1*(A>30)+1*(A>50)+1*(A>75)",
"resampling": "nearest",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.__class__.__name__ == "RasterSrcLayer"
assert layer.dst_profile["dtype"] == "uint8"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.dst_profile["nodata"] == 0
assert layer.dst_profile["nbits"] == 3
assert layer.resampling == Resampling.nearest
assert (layer.calc ==
"1*(A>10)+1*(A>15)+1*(A>20)+1*(A>25)+1*(A>30)+1*(A>50)+1*(A>75)")
assert layer.rasterize_method is None
assert layer.order is None
def test_multi_source_layer():
layer_dict = {
**minimal_layer_dict,
"source_uri": [
f"s3://{BUCKET}/{GEOJSON_NAME}",
f"s3://{BUCKET}/{GEOJSON_2_NAME}",
],
"calc":
"A + B",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.__class__.__name__ == "RasterSrcLayer"
assert layer.dst_profile["dtype"] == "uint16"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.resampling == Resampling.nearest
assert layer.calc == "A + B"
assert layer.rasterize_method is None
assert layer.order is None
assert layer.geom == MultiPolygon([
Polygon([[10.0, 10.0], [20.0, 10.0], [20.0, 0.0], [10.0, 0.0],
[10.0, 10.0]]),
Polygon([[-10.0, 10.0], [0.0, 10.0], [0.0, 0.0], [-10.0, 0.0],
[-10.0, 10.0]]),
])
def LAYER_MULTI():
layer_dict_multi = deepcopy(LAYER_DICT)
layer_dict_multi["source_uri"] = [
f"s3://{BUCKET}/{GEOJSON_NAME}",
f"s3://{BUCKET}/{GEOJSON_NAME}",
]
layer_dict_multi["calc"] = "A + B"
yield layer_factory(LayerModel(**layer_dict_multi))
def test_folder_source_uri():
layer_dict = {
**minimal_layer_dict,
"source_uri": [f"s3://{BUCKET}/folder"],
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.geom == MultiPolygon([
Polygon([[10.0, 10.0], [20.0, 10.0], [20.0, 0.0], [10.0, 0.0],
[10.0, 10.0]]),
Polygon([[-10.0, 10.0], [0.0, 10.0], [0.0, 0.0], [-10.0, 0.0],
[-10.0, 10.0]]),
])
def test_get_grid_tiles():
# message = ""
# try:
# len(PIPE.get_grid_tiles())
# except NotImplementedError:
# message = "not implemented"
# assert message == "not implemented"
layer_dict = {
**LAYER_DICT,
"grid": "10/40000",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
pipe = RasterPipe(layer)
assert len(pipe.get_grid_tiles()) == 648
def test_pipe_factory_raster_src_layer():
layer_dict = {
**minimal_layer_dict,
"dataset": "aqueduct_erosion_risk",
"version": "v201911",
"pixel_meaning": "level",
"no_data": 0,
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
pipe = pipe_factory(layer)
assert isinstance(pipe, RasterPipe)
assert pipe.__class__.__name__ == "RasterPipe"
pipe = pipe_factory(layer, SUBSET)
assert isinstance(pipe, RasterPipe)
assert pipe.__class__.__name__ == "RasterPipe"
def test_raster_layer_uri():
layer_dict = {
**minimal_layer_dict,
"no_data": 0,
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.__class__.__name__ == "RasterSrcLayer"
assert layer.dst_profile["dtype"] == "uint16"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.dst_profile["nodata"] == 0
assert layer.resampling == Resampling.nearest
assert layer.calc is None
assert layer.rasterize_method is None
assert layer.order is None
def pixetl(
layer_def: LayerModel,
subset: Optional[List[str]] = None,
overwrite: bool = False,
) -> Tuple[List[Tile], List[Tile], List[Tile], List[Tile]]:
click.echo(logo)
LOGGER.info(
f"Start tile preparation for dataset {layer_def.dataset}, "
f"version {layer_def.version}, grid {layer_def.grid}, "
f"source type {layer_def.source_type}, field {layer_def.pixel_meaning}, "
f"with overwrite set to {overwrite}.")
LOGGER.debug(f"Full layer_def: {layer_def.json()}")
old_cwd = os.getcwd()
cwd = set_cwd()
# set available memory here before any major process is running
# utils.set_available_memory()
try:
if subset:
LOGGER.info("Running on subset: {}".format(subset))
else:
LOGGER.info("Running on full extent")
layer: Layer = layer_factory(layer_def)
pipe: Pipe = pipe_factory(layer, subset)
tiles, skipped_tiles, failed_tiles, existing_tiles = pipe.create_tiles(
overwrite)
remove_work_directory(old_cwd, cwd)
return tiles, skipped_tiles, failed_tiles, existing_tiles
except Exception as e:
remove_work_directory(old_cwd, cwd)
LOGGER.exception(e)
raise
def test_transform_final_wm():
layer_dict_wm = deepcopy(LAYER_DICT)
layer_dict_wm["grid"] = "zoom_0"
layer_dict_wm["source_uri"] = [f"s3://{BUCKET}/{GEOJSON_2_NAME}"]
layer_wm = layers.layer_factory(LayerModel(**layer_dict_wm))
assert isinstance(layer_wm, layers.RasterSrcLayer)
tile = RasterSrcTile("000R_000C", layer_wm.grid, layer_wm)
assert tile.dst[tile.default_format].crs.is_valid
tile.transform()
LOGGER.debug(tile.local_dst[tile.default_format].uri)
with rasterio.Env(**GDAL_ENV), rasterio.open(
tile.local_dst[tile.default_format].uri) as src:
src_profile = src.profile
output = src.read(1)
LOGGER.debug(src_profile)
assert output.shape == (256, 256)
assert src_profile["blockxsize"] == layer_wm.grid.blockxsize
assert src_profile["blockysize"] == layer_wm.grid.blockysize
assert src_profile["compress"].lower(
) == layer_wm.dst_profile["compress"].lower()
assert src_profile["count"] == 1
assert src_profile["crs"] == {"init": layer_wm.grid.crs.srs}
assert src_profile["crs"].is_valid
assert src_profile["driver"] == "GTiff"
assert src_profile["dtype"] == layer_wm.dst_profile["dtype"]
assert src_profile["height"] == layer_wm.grid.cols
assert src_profile["interleave"] == "band"
assert src_profile["nodata"] == layer_wm.dst_profile["nodata"]
assert src_profile["tiled"] is True
assert src_profile["width"] == layer_wm.grid.rows
# assert src_profile["nbits"] == nbits # Not exposed in rasterio API
assert not hasattr(src_profile, "compress")
os.remove(tile.local_dst[tile.default_format].uri)
def _get_subset_tiles() -> Set[RasterSrcTile]:
layer_dict = {
**minimal_layer_dict,
"dataset": "aqueduct_erosion_risk",
"version": "v201911",
"pixel_meaning": "level",
"no_data": 0,
"grid": "1/4000",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
pipe = RasterPipe(layer)
tiles = set()
for i in range(10, 12):
for j in range(10, 12):
assert isinstance(pipe.grid, LatLngGrid)
tile_id = pipe.grid.xy_to_tile_id(j, i)
tiles.add(
RasterSrcTile(tile_id=tile_id, grid=pipe.grid, layer=layer))
return tiles
def LAYER_WM():
layer_dict_wm = deepcopy(LAYER_DICT)
layer_dict_wm["grid"] = "zoom_14"
yield layer_factory(LayerModel(**layer_dict_wm))
def LAYER():
layer_def = LayerModel.parse_obj(LAYER_DICT)
yield layer_factory(layer_def)
