def test_layer_model_floats():
layer_def = LayerModel(
dataset="test",
version="v1.1.1",
source_type="raster",
pixel_meaning="test",
data_type=DataTypeEnum.float32,
nbits=6,
grid="10/40000",
resampling="bilinear",
source_uri=["s3://test/tiles.geojson"],
no_data="nan",
)
assert isinstance(layer_def.no_data, float)
assert math.isnan(layer_def.no_data)
layer_def = LayerModel(
dataset="test",
version="v1.1.1",
source_type=SourceType.raster,
pixel_meaning="test",
data_type=DataTypeEnum.float32,
nbits=6,
grid="10/40000",
resampling="bilinear",
source_uri=["s3://test/tiles.geojson"],
no_data="2.2",
)
assert isinstance(layer_def.no_data, float)
def test_layer_model():
with pytest.raises(ValidationError):
layer_def = LayerModel(
dataset="test",
version="v1.1.1",
source_type="raster",
pixel_meaning="test",
data_type=DataTypeEnum.uint8,
nbits=6,
no_data=0,
grid="10/40000",
resampling="wrong",
source_uri=["s3://test/tiles.geojson"],
)
layer_def = LayerModel(
dataset="test",
version="v1.1.1",
source_type="raster",
pixel_meaning="test",
data_type=DataTypeEnum.uint8,
nbits=6,
no_data=0,
grid="10/40000",
resampling="bilinear",
source_uri=["s3://test/tiles.geojson"],
)
resampling = resampling_factory(layer_def.resampling)
assert resampling == Resampling.bilinear
def test_models_nan(self):
good_layer_dict = copy.deepcopy(minimal_layer_dict)
good_layer_dict.update(no_data="NaN")
layer = LayerModel.parse_obj(good_layer_dict)
assert math.isnan(layer.no_data)
bad_layer_dict = copy.deepcopy(minimal_layer_dict)
bad_layer_dict.update(no_data="NaNa")
with pytest.raises(ValidationError):
LayerModel.parse_obj(bad_layer_dict)
def test_vector_layer():
layer_dict = {
**minimal_layer_dict,
"source_type": "vector",
"no_data": 0,
"nbits": 2,
"data_type": "uint8",
"order": "desc",
}
layer_dict.pop("source_uri")
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.VectorSrcLayer)
assert layer.__class__.__name__ == "VectorSrcLayer"
assert layer.dst_profile["dtype"] == "uint8"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.dst_profile["nodata"] == 0
assert layer.dst_profile["nbits"] == 2
assert layer.calc == "Mg_ha-1"
assert layer.resampling == Resampling.nearest
assert layer.rasterize_method is None
assert layer.order == "desc"
def test_raster_calc_layer():
layer_dict = {
**minimal_layer_dict,
"data_type": "uint8",
"nbits": 3,
"no_data": 0,
"calc":
"1*(A>10)+1*(A>15)+1*(A>20)+1*(A>25)+1*(A>30)+1*(A>50)+1*(A>75)",
"resampling": "nearest",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.__class__.__name__ == "RasterSrcLayer"
assert layer.dst_profile["dtype"] == "uint8"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.dst_profile["nodata"] == 0
assert layer.dst_profile["nbits"] == 3
assert layer.resampling == Resampling.nearest
assert (layer.calc ==
"1*(A>10)+1*(A>15)+1*(A>20)+1*(A>25)+1*(A>30)+1*(A>50)+1*(A>75)")
assert layer.rasterize_method is None
assert layer.order is None
def test_multi_source_layer():
layer_dict = {
**minimal_layer_dict,
"source_uri": [
f"s3://{BUCKET}/{GEOJSON_NAME}",
f"s3://{BUCKET}/{GEOJSON_2_NAME}",
],
"calc":
"A + B",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.__class__.__name__ == "RasterSrcLayer"
assert layer.dst_profile["dtype"] == "uint16"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.resampling == Resampling.nearest
assert layer.calc == "A + B"
assert layer.rasterize_method is None
assert layer.order is None
assert layer.geom == MultiPolygon([
Polygon([[10.0, 10.0], [20.0, 10.0], [20.0, 0.0], [10.0, 0.0],
[10.0, 10.0]]),
Polygon([[-10.0, 10.0], [0.0, 10.0], [0.0, 0.0], [-10.0, 0.0],
[-10.0, 10.0]]),
])
def test_version_pattern(self):
good_versions = [
"v2019", "v201911", "v20191122", "v1", "v1.2", "v1.2.3"
]
bad_versions = ["v1.beta", "1.2", "version1.2.3", "v.1.2.3"]
for v in good_versions:
good_layer_dict = copy.deepcopy(minimal_layer_dict)
good_layer_dict.update(version=v)
layer = LayerModel(**good_layer_dict)
assert layer.version == v
for v in bad_versions:
bad_layer_dict = copy.deepcopy(minimal_layer_dict)
bad_layer_dict.update(version=v)
with pytest.raises(ValidationError):
LayerModel(**bad_layer_dict)
def LAYER_MULTI():
layer_dict_multi = deepcopy(LAYER_DICT)
layer_dict_multi["source_uri"] = [
f"s3://{BUCKET}/{GEOJSON_NAME}",
f"s3://{BUCKET}/{GEOJSON_NAME}",
]
layer_dict_multi["calc"] = "A + B"
yield layer_factory(LayerModel(**layer_dict_multi))
def test_multi_source_layer_no_calc():
layer_dict = {
**minimal_layer_dict,
"source_uri": [
f"s3://{BUCKET}/{GEOJSON_NAME}",
f"s3://{BUCKET}/{GEOJSON_2_NAME}",
],
}
with pytest.raises(ValidationError):
layers.layer_factory(LayerModel.parse_obj(layer_dict))
def test_layer_model_multi_band_output_no_no_data():
LayerModel(
dataset="test",
version="v1.1.1",
source_type=SourceType.raster,
pixel_meaning="test",
data_type=DataTypeEnum.uint8,
nbits=6,
no_data=None,
band_count=3,
calc="test",
grid="10/40000",
source_uri=["s3://test/tiles.geojson"],
)
def test_folder_source_uri():
layer_dict = {
**minimal_layer_dict,
"source_uri": [f"s3://{BUCKET}/folder"],
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.geom == MultiPolygon([
Polygon([[10.0, 10.0], [20.0, 10.0], [20.0, 0.0], [10.0, 0.0],
[10.0, 10.0]]),
Polygon([[-10.0, 10.0], [0.0, 10.0], [0.0, 0.0], [-10.0, 0.0],
[-10.0, 10.0]]),
])
def test_layer_model_multi_band_no_calc_multi_input():
# no calc, multi band input
with pytest.raises(ValidationError):
LayerModel(
dataset="test",
version="v1.1.1",
source_type=SourceType.raster,
pixel_meaning="test",
data_type=DataTypeEnum.uint8,
nbits=6,
no_data=0,
band_count=1,
grid="10/40000",
source_uri=["s3://test/tiles.geojson", "s3://test/tiles.geojson"],
)
def test_layer_model_multi_band_no_data_different():
# varying no data types
with pytest.raises(ValidationError):
LayerModel(
dataset="test",
version="v1.1.1",
source_type=SourceType.raster,
pixel_meaning="test",
data_type=DataTypeEnum.uint8,
nbits=6,
no_data=[1, 2, 3],
band_count=3,
calc="A * 5",
grid="10/40000",
source_uri=["s3://test/tiles.geojson"],
)
def test_get_grid_tiles():
# message = ""
# try:
# len(PIPE.get_grid_tiles())
# except NotImplementedError:
# message = "not implemented"
# assert message == "not implemented"
layer_dict = {
**LAYER_DICT,
"grid": "10/40000",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
pipe = RasterPipe(layer)
assert len(pipe.get_grid_tiles()) == 648
def test_pipe_factory_raster_src_layer():
layer_dict = {
**minimal_layer_dict,
"dataset": "aqueduct_erosion_risk",
"version": "v201911",
"pixel_meaning": "level",
"no_data": 0,
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
pipe = pipe_factory(layer)
assert isinstance(pipe, RasterPipe)
assert pipe.__class__.__name__ == "RasterPipe"
pipe = pipe_factory(layer, SUBSET)
assert isinstance(pipe, RasterPipe)
assert pipe.__class__.__name__ == "RasterPipe"
def cli(
dataset: str,
version: str,
subset: Optional[List[str]],
overwrite: bool,
layer_json: str,
):
layer_dict = json.loads(layer_json)
layer_dict.update({"dataset": dataset, "version": version})
layer_def = LayerModel.parse_obj(layer_dict)
# Raster sources must have an source URI
if layer_def.source_type == "raster" and layer_def.source_uri is None:
raise ValueError("URI specification is required for raster sources")
# Finally, actually process the layer
tiles, skipped_tiles, failed_tiles, existing_tiles = pixetl(
layer_def,
subset,
overwrite,
)
nb_tiles = len(tiles)
nb_skipped_tiles = len(skipped_tiles)
nb_failed_tiles = len(failed_tiles)
nb_existing_tiles = len(existing_tiles)
LOGGER.info(f"Successfully processed {len(tiles)} tiles")
LOGGER.info(f"{nb_skipped_tiles} tiles skipped.")
LOGGER.info(f"{nb_existing_tiles} tiles already existed.")
LOGGER.info(f"{nb_failed_tiles} tiles failed.")
if nb_tiles:
LOGGER.info(f"Processed tiles: {tiles}")
if nb_skipped_tiles:
LOGGER.info(f"Skipped tiles: {skipped_tiles}")
if nb_existing_tiles:
LOGGER.info(f"Existing tiles: {existing_tiles}")
if nb_failed_tiles:
LOGGER.info(f"Failed tiles: {failed_tiles}")
sys.exit(
"Program terminated with Errors. Some tiles failed to process")
def test_raster_layer_uri():
layer_dict = {
**minimal_layer_dict,
"no_data": 0,
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
assert layer.__class__.__name__ == "RasterSrcLayer"
assert layer.dst_profile["dtype"] == "uint16"
assert layer.dst_profile["compress"] == "DEFLATE"
assert layer.dst_profile["tiled"] is True
assert layer.dst_profile["blockxsize"] == 400
assert layer.dst_profile["blockysize"] == 400
assert layer.dst_profile["pixeltype"] == "DEFAULT"
assert layer.dst_profile["nodata"] == 0
assert layer.resampling == Resampling.nearest
assert layer.calc is None
assert layer.rasterize_method is None
assert layer.order is None
def test_transform_final_wm():
layer_dict_wm = deepcopy(LAYER_DICT)
layer_dict_wm["grid"] = "zoom_0"
layer_dict_wm["source_uri"] = [f"s3://{BUCKET}/{GEOJSON_2_NAME}"]
layer_wm = layers.layer_factory(LayerModel(**layer_dict_wm))
assert isinstance(layer_wm, layers.RasterSrcLayer)
tile = RasterSrcTile("000R_000C", layer_wm.grid, layer_wm)
assert tile.dst[tile.default_format].crs.is_valid
tile.transform()
LOGGER.debug(tile.local_dst[tile.default_format].uri)
with rasterio.Env(**GDAL_ENV), rasterio.open(
tile.local_dst[tile.default_format].uri) as src:
src_profile = src.profile
output = src.read(1)
LOGGER.debug(src_profile)
assert output.shape == (256, 256)
assert src_profile["blockxsize"] == layer_wm.grid.blockxsize
assert src_profile["blockysize"] == layer_wm.grid.blockysize
assert src_profile["compress"].lower(
) == layer_wm.dst_profile["compress"].lower()
assert src_profile["count"] == 1
assert src_profile["crs"] == {"init": layer_wm.grid.crs.srs}
assert src_profile["crs"].is_valid
assert src_profile["driver"] == "GTiff"
assert src_profile["dtype"] == layer_wm.dst_profile["dtype"]
assert src_profile["height"] == layer_wm.grid.cols
assert src_profile["interleave"] == "band"
assert src_profile["nodata"] == layer_wm.dst_profile["nodata"]
assert src_profile["tiled"] is True
assert src_profile["width"] == layer_wm.grid.rows
# assert src_profile["nbits"] == nbits # Not exposed in rasterio API
assert not hasattr(src_profile, "compress")
os.remove(tile.local_dst[tile.default_format].uri)
def _get_subset_tiles() -> Set[RasterSrcTile]:
layer_dict = {
**minimal_layer_dict,
"dataset": "aqueduct_erosion_risk",
"version": "v201911",
"pixel_meaning": "level",
"no_data": 0,
"grid": "1/4000",
}
layer = layers.layer_factory(LayerModel.parse_obj(layer_dict))
assert isinstance(layer, layers.RasterSrcLayer)
pipe = RasterPipe(layer)
tiles = set()
for i in range(10, 12):
for j in range(10, 12):
assert isinstance(pipe.grid, LatLngGrid)
tile_id = pipe.grid.xy_to_tile_id(j, i)
tiles.add(
RasterSrcTile(tile_id=tile_id, grid=pipe.grid, layer=layer))
return tiles
def test_models_bad_order(self):
bad_layer_dict = {**minimal_layer_dict, "order": "random"}
with self.assertRaises(ValidationError) as e:
_ = LayerModel.parse_obj(bad_layer_dict)
assert "order" in str(e)
def LAYER_WM():
layer_dict_wm = deepcopy(LAYER_DICT)
layer_dict_wm["grid"] = "zoom_14"
yield layer_factory(LayerModel(**layer_dict_wm))
from unittest import mock
from gfw_pixetl.models.pydantic import LayerModel
from gfw_pixetl.pipes import RasterPipe
from gfw_pixetl.pixetl import pixetl
from tests.conftest import minimal_layer_dict
LAYER_DICT = deepcopy(minimal_layer_dict)
LAYER_DICT.update({
"dataset": "aqueduct_erosion_risk",
"version": "v201911",
"pixel_meaning": "level",
"grid": "1/4000",
})
RASTER_LAYER_DEF = LayerModel.parse_obj(LAYER_DICT)
SUBSET = ["10N_010E"]
def test_pixetl():
cwd = os.getcwd()
with mock.patch.object(RasterPipe,
"create_tiles",
return_value=(list(), list(), list(), list())):
tiles, skipped_tiles, failed_tiles, existing_tiles = pixetl(
RASTER_LAYER_DEF,
subset=SUBSET,
overwrite=True,
def test_models_good_rasterize_method(self):
good_layer_dict = {**minimal_layer_dict, "rasterize_method": "count"}
_ = LayerModel.parse_obj(good_layer_dict)
def test_models_bad_rasterize_method(self):
bad_layer_dict = {**minimal_layer_dict, "rasterize_method": "random"}
with self.assertRaises(ValidationError) as e:
_ = LayerModel.parse_obj(bad_layer_dict)
assert "rasterize_method" in str(e)
def test_models_bad_source_type(self):
bad_layer_dict = {**minimal_layer_dict, "source_type": "frog"}
with self.assertRaises(ValidationError) as e:
_ = LayerModel.parse_obj(bad_layer_dict)
assert "source_type" in str(e)
def LAYER():
layer_def = LayerModel.parse_obj(LAYER_DICT)
yield layer_factory(layer_def)