def test_compute_metadata_invalid_raster(invalid_raster_file, use_chunks):
from terracotta.drivers.raster_base import RasterDriver
if use_chunks:
pytest.importorskip('crick')
with pytest.raises(ValueError):
RasterDriver.compute_metadata(str(invalid_raster_file), use_chunks=use_chunks)
def test_compute_metadata_invalid_options(big_raster_file_nodata):
from terracotta.drivers.raster_base import RasterDriver
with pytest.raises(ValueError):
RasterDriver.compute_metadata(
str(big_raster_file_nodata), max_shape=(256, 256), use_chunks=True
)
with pytest.raises(ValueError):
RasterDriver.compute_metadata(str(big_raster_file_nodata), max_shape=(256, 256, 1))
def test_compute_metadata(big_raster_file_nodata, big_raster_file_nomask,
nodata_type, use_chunks):
from terracotta.drivers.raster_base import RasterDriver
if nodata_type == 'nodata':
raster_file = big_raster_file_nodata
elif nodata_type == 'nomask':
raster_file = big_raster_file_nomask
if use_chunks:
pytest.importorskip('crick')
with rasterio.open(str(raster_file)) as src:
data = src.read(1, masked=True)
valid_data = data.compressed()
dataset_shape = list(
rasterio.features.dataset_features(src,
bidx=1,
band=False,
as_mask=True,
geographic=True))
convex_hull = MultiPolygon([shape(s['geometry'])
for s in dataset_shape]).convex_hull
# compare
if nodata_type == 'nomask':
with pytest.warns(UserWarning) as record:
mtd = RasterDriver.compute_metadata(str(raster_file),
use_chunks=use_chunks)
assert 'does not have a valid nodata value' in str(
record[0].message)
else:
mtd = RasterDriver.compute_metadata(str(raster_file),
use_chunks=use_chunks)
np.testing.assert_allclose(mtd['valid_percentage'],
100 * valid_data.size / data.size)
np.testing.assert_allclose(mtd['range'],
(valid_data.min(), valid_data.max()))
np.testing.assert_allclose(mtd['mean'], valid_data.mean())
np.testing.assert_allclose(mtd['stdev'], valid_data.std())
# allow some error margin since we only compute approximate quantiles
np.testing.assert_allclose(mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
rtol=2e-2,
atol=valid_data.max() / 100)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 1e-8
def test_compute_metadata_unoptimized(unoptimized_raster_file):
from terracotta import exceptions
from terracotta.drivers.raster_base import RasterDriver
with rasterio.open(str(unoptimized_raster_file)) as src:
data = src.read(1, masked=True)
valid_data = np.ma.masked_invalid(data).compressed()
convex_hull = convex_hull_exact(src)
# compare
with pytest.warns(exceptions.PerformanceWarning):
mtd = RasterDriver.compute_metadata(str(unoptimized_raster_file), use_chunks=False)
np.testing.assert_allclose(mtd['valid_percentage'], 100 * valid_data.size / data.size)
np.testing.assert_allclose(mtd['range'], (valid_data.min(), valid_data.max()))
np.testing.assert_allclose(mtd['mean'], valid_data.mean())
np.testing.assert_allclose(mtd['stdev'], valid_data.std())
# allow some error margin since we only compute approximate quantiles
np.testing.assert_allclose(
mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
rtol=2e-2
)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 1e-6
def test_compute_metadata_unoptimized(unoptimized_raster_file):
from terracotta import exceptions
from terracotta.drivers.raster_base import RasterDriver
with rasterio.open(str(unoptimized_raster_file)) as src:
data = src.read(1, masked=True)
valid_data = data.compressed()
dataset_shape = list(
rasterio.features.dataset_features(src,
bidx=1,
band=False,
as_mask=True,
geographic=True))
convex_hull = MultiPolygon([shape(s['geometry'])
for s in dataset_shape]).convex_hull
# compare
with pytest.warns(exceptions.PerformanceWarning):
mtd = RasterDriver.compute_metadata(str(unoptimized_raster_file),
use_chunks=False)
np.testing.assert_allclose(mtd['valid_percentage'],
100 * valid_data.size / data.size)
np.testing.assert_allclose(mtd['range'],
(valid_data.min(), valid_data.max()))
np.testing.assert_allclose(mtd['mean'], valid_data.mean())
np.testing.assert_allclose(mtd['stdev'], valid_data.std())
# allow some error margin since we only compute approximate quantiles
np.testing.assert_allclose(mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
rtol=2e-2)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 1e-8
def test_compute_metadata_approximate(nodata_type, big_raster_file_nodata, big_raster_file_mask):
from terracotta.drivers.raster_base import RasterDriver
if nodata_type == 'nodata':
raster_file = big_raster_file_nodata
elif nodata_type == 'masked':
raster_file = big_raster_file_mask
with rasterio.open(str(raster_file)) as src:
data = src.read(1, masked=True)
valid_data = data.compressed()
dataset_shape = list(rasterio.features.dataset_features(
src, bidx=1, band=False, as_mask=True, geographic=True
))
convex_hull = MultiPolygon([shape(s['geometry']) for s in dataset_shape]).convex_hull
# compare
mtd = RasterDriver.compute_metadata(str(raster_file), max_shape=(512, 512))
np.testing.assert_allclose(mtd['valid_percentage'], 100 * valid_data.size / data.size, atol=1)
np.testing.assert_allclose(
mtd['range'], (valid_data.min(), valid_data.max()), atol=valid_data.max() / 100
)
np.testing.assert_allclose(mtd['mean'], valid_data.mean(), rtol=0.02)
np.testing.assert_allclose(mtd['stdev'], valid_data.std(), rtol=0.02)
np.testing.assert_allclose(
mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
atol=valid_data.max() / 100, rtol=0.02
)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 0.05
def test_compute_metadata(big_raster_file_nodata, big_raster_file_nomask,
big_raster_file_mask, raster_file_float, nodata_type, use_chunks):
from terracotta.drivers.raster_base import RasterDriver
if nodata_type == 'nodata':
raster_file = big_raster_file_nodata
elif nodata_type == 'masked':
raster_file = big_raster_file_mask
elif nodata_type == 'none':
raster_file = big_raster_file_nomask
elif nodata_type == 'nan':
raster_file = raster_file_float
if use_chunks:
pytest.importorskip('crick')
with rasterio.open(str(raster_file)) as src:
data = src.read(1, masked=True)
valid_data = np.ma.masked_invalid(data).compressed()
convex_hull = convex_hull_exact(src)
# compare
if nodata_type == 'none':
with pytest.warns(UserWarning) as record:
mtd = RasterDriver.compute_metadata(str(raster_file), use_chunks=use_chunks)
assert 'does not have a valid nodata value' in str(record[0].message)
else:
mtd = RasterDriver.compute_metadata(str(raster_file), use_chunks=use_chunks)
np.testing.assert_allclose(mtd['valid_percentage'], 100 * valid_data.size / data.size)
np.testing.assert_allclose(mtd['range'], (valid_data.min(), valid_data.max()))
np.testing.assert_allclose(mtd['mean'], valid_data.mean())
np.testing.assert_allclose(mtd['stdev'], valid_data.std())
# allow some error margin since we only compute approximate quantiles
np.testing.assert_allclose(
mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
rtol=2e-2, atol=valid_data.max() / 100
)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 1e-6
def _get_vrt(src: DatasetReader, rs_method: int) -> WarpedVRT:
from terracotta.drivers.raster_base import RasterDriver
target_crs = RasterDriver._TARGET_CRS
vrt_transform, vrt_width, vrt_height = RasterDriver._calculate_default_transform(
src.crs, target_crs, src.width, src.height, *src.bounds
)
vrt = WarpedVRT(
src, crs=target_crs, resampling=rs_method, transform=vrt_transform,
width=vrt_width, height=vrt_height
)
return vrt
def test_compute_metadata_approximate(nodata_type, big_raster_file_nodata, big_raster_file_mask,
big_raster_file_nomask, raster_file_float):
from terracotta.drivers.raster_base import RasterDriver
if nodata_type == 'nodata':
raster_file = big_raster_file_nodata
elif nodata_type == 'masked':
raster_file = big_raster_file_mask
elif nodata_type == 'none':
raster_file = big_raster_file_nomask
elif nodata_type == 'nan':
raster_file = raster_file_float
with rasterio.open(str(raster_file)) as src:
data = src.read(1, masked=True)
valid_data = np.ma.masked_invalid(data).compressed()
convex_hull = convex_hull_exact(src)
# compare
if nodata_type == 'none':
with pytest.warns(UserWarning) as record:
mtd = RasterDriver.compute_metadata(str(raster_file), max_shape=(512, 512))
assert 'does not have a valid nodata value' in str(record[0].message)
else:
mtd = RasterDriver.compute_metadata(str(raster_file), max_shape=(512, 512))
np.testing.assert_allclose(mtd['valid_percentage'], 100 * valid_data.size / data.size, atol=1)
np.testing.assert_allclose(
mtd['range'], (valid_data.min(), valid_data.max()), atol=valid_data.max() / 100
)
np.testing.assert_allclose(mtd['mean'], valid_data.mean(), rtol=0.02)
np.testing.assert_allclose(mtd['stdev'], valid_data.std(), rtol=0.02)
np.testing.assert_allclose(
mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
atol=valid_data.max() / 100, rtol=0.02
)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 0.05
def test_default_transform():
from rasterio.warp import calculate_default_transform
from terracotta.drivers.raster_base import RasterDriver
args = ('epsg:4326', 'epsg:4326', 2 * 10**6, 10**6, -10, -10, 10, 10)
# GDAL defaults don't round-trip non-square pixels
_, rio_width, rio_height = calculate_default_transform(*args)
assert rio_width != args[2]
assert rio_height != args[3]
# we do!
our_transform, our_width, our_height = RasterDriver._calculate_default_transform(
*args)
assert our_width == args[2]
assert our_height == args[3]