def test_grid_factory():
grid: Grid = grid_factory("3/33600")
assert isinstance(grid, Grid)
assert grid.width == 3
assert grid.height == 3
assert grid.cols == 33600
assert grid.rows == 33600
assert grid.blockxsize == 480
assert grid.blockysize == 480
assert grid.crs.to_string() == "EPSG:4326"
grid: Grid = grid_factory("10/40000")
assert isinstance(grid, Grid)
assert grid.width == 10
assert grid.height == 10
assert grid.cols == 40000
assert grid.rows == 40000
assert grid.blockxsize == 400
assert grid.blockysize == 400
assert grid.crs.to_string() == "EPSG:4326"
grid: Grid = grid_factory("8/32000")
assert isinstance(grid, Grid)
assert grid.width == 8
assert grid.height == 8
assert grid.cols == 32000
assert grid.rows == 32000
assert grid.blockxsize == 400
assert grid.blockysize == 400
assert grid.crs.to_string() == "EPSG:4326"
grid: Grid = grid_factory("90/27008")
assert isinstance(grid, Grid)
assert grid.width == 90
assert grid.height == 90
assert grid.cols == 27008
assert grid.rows == 27008
assert grid.blockxsize == 128
assert grid.blockysize == 128
assert grid.crs.to_string() == "EPSG:4326"
grid: Grid = grid_factory("90/9984")
assert isinstance(grid, Grid)
assert grid.width == 90
assert grid.height == 90
assert grid.cols == 9984
assert grid.rows == 9984
assert grid.blockxsize == 416
assert grid.blockysize == 416
assert grid.crs.to_string() == "EPSG:4326"
def test_get_tile_id():
# Grid IDs for 10x10 degree grid, 40000x40000 pixels
grid: Grid = grid_factory("10/40000")
assert isinstance(grid, LatLngGrid)
grid_id: str = grid.xy_to_tile_id(0, 0)
assert grid_id == "00N_000E"
grid_id = grid.xy_to_tile_id(1, 1)
assert grid_id == "10N_000E"
grid_id = grid.xy_to_tile_id(-1, -1)
assert grid_id == "00N_010W"
grid_id = grid.xy_to_tile_id(90, 90)
assert grid_id == "90N_090E"
with pytest.raises(AssertionError):
grid.xy_to_tile_id(-90, -90)
# Grid IDs for 8x8 degree grid, 32000x32000 pixels
# Grid edges do not intersect with equator or central meridian
grid = grid_factory("8/32000")
assert isinstance(grid, LatLngGrid)
grid_id = grid.xy_to_tile_id(0, 0)
assert grid_id == "04N_004W"
grid_id = grid.xy_to_tile_id(1, 1)
assert grid_id == "04N_004W"
grid_id = grid.xy_to_tile_id(-1, -1)
assert grid_id == "04N_004W"
grid_id = grid.xy_to_tile_id(-5, 5)
assert grid_id == "12N_012W"
grid_id = grid.xy_to_tile_id(5, -5)
assert grid_id == "04S_004E"
grid_id = grid.xy_to_tile_id(-1, -1)
assert grid_id == "04N_004W"
with pytest.raises(AssertionError):
grid.xy_to_tile_id(90, 90)
with pytest.raises(AssertionError):
grid.xy_to_tile_id(-90, -90)
def test_wm_grids():
grid = grid_factory("zoom_1")
assert isinstance(grid, WebMercatorGrid)
assert len(grid.get_tile_ids()) == 1 == grid.nb_tiles
grid = grid_factory("zoom_10")
assert isinstance(grid, WebMercatorGrid)
assert len(grid.get_tile_ids()) == 16 == grid.nb_tiles
grid = grid_factory("zoom_14")
assert isinstance(grid, WebMercatorGrid)
assert len(grid.get_tile_ids()) == 4096 == grid.nb_tiles
with pytest.raises(ValueError):
grid_factory("zoom_30")
def layer_factory(layer_def: LayerModel) -> Layer:
layer_constructor = {"vector": VectorSrcLayer, "raster": RasterSrcLayer}
source_type: str = layer_def.source_type
grid: Grid = grid_factory(layer_def.grid)
try:
layer = layer_constructor[source_type](layer_def, grid)
except KeyError:
raise NotImplementedError(
f"Cannot create layer. Source type {source_type} not implemented.")
return layer
def resample(
dataset: str = Option(..., help="Dataset name."),
version: str = Option(..., help="Version string."),
source_uri: str = Option(..., help="URI of asset's tiles.geojson."),
resampling_method: str = Option(..., help="Resampling method to use."),
target_zoom: int = Option(..., help="Target zoom level."),
target_prefix: str = Option(..., help="Destination S3 prefix."),
):
"""Resample source raster tile set to target zoom level."""
log_queue = multiprocessing.Manager().Queue()
listener = multiprocessing.Process(target=log_listener,
args=(log_queue, listener_configurer))
listener.start()
log_client_configurer(log_queue)
logger = logging.getLogger("main")
logger.log(logging.INFO, f"Reprojecting/resampling tiles in {source_uri}")
logger.log(
logging.INFO,
f"# procs:{NUM_PROCESSES} MEM_PER_PROC:{MEM_PER_PROC} WARP_MEM:{WARP_MEM} CACHE_MEM:{CACHE_MEM}",
)
src_tiles_info = get_source_tiles_info(source_uri)
if not src_tiles_info:
logger.log(logging.INFO, "No input files! I guess we're good then.")
return
base_dir = os.path.curdir
source_dir = os.path.join(base_dir, "source_tiles")
os.makedirs(source_dir, exist_ok=True)
# First download all the source tiles
dl_process_args = ((tile_info[0], source_dir, log_queue,
log_client_configurer) for tile_info in src_tiles_info)
# Cannot use normal pool here since we run sub-processes
# https://stackoverflow.com/a/61470465/1410317
tile_paths: List[str] = list()
with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
for tile_path in executor.map(download_tile, dl_process_args):
tile_paths.append(tile_path)
logger.log(logging.INFO,
f"Finished downloading source tile to {tile_path}")
# Create VRTs for each of the bands of each of the input files
# In this case we can assume each file has the same number of bands
input_vrts: List[str] = list()
with rasterio.open(tile_paths[0]) as input_file:
band_count = input_file.count
for i in range(band_count):
vrt_file_path = os.path.join(source_dir, f"source_band_{i+1}.vrt")
create_vrt(tile_paths, src_file_band=i + 1, vrt_path=vrt_file_path)
input_vrts.append(vrt_file_path)
# And finally the overall VRT
overall_vrt: str = create_vrt(input_vrts,
None,
os.path.join(source_dir, "everything.vrt"),
separate=True)
logger.log(logging.INFO, "VRTs created")
# Determine which tiles in the target CRS + zoom level intersect with the
# extent of the source
dest_proj_grid = grid_factory(f"zoom_{target_zoom}")
all_dest_proj_tile_ids = dest_proj_grid.get_tile_ids()
wm_extent = Polygon()
for tile_info in src_tiles_info:
left, bottom, right, top = reproject_bounds(
shape(tile_info[1]).bounds, CRS.from_epsg(4326),
CRS.from_epsg(3857))
wm_extent = unary_union([
wm_extent,
Polygon((
(left, top),
(right, top),
(right, bottom),
(left, bottom),
(left, top),
)),
])
target_tiles: List[Tuple[str, Bounds]] = list()
for tile_id in all_dest_proj_tile_ids:
left, bottom, right, top = dest_proj_grid.get_tile_bounds(tile_id)
tile_geom = Polygon(((left, top), (right, top), (right, bottom),
(left, bottom), (left, top)))
if tile_geom.intersects(
wm_extent) and not tile_geom.touches(wm_extent):
logger.log(logging.INFO, f"Tile {tile_id} intersects!")
target_tiles.append((tile_id, (left, bottom, right, top)))
logger.log(logging.INFO, f"Found {len(target_tiles)} intersecting tiles")
bucket, _ = get_s3_path_parts(source_uri)
process_tile_args = [(
tile_id,
tile_bounds,
resampling_method,
target_zoom,
overall_vrt,
bucket,
target_prefix,
log_queue,
log_client_configurer,
) for tile_id, tile_bounds in target_tiles]
# Cannot use normal pool here since we run sub-processes
# https://stackoverflow.com/a/61470465/1410317
with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor:
for tile_id in executor.map(process_tile, process_tile_args):
logger.log(logging.INFO, f"Finished processing tile {tile_id}")
# Now run pixetl_prep.create_geojsons to generate a tiles.geojson and
# extent.geojson in the target prefix.
create_geojsons_prefix = target_prefix.split(f"{dataset}/{version}/")[1]
logger.log(logging.INFO,
f"Uploading tiles.geojson to {create_geojsons_prefix}")
create_geojsons(list(), dataset, version, create_geojsons_prefix, True)
log_queue.put_nowait(None)
listener.join()
from typing import Set
from unittest import mock
from gfw_pixetl import layers
from gfw_pixetl.grids import LatLngGrid, grid_factory
from gfw_pixetl.models.pydantic import LayerModel, Metadata
from gfw_pixetl.pipes import RasterPipe
from gfw_pixetl.sources import Destination
from gfw_pixetl.tiles import RasterSrcTile
from tests.conftest import minimal_layer_dict
GRID_10 = grid_factory("10/40000")
GRID_1 = grid_factory("1/4000")
EMPTY_METADATA = Metadata(
extent=(0.0, 0.0, 0.0, 0.0),
width=0,
height=0,
pixelxsize=0.0,
pixelysize=0.0,
crs="",
driver="",
compression=None,
bands=list(),
)
def test_create_tiles_subset(PIPE_10x10):
with mock.patch.object(RasterPipe,
"get_grid_tiles",
return_value=_get_subset_tiles()):
with mock.patch.object(