def test_metatile_bbox(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(2, 2))
meta_tile = mgrid.meta_tile((0, 0, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244, 0.0, 0.0)
meta_tile = mgrid.meta_tile((1, 1, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244, 0.0, 0.0)
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(meta_tile.bbox, (0.0, 0.0, 10018754.171394622, 10018754.171394622))
def test_metatile_bbox(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(2, 2))
meta_tile = mgrid.meta_tile((0, 0, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244, 0.0, 0.0)
meta_tile = mgrid.meta_tile((1, 1, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244, 0.0, 0.0)
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(meta_tile.bbox, (0.0, 0.0, 10018754.171394622, 10018754.171394622))
def test_metagrid_tiles():
mgrid = MetaGrid(grid=TileGrid(), meta_size=(2, 2))
assert list(mgrid.meta_tile((0, 0, 0)).tile_patterns) == \
[((0, 0, 0), (0, 0))]
assert list(mgrid.meta_tile((0, 1, 1)).tile_patterns) == \
[((0, 1, 1), (0, 0)), ((1, 1, 1), (256, 0)),
((0, 0, 1), (0, 256)), ((1, 0, 1), (256, 256))]
assert list(mgrid.meta_tile((1, 2, 2)).tile_patterns) == \
[((0, 3, 2), (0, 0)), ((1, 3, 2), (256, 0)),
((0, 2, 2), (0, 256)), ((1, 2, 2), (256, 256))]
def test_metagrid_tiles():
mgrid = MetaGrid(grid=TileGrid(), meta_size=(2, 2))
assert list(mgrid.meta_tile((0, 0, 0)).tile_patterns) == \
[((0, 0, 0), (0, 0))]
assert list(mgrid.meta_tile((0, 1, 1)).tile_patterns) == \
[((0, 1, 1), (0, 0)), ((1, 1, 1), (256, 0)),
((0, 0, 1), (0, 256)), ((1, 0, 1), (256, 256))]
assert list(mgrid.meta_tile((1, 2, 2)).tile_patterns) == \
[((0, 3, 2), (0, 0)), ((1, 3, 2), (256, 0)),
((0, 2, 2), (0, 256)), ((1, 2, 2), (256, 256))]
def test_metagrid_tiles_w_meta_size():
mgrid = MetaGrid(grid=TileGrid(), meta_size=(4, 2))
assert list(mgrid.meta_tile((1, 2, 2)).tile_patterns) == \
[((0, 3, 2), (0, 0)), ((1, 3, 2), (256, 0)),
((2, 3, 2), (512, 0)), ((3, 3, 2), (768, 0)),
((0, 2, 2), (0, 256)), ((1, 2, 2), (256, 256)),
((2, 2, 2), (512, 256)), ((3, 2, 2), (768, 256))]
def test_metagrid_tiles_w_meta_size():
mgrid = MetaGrid(grid=TileGrid(), meta_size=(4, 2))
assert list(mgrid.meta_tile((1, 2, 2)).tile_patterns) == \
[((0, 3, 2), (0, 0)), ((1, 3, 2), (256, 0)),
((2, 3, 2), (512, 0)), ((3, 3, 2), (768, 0)),
((0, 2, 2), (0, 256)), ((1, 2, 2), (256, 256)),
((2, 2, 2), (512, 256)), ((3, 2, 2), (768, 256))]
class TestMetaTile(object):
def setup(self):
self.mgrid = MetaGrid(grid=tile_grid('EPSG:4326'),
meta_size=(2, 2),
meta_buffer=10)
def test_meta_tile(self):
meta_tile = self.mgrid.meta_tile((2, 0, 2))
eq_(meta_tile.size, (522, 512))
def test_metatile_bbox(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(2, 2))
meta_tile = mgrid.meta_tile((0, 0, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244,
0.0, 0.0)
meta_tile = mgrid.meta_tile((1, 1, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244,
0.0, 0.0)
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(
meta_tile.bbox, (0.0, 0.0, 10018754.171394622, 10018754.171394622))
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(4, 2))
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(
meta_tile.bbox, (0.0, 0.0, 20037508.342789244, 10018754.171394622))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(4, 2))
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(
meta_tile.bbox, (0.0, 0.0, 20037508.342789244, 10018754.171394622))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
class TestMetaTileSQRT2(object):
def setup(self):
self.grid = tile_grid('EPSG:4326', res_factor='sqrt2')
self.mgrid = MetaGrid(grid=self.grid, meta_size=(4, 4), meta_buffer=10)
def test_meta_tile(self):
meta_tile = self.mgrid.meta_tile((0, 0, 8))
eq_(meta_tile.size, (1034, 1034))
def test_metatile_bbox(self):
meta_tile = self.mgrid.meta_tile((0, 0, 2))
eq_(meta_tile.bbox, (-180, -90, 180, 90))
eq_(meta_tile.size, (512, 256))
eq_(meta_tile.grid_size, (2, 1))
eq_(meta_tile.tile_patterns, [((0, 0, 2), (0, 0)),
((1, 0, 2), (256, 0))])
meta_tile = self.mgrid.meta_tile((1, 0, 2))
eq_(meta_tile.bbox, (-180.0, -90, 180.0, 90.0))
eq_(meta_tile.size, (512, 256))
eq_(meta_tile.grid_size, (2, 1))
meta_tile = self.mgrid.meta_tile((0, 0, 3))
eq_(meta_tile.bbox, (-180.0, -90, 180.0, 90.0))
eq_(meta_tile.size, (724, 362))
eq_(meta_tile.tile_patterns, [((0, 1, 3), (0, -149)),
((1, 1, 3), (256, -149)),
((2, 1, 3), (512, -149)),
((0, 0, 3), (0, 107)),
((1, 0, 3), (256, 107)),
((2, 0, 3), (512, 107))])
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=self.grid, meta_size=(4, 2), meta_buffer=0)
meta_tile = mgrid.meta_tile((4, 3, 6))
eq_(meta_tile.bbox, (0.0, 0.0, 180.0, 90.0))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
eq_(meta_tile.tile_patterns, [((4, 3, 6), (0, 0)),
((5, 3, 6), (256, 0)),
((6, 3, 6), (512, 0)),
((7, 3, 6), (768, 0)),
((4, 2, 6), (0, 256)),
((5, 2, 6), (256, 256)),
((6, 2, 6), (512, 256)),
((7, 2, 6), (768, 256))])
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=self.grid, meta_size=(4, 2), meta_buffer=0)
meta_tile = mgrid.meta_tile((4, 3, 6))
eq_(meta_tile.bbox, (0.0, 0.0, 180.0, 90.0))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
eq_(meta_tile.tile_patterns, [((4, 3, 6), (0, 0)), ((5, 3, 6), (256, 0)),
((6, 3, 6), (512, 0)), ((7, 3, 6), (768, 0)), ((4, 2, 6), (0, 256)),
((5, 2, 6), (256, 256)), ((6, 2, 6), (512, 256)), ((7, 2, 6), (768, 256))])
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=self.grid, meta_size=(4, 2), meta_buffer=0)
meta_tile = mgrid.meta_tile((4, 3, 6))
eq_(meta_tile.bbox, (0.0, 0.0, 180.0, 90.0))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
eq_(meta_tile.tile_patterns, [((4, 3, 6), (0, 0)), ((5, 3, 6), (256, 0)),
((6, 3, 6), (512, 0)), ((7, 3, 6), (768, 0)), ((4, 2, 6), (0, 256)),
((5, 2, 6), (256, 256)), ((6, 2, 6), (512, 256)), ((7, 2, 6), (768, 256))])
class TestMetaTileSQRT2(object):
def setup(self):
self.grid = tile_grid('EPSG:4326', res_factor='sqrt2')
self.mgrid = MetaGrid(grid=self.grid, meta_size=(4, 4), meta_buffer=10)
def test_meta_tile(self):
meta_tile = self.mgrid.meta_tile((0, 0, 8))
eq_(meta_tile.size, (1034, 1034))
def test_metatile_bbox(self):
meta_tile = self.mgrid.meta_tile((0, 0, 2))
eq_(meta_tile.bbox, (-180, -90, 180, 90))
eq_(meta_tile.size, (512, 256))
eq_(meta_tile.grid_size, (2, 1))
eq_(meta_tile.tile_patterns, [((0, 0, 2), (0, 0)), ((1, 0, 2), (256, 0))])
meta_tile = self.mgrid.meta_tile((1, 0, 2))
eq_(meta_tile.bbox, (-180.0, -90, 180.0, 90.0))
eq_(meta_tile.size, (512, 256))
eq_(meta_tile.grid_size, (2, 1))
meta_tile = self.mgrid.meta_tile((0, 0, 3))
eq_(meta_tile.bbox, (-180.0, -90, 180.0, 90.0))
eq_(meta_tile.size, (724, 362))
eq_(meta_tile.tile_patterns, [((0, 1, 3), (0, -149)), ((1, 1, 3), (256, -149)),
((2, 1, 3), (512, -149)), ((0, 0, 3), (0, 107)), ((1, 0, 3), (256, 107)),
((2, 0, 3), (512, 107))])
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=self.grid, meta_size=(4, 2), meta_buffer=0)
meta_tile = mgrid.meta_tile((4, 3, 6))
eq_(meta_tile.bbox, (0.0, 0.0, 180.0, 90.0))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
eq_(meta_tile.tile_patterns, [((4, 3, 6), (0, 0)), ((5, 3, 6), (256, 0)),
((6, 3, 6), (512, 0)), ((7, 3, 6), (768, 0)), ((4, 2, 6), (0, 256)),
((5, 2, 6), (256, 256)), ((6, 2, 6), (512, 256)), ((7, 2, 6), (768, 256))])
class TestMetaTile(object):
def setup(self):
self.mgrid = MetaGrid(grid=tile_grid('EPSG:4326'), meta_size=(2, 2), meta_buffer=10)
def test_meta_tile(self):
meta_tile = self.mgrid.meta_tile((2, 0, 2))
eq_(meta_tile.size, (522, 512))
def test_metatile_bbox(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(2, 2))
meta_tile = mgrid.meta_tile((0, 0, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244, 0.0, 0.0)
meta_tile = mgrid.meta_tile((1, 1, 2))
assert meta_tile.bbox == (-20037508.342789244, -20037508.342789244, 0.0, 0.0)
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(meta_tile.bbox, (0.0, 0.0, 10018754.171394622, 10018754.171394622))
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(4, 2))
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(meta_tile.bbox, (0.0, 0.0, 20037508.342789244, 10018754.171394622))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
def test_metatile_non_default_meta_size(self):
mgrid = MetaGrid(grid=TileGrid(), meta_size=(4, 2))
meta_tile = mgrid.meta_tile((4, 5, 3))
assert_almost_equal_bbox(meta_tile.bbox, (0.0, 0.0, 20037508.342789244, 10018754.171394622))
eq_(meta_tile.size, (1024, 512))
eq_(meta_tile.grid_size, (4, 2))
class TestMetaGridGeodeticUL(object):
def setup(self):
self.tile_grid = tile_grid('EPSG:4326', origin='ul')
self.mgrid = MetaGrid(grid=self.tile_grid, meta_size=(2, 2), meta_buffer=10)
def test_tiles_level_0(self):
meta_tile = self.mgrid.meta_tile((0, 0, 0))
eq_(meta_tile.bbox, (-180, -90, 180, 90))
eq_(meta_tile.size, (256, 128))
eq_(meta_tile.grid_size, (1, 1))
eq_(meta_tile.tile_patterns, [((0, 0, 0), (0, 0))])
def test_tiles_level_1(self):
meta_tile = self.mgrid.meta_tile((0, 0, 1))
eq_(meta_tile.bbox, (-180, -90, 180, 90))
eq_(meta_tile.size, (512, 256))
eq_(meta_tile.grid_size, (2, 1))
eq_(list(meta_tile.tile_patterns),
[
((0, 0, 1), (0, 0)),
((1, 0, 1), (256, 0))
])
def test_tile_list_level_1(self):
eq_(list(self.mgrid.tile_list((0, 0, 1))),
[(0, 0, 1), (1, 0, 1)])
def test_tiles_level_2(self):
meta_tile = self.mgrid.meta_tile((0, 0, 2))
eq_(meta_tile.bbox, (-180, -90, 3.515625, 90))
eq_(meta_tile.size, (522, 512))
eq_(meta_tile.grid_size, (2, 2))
eq_(meta_tile.tile_patterns,
[
((0, 0, 2), (0, 0)),
((1, 0, 2), (256, 0)),
((0, 1, 2), (0, 256)),
((1, 1, 2), (256, 256)),
])
eq_(list(self.mgrid.meta_tile((2, 0, 2)).tile_patterns),
[
((2, 0, 2), (10, 0)),
((3, 0, 2), (266, 0)),
((2, 1, 2), (10, 256)),
((3, 1, 2), (266, 256)),
])
def test_tile_list_level_2(self):
eq_(list(self.mgrid.tile_list((0, 0, 2))),
[(0, 0, 2), (1, 0, 2), (0, 1, 2), (1, 1, 2)])
eq_(list(self.mgrid.tile_list((1, 1, 2))),
[(0, 0, 2), (1, 0, 2), (0, 1, 2), (1, 1, 2)])
def test_tiles_level_3(self):
meta_tile = self.mgrid.meta_tile((2, 0, 3))
eq_(meta_tile.bbox, (-91.7578125, -1.7578125, 1.7578125, 90))
eq_(meta_tile.size, (532, 522))
eq_(meta_tile.grid_size, (2, 2))
eq_(list(self.mgrid.meta_tile((2, 0, 3)).tile_patterns),
[
((2, 0, 3), (10, 0)),
((3, 0, 3), (266, 0)),
((2, 1, 3), (10, 256)),
((3, 1, 3), (266, 256)),
])
eq_(list(self.mgrid.meta_tile((2, 2, 3)).tile_patterns),
[
((2, 2, 3), (10, 10)),
((3, 2, 3), (266, 10)),
((2, 3, 3), (10, 266)),
((3, 3, 3), (266, 266)),
])
class TestMetaGridGeodetic(object):
def setup(self):
self.mgrid = MetaGrid(grid=tile_grid('EPSG:4326'), meta_size=(2, 2), meta_buffer=10)
def test_meta_bbox_level_0(self):
eq_(self.mgrid._meta_bbox((0, 0, 0)), ((-180, -90, 180, 90), (0, 0, 0, -128)))
eq_(self.mgrid._meta_bbox((0, 0, 0), limit_to_bbox=False),
((-194.0625, -104.0625, 194.0625, 284.0625), (10, 10, 10, 10)))
eq_(self.mgrid.meta_tile((0, 0, 0)).size, (256, 128))
def test_tiles_level_0(self):
meta_tile = self.mgrid.meta_tile((0, 0, 0))
eq_(meta_tile.size, (256, 128))
eq_(meta_tile.grid_size, (1, 1))
eq_(meta_tile.tile_patterns, [((0, 0, 0), (0, -128))])
def test_meta_bbox_level_1(self):
eq_(self.mgrid._meta_bbox((0, 0, 1)), ((-180, -90, 180, 90), (0, 0, 0, 0)))
eq_(self.mgrid._meta_bbox((0, 0, 1), limit_to_bbox=False),
((-187.03125, -97.03125, 187.03125, 97.03125), (10, 10, 10, 10)))
eq_(self.mgrid.meta_tile((0, 0, 1)).size, (512, 256))
def test_tiles_level_1(self):
eq_(list(self.mgrid.meta_tile((0, 0, 1)).tile_patterns),
[
((0, 0, 1), (0, 0)),
((1, 0, 1), (256, 0))
])
def test_tile_list_level_1(self):
eq_(list(self.mgrid.tile_list((0, 0, 1))),
[(0, 0, 1), (1, 0, 1)])
def test_meta_bbox_level_2(self):
eq_(self.mgrid._meta_bbox((0, 0, 2)), ((-180, -90, 3.515625, 90), (0, 0, 10, 0)))
eq_(self.mgrid._meta_bbox((0, 0, 2), limit_to_bbox=False),
((-183.515625, -93.515625, 3.515625, 93.515625), (10, 10, 10, 10)))
eq_(self.mgrid.meta_tile((0, 0, 2)).size, (522, 512))
eq_(self.mgrid._meta_bbox((2, 0, 2)), ((-3.515625, -90, 180, 90), (10, 0, 0, 0)))
meta_tile = self.mgrid.meta_tile((2, 0, 2))
eq_(meta_tile.size, (522, 512))
eq_(meta_tile.grid_size, (2, 2))
def test_tiles_level_2(self):
eq_(list(self.mgrid.meta_tile((0, 0, 2)).tile_patterns),
[
((0, 1, 2), (0, 0)),
((1, 1, 2), (256, 0)),
((0, 0, 2), (0, 256)),
((1, 0, 2), (256, 256)),
])
eq_(list(self.mgrid.meta_tile((2, 0, 2)).tile_patterns),
[
((2, 1, 2), (10, 0)),
((3, 1, 2), (266, 0)),
((2, 0, 2), (10, 256)),
((3, 0, 2), (266, 256)),
])
def test_tile_list_level_2(self):
eq_(list(self.mgrid.tile_list((0, 0, 2))),
[(0, 1, 2), (1, 1, 2), (0, 0, 2), (1, 0, 2)])
eq_(list(self.mgrid.tile_list((1, 1, 2))),
[(0, 1, 2), (1, 1, 2), (0, 0, 2), (1, 0, 2)])
def test_tiles_level_3(self):
eq_(list(self.mgrid.meta_tile((2, 0, 3)).tile_patterns),
[
((2, 1, 3), (10, 10)),
((3, 1, 3), (266, 10)),
((2, 0, 3), (10, 266)),
((3, 0, 3), (266, 266)),
])
eq_(list(self.mgrid.meta_tile((2, 2, 3)).tile_patterns),
[
((2, 3, 3), (10, 0)),
((3, 3, 3), (266, 0)),
((2, 2, 3), (10, 256)),
((3, 2, 3), (266, 256)),
])
class TileWalker(object):
"""
TileWalker traverses through all tiles in a tile grid and calls worker_pool.process
for each (meta) tile. It traverses the tile grid (pyramid) depth-first.
Intersection with coverages are checked before handling subtiles in the next level,
allowing to determine if all subtiles should be seeded or skipped.
"""
def __init__(self,
task,
worker_pool,
handle_stale=False,
handle_uncached=False,
work_on_metatiles=True,
skip_geoms_for_last_levels=0,
progress_logger=None,
seed_progress=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
if num_seed_levels >= 4:
self.report_till_level = task.levels[num_seed_levels - 2]
else:
self.report_till_level = task.levels[num_seed_levels - 1]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (
1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid,
meta_size=meta_size,
meta_buffer=0)
self.count = 0
self.seed_progress = seed_progress or SeedProgress()
# It is possible that we 'walk' through the same tile multiple times
# when seeding irregular tile grids[0]. limit_sub_bbox prevents that we
# recurse into the same area multiple times, but it is still possible
# that a tile is processed multiple times. Locking prevents that a tile
# is seeded multiple times, but it is possible that we count the same tile
# multiple times (in dry-mode, or while the tile is in the process queue).
# Tile counts can be off by 280% with sqrt2 grids.
# We keep a small cache of already processed tiles to skip most duplicates.
# A simple cache of 64 tile coordinates for each level already brings the
# difference down to ~8%, which is good enough and faster than a more
# sophisticated FIFO cache with O(1) lookup, or even caching all tiles.
# [0] irregular tile grids: where one tile does not have exactly 4 subtiles
# Typically when you use res_factor, or a custom res list.
self.seeded_tiles = {l: deque(maxlen=64) for l in task.levels}
def walk(self):
assert self.handle_stale or self.handle_uncached
bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs)
if self.seed_progress.already_processed():
# nothing to seed
self.seed_progress.step_forward()
else:
try:
self._walk(bbox, self.task.levels)
except StopProcess:
pass
self.report_progress(self.task.levels[0], self.task.coverage.bbox)
def _walk(self, cur_bbox, levels, current_level=0, all_subtiles=False):
"""
:param cur_bbox: the bbox to seed in this call
:param levels: list of levels to seed
:param all_subtiles: seed all subtiles and do not check for
intersections with bbox/geom
"""
bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(
cur_bbox, current_level)
total_subtiles = tiles[0] * tiles[1]
if len(levels) < self.skip_geoms_for_last_levels:
# do not filter in last levels
all_subtiles = True
subtiles = self._filter_subtiles(subtiles, all_subtiles)
if current_level in levels and current_level <= self.report_till_level:
self.report_progress(current_level, cur_bbox)
if not self.seed_progress.running():
if current_level in levels:
self.report_progress(current_level, cur_bbox)
self.tile_mgr.cleanup()
raise StopProcess()
process = False
if current_level in levels:
levels = levels[1:]
process = True
for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
if subtile is None: # no intersection
self.seed_progress.step_forward(total_subtiles)
continue
if levels: # recurse to next level
sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
if intersection == CONTAINS:
all_subtiles = True
else:
all_subtiles = False
with self.seed_progress.step_down(i, total_subtiles):
if self.seed_progress.already_processed():
self.seed_progress.step_forward()
else:
self._walk(sub_bbox,
levels,
current_level=current_level + 1,
all_subtiles=all_subtiles)
if not process:
continue
# check if subtile was already processed. see comment in __init__
if subtile in self.seeded_tiles[current_level]:
if not levels:
self.seed_progress.step_forward(total_subtiles)
continue
self.seeded_tiles[current_level].appendleft(subtile)
if not self.work_on_metatiles:
# collect actual tiles
handle_tiles = self.grid.tile_list(subtile)
else:
handle_tiles = [subtile]
if self.handle_uncached:
handle_tiles = [
t for t in handle_tiles
if t is not None and not self.tile_mgr.is_cached(t)
]
elif self.handle_stale:
handle_tiles = [
t for t in handle_tiles
if t is not None and self.tile_mgr.is_stale(t)
]
if handle_tiles:
self.count += 1
self.worker_pool.process(handle_tiles, self.seed_progress)
if not levels:
self.seed_progress.step_forward(total_subtiles)
if len(levels) >= 4:
# call cleanup to close open caches
# for connection based caches
self.tile_mgr.cleanup()
def report_progress(self, level, bbox):
if self.progress_logger:
self.progress_logger.log_progress(
self.seed_progress, level, bbox,
self.count * self.tiles_per_metatile)
def _filter_subtiles(self, subtiles, all_subtiles):
"""
Return an iterator with all sub tiles.
Yields (None, None, None) for non-intersecting tiles,
otherwise (subtile, subtile_bbox, intersection).
"""
for subtile in subtiles:
if subtile is None:
yield None, None, None
else:
sub_bbox = self.grid.meta_tile(subtile).bbox
if all_subtiles:
intersection = CONTAINS
else:
intersection = self.task.intersects(sub_bbox)
if intersection:
yield subtile, sub_bbox, intersection
else:
yield None, None, None
class TileWalker(object):
def __init__(self,
task,
worker_pool,
handle_stale=False,
handle_uncached=False,
work_on_metatiles=True,
skip_geoms_for_last_levels=0,
progress_logger=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
self.report_till_level = task.levels[int(num_seed_levels * 0.8)]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (
1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid,
meta_size=meta_size,
meta_buffer=0)
self.progress = 0.0
self.eta = ETA()
self.count = 0
def walk(self):
assert self.handle_stale or self.handle_uncached
bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs)
self._walk(bbox, self.task.levels)
self.report_progress(self.task.levels[0], self.task.coverage.bbox)
def _walk(self,
cur_bbox,
levels,
progess_str='',
progress=1.0,
all_subtiles=False):
"""
:param cur_bbox: the bbox to seed in this call
:param levels: list of levels to seed
:param all_subtiles: seed all subtiles and do not check for
intersections with bbox/geom
"""
current_level, levels = levels[0], levels[1:]
bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(
cur_bbox, current_level)
total_subtiles = tiles[0] * tiles[1]
if len(levels) < self.skip_geoms_for_last_levels:
# do not filter in last levels
all_subtiles = True
subtiles = self._filter_subtiles(subtiles, all_subtiles)
if current_level <= self.report_till_level:
self.report_progress(current_level, cur_bbox)
progress = progress / total_subtiles
for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
if subtile is None: # no intersection
self.progress += progress
continue
if levels: # recurse to next level
sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
cur_progess_str = progess_str + status_symbol(
i, total_subtiles)
if intersection == CONTAINS:
all_subtiles = True
else:
all_subtiles = False
self._walk(sub_bbox,
levels,
cur_progess_str,
all_subtiles=all_subtiles,
progress=progress)
if not self.work_on_metatiles:
# collect actual tiles
handle_tiles = self.grid.tile_list(subtile)
else:
handle_tiles = [subtile]
if self.handle_uncached:
handle_tiles = [
t for t in handle_tiles
if t is not None and not self.tile_mgr.is_cached(t)
]
elif self.handle_stale:
handle_tiles = [
t for t in handle_tiles
if t is not None and self.tile_mgr.is_stale(t)
]
if handle_tiles:
self.count += 1
self.worker_pool.process(
handle_tiles, (progess_str, self.progress, self.eta))
if not levels:
self.progress += progress
if len(levels) >= 4:
# call cleanup to close open caches
# for connection based caches
self.tile_mgr.cleanup()
self.eta.update(self.progress)
def report_progress(self, level, bbox):
if self.progress_logger:
self.progress_logger.log_progress(
self.progress, level, bbox,
self.count * self.tiles_per_metatile, self.eta)
def _filter_subtiles(self, subtiles, all_subtiles):
"""
Return an iterator with all sub tiles.
Yields (None, None, None) for non-intersecting tiles,
otherwise (subtile, subtile_bbox, intersection).
"""
for subtile in subtiles:
if subtile is None:
yield None, None, None
else:
sub_bbox = self.grid.meta_tile(subtile).bbox
if all_subtiles:
intersection = CONTAINS
else:
intersection = self.task.intersects(sub_bbox)
if intersection:
yield subtile, sub_bbox, intersection
else:
yield None, None, None
class TestMetaGridGeodetic(object):
def setup(self):
self.mgrid = MetaGrid(grid=tile_grid('EPSG:4326'),
meta_size=(2, 2),
meta_buffer=10)
def test_meta_bbox_level_0(self):
eq_(self.mgrid._meta_bbox((0, 0, 0)),
((-180, -90, 180, 90), (0, 0, 0, -128)))
eq_(self.mgrid._meta_bbox((0, 0, 0), limit_to_bbox=False),
((-194.0625, -104.0625, 194.0625, 284.0625), (10, 10, 10, 10)))
eq_(self.mgrid.meta_tile((0, 0, 0)).size, (256, 128))
def test_tiles_level_0(self):
meta_tile = self.mgrid.meta_tile((0, 0, 0))
eq_(meta_tile.size, (256, 128))
eq_(meta_tile.grid_size, (1, 1))
eq_(meta_tile.tile_patterns, [((0, 0, 0), (0, -128))])
def test_meta_bbox_level_1(self):
eq_(self.mgrid._meta_bbox((0, 0, 1)),
((-180, -90, 180, 90), (0, 0, 0, 0)))
eq_(self.mgrid._meta_bbox((0, 0, 1), limit_to_bbox=False),
((-187.03125, -97.03125, 187.03125, 97.03125), (10, 10, 10, 10)))
eq_(self.mgrid.meta_tile((0, 0, 1)).size, (512, 256))
def test_tiles_level_1(self):
eq_(list(self.mgrid.meta_tile((0, 0, 1)).tile_patterns),
[((0, 0, 1), (0, 0)), ((1, 0, 1), (256, 0))])
def test_tile_list_level_1(self):
eq_(list(self.mgrid.tile_list((0, 0, 1))), [(0, 0, 1), (1, 0, 1)])
def test_meta_bbox_level_2(self):
eq_(self.mgrid._meta_bbox((0, 0, 2)),
((-180, -90, 3.515625, 90), (0, 0, 10, 0)))
eq_(self.mgrid._meta_bbox((0, 0, 2), limit_to_bbox=False),
((-183.515625, -93.515625, 3.515625, 93.515625), (10, 10, 10, 10)))
eq_(self.mgrid.meta_tile((0, 0, 2)).size, (522, 512))
eq_(self.mgrid._meta_bbox((2, 0, 2)),
((-3.515625, -90, 180, 90), (10, 0, 0, 0)))
meta_tile = self.mgrid.meta_tile((2, 0, 2))
eq_(meta_tile.size, (522, 512))
eq_(meta_tile.grid_size, (2, 2))
def test_tiles_level_2(self):
eq_(list(self.mgrid.meta_tile((0, 0, 2)).tile_patterns), [
((0, 1, 2), (0, 0)),
((1, 1, 2), (256, 0)),
((0, 0, 2), (0, 256)),
((1, 0, 2), (256, 256)),
])
eq_(list(self.mgrid.meta_tile((2, 0, 2)).tile_patterns), [
((2, 1, 2), (10, 0)),
((3, 1, 2), (266, 0)),
((2, 0, 2), (10, 256)),
((3, 0, 2), (266, 256)),
])
def test_tile_list_level_2(self):
eq_(list(self.mgrid.tile_list((0, 0, 2))), [(0, 1, 2), (1, 1, 2),
(0, 0, 2), (1, 0, 2)])
eq_(list(self.mgrid.tile_list((1, 1, 2))), [(0, 1, 2), (1, 1, 2),
(0, 0, 2), (1, 0, 2)])
def test_tiles_level_3(self):
eq_(list(self.mgrid.meta_tile((2, 0, 3)).tile_patterns), [
((2, 1, 3), (10, 10)),
((3, 1, 3), (266, 10)),
((2, 0, 3), (10, 266)),
((3, 0, 3), (266, 266)),
])
eq_(list(self.mgrid.meta_tile((2, 2, 3)).tile_patterns), [
((2, 3, 3), (10, 0)),
((3, 3, 3), (266, 0)),
((2, 2, 3), (10, 256)),
((3, 2, 3), (266, 256)),
])
class TestMetaGridGeodeticUL(object):
def setup(self):
self.tile_grid = tile_grid('EPSG:4326', origin='ul')
self.mgrid = MetaGrid(grid=self.tile_grid,
meta_size=(2, 2),
meta_buffer=10)
def test_tiles_level_0(self):
meta_tile = self.mgrid.meta_tile((0, 0, 0))
eq_(meta_tile.bbox, (-180, -90, 180, 90))
eq_(meta_tile.size, (256, 128))
eq_(meta_tile.grid_size, (1, 1))
eq_(meta_tile.tile_patterns, [((0, 0, 0), (0, 0))])
def test_tiles_level_1(self):
meta_tile = self.mgrid.meta_tile((0, 0, 1))
eq_(meta_tile.bbox, (-180, -90, 180, 90))
eq_(meta_tile.size, (512, 256))
eq_(meta_tile.grid_size, (2, 1))
eq_(list(meta_tile.tile_patterns), [((0, 0, 1), (0, 0)),
((1, 0, 1), (256, 0))])
def test_tile_list_level_1(self):
eq_(list(self.mgrid.tile_list((0, 0, 1))), [(0, 0, 1), (1, 0, 1)])
def test_tiles_level_2(self):
meta_tile = self.mgrid.meta_tile((0, 0, 2))
eq_(meta_tile.bbox, (-180, -90, 3.515625, 90))
eq_(meta_tile.size, (522, 512))
eq_(meta_tile.grid_size, (2, 2))
eq_(meta_tile.tile_patterns, [
((0, 0, 2), (0, 0)),
((1, 0, 2), (256, 0)),
((0, 1, 2), (0, 256)),
((1, 1, 2), (256, 256)),
])
eq_(list(self.mgrid.meta_tile((2, 0, 2)).tile_patterns), [
((2, 0, 2), (10, 0)),
((3, 0, 2), (266, 0)),
((2, 1, 2), (10, 256)),
((3, 1, 2), (266, 256)),
])
def test_tile_list_level_2(self):
eq_(list(self.mgrid.tile_list((0, 0, 2))), [(0, 0, 2), (1, 0, 2),
(0, 1, 2), (1, 1, 2)])
eq_(list(self.mgrid.tile_list((1, 1, 2))), [(0, 0, 2), (1, 0, 2),
(0, 1, 2), (1, 1, 2)])
def test_tiles_level_3(self):
meta_tile = self.mgrid.meta_tile((2, 0, 3))
eq_(meta_tile.bbox, (-91.7578125, -1.7578125, 1.7578125, 90))
eq_(meta_tile.size, (532, 522))
eq_(meta_tile.grid_size, (2, 2))
eq_(list(self.mgrid.meta_tile((2, 0, 3)).tile_patterns), [
((2, 0, 3), (10, 0)),
((3, 0, 3), (266, 0)),
((2, 1, 3), (10, 256)),
((3, 1, 3), (266, 256)),
])
eq_(list(self.mgrid.meta_tile((2, 2, 3)).tile_patterns), [
((2, 2, 3), (10, 10)),
((3, 2, 3), (266, 10)),
((2, 3, 3), (10, 266)),
((3, 3, 3), (266, 266)),
])
class TileWalker(object):
def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False,
work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None,
seed_progress=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
self.report_till_level = task.levels[int(num_seed_levels * 0.8)]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0)
self.count = 0
self.seed_progress = seed_progress or SeedProgress()
def walk(self):
assert self.handle_stale or self.handle_uncached
bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs)
if self.seed_progress.already_processed():
# nothing to seed
self.seed_progress.step_forward()
else:
try:
self._walk(bbox, self.task.levels)
except StopProcess:
pass
self.report_progress(self.task.levels[0], self.task.coverage.bbox)
def _walk(self, cur_bbox, levels, all_subtiles=False):
"""
:param cur_bbox: the bbox to seed in this call
:param levels: list of levels to seed
:param all_subtiles: seed all subtiles and do not check for
intersections with bbox/geom
"""
current_level, levels = levels[0], levels[1:]
bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(cur_bbox, current_level)
total_subtiles = tiles[0] * tiles[1]
if len(levels) < self.skip_geoms_for_last_levels:
# do not filter in last levels
all_subtiles = True
subtiles = self._filter_subtiles(subtiles, all_subtiles)
if current_level <= self.report_till_level:
self.report_progress(current_level, cur_bbox)
if not self.seed_progress.running():
self.report_progress(current_level, cur_bbox)
self.tile_mgr.cleanup()
raise StopProcess()
for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
if subtile is None: # no intersection
self.seed_progress.step_forward(total_subtiles)
continue
if levels: # recurse to next level
sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
if intersection == CONTAINS:
all_subtiles = True
else:
all_subtiles = False
with self.seed_progress.step_down(i, total_subtiles):
if self.seed_progress.already_processed():
self.seed_progress.step_forward()
else:
self._walk(sub_bbox, levels,
all_subtiles=all_subtiles)
if not self.work_on_metatiles:
# collect actual tiles
handle_tiles = self.grid.tile_list(subtile)
else:
handle_tiles = [subtile]
if self.handle_uncached:
handle_tiles = [t for t in handle_tiles if
t is not None and
not self.tile_mgr.is_cached(t)]
elif self.handle_stale:
handle_tiles = [t for t in handle_tiles if
t is not None and
self.tile_mgr.is_stale(t)]
if handle_tiles:
self.count += 1
self.worker_pool.process(handle_tiles, self.seed_progress)
if not levels:
self.seed_progress.step_forward(total_subtiles)
if len(levels) >= 4:
# call cleanup to close open caches
# for connection based caches
self.tile_mgr.cleanup()
def report_progress(self, level, bbox):
if self.progress_logger:
self.progress_logger.log_progress(self.seed_progress, level, bbox,
self.count * self.tiles_per_metatile)
def _filter_subtiles(self, subtiles, all_subtiles):
"""
Return an iterator with all sub tiles.
Yields (None, None, None) for non-intersecting tiles,
otherwise (subtile, subtile_bbox, intersection).
"""
for subtile in subtiles:
if subtile is None:
yield None, None, None
else:
sub_bbox = self.grid.meta_tile(subtile).bbox
if all_subtiles:
intersection = CONTAINS
else:
intersection = self.task.intersects(sub_bbox)
if intersection:
yield subtile, sub_bbox, intersection
else:
yield None, None, None
class TileWalker(object):
"""
TileWalker traverses through all tiles in a tile grid and calls worker_pool.process
for each (meta) tile. It traverses the tile grid (pyramid) depth-first.
Intersection with coverages are checked before handling subtiles in the next level,
allowing to determine if all subtiles should be seeded or skipped.
"""
def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False,
work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None,
seed_progress=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
if num_seed_levels >= 4:
self.report_till_level = task.levels[num_seed_levels-2]
else:
self.report_till_level = task.levels[num_seed_levels-1]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0)
self.count = 0
self.seed_progress = seed_progress or SeedProgress()
# It is possible that we 'walk' through the same tile multiple times
# when seeding irregular tile grids[0]. limit_sub_bbox prevents that we
# recurse into the same area multiple times, but it is still possible
# that a tile is processed multiple times. Locking prevents that a tile
# is seeded multiple times, but it is possible that we count the same tile
# multiple times (in dry-mode, or while the tile is in the process queue).
# Tile counts can be off by 280% with sqrt2 grids.
# We keep a small cache of already processed tiles to skip most duplicates.
# A simple cache of 64 tile coordinates for each level already brings the
# difference down to ~8%, which is good enough and faster than a more
# sophisticated FIFO cache with O(1) lookup, or even caching all tiles.
# [0] irregular tile grids: where one tile does not have exactly 4 subtiles
# Typically when you use res_factor, or a custom res list.
self.seeded_tiles = {l: deque(maxlen=64) for l in task.levels}
def walk(self):
assert self.handle_stale or self.handle_uncached
bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs)
if self.seed_progress.already_processed():
# nothing to seed
self.seed_progress.step_forward()
else:
try:
self._walk(bbox, self.task.levels)
except StopProcess:
pass
self.report_progress(self.task.levels[0], self.task.coverage.bbox)
def _walk(self, cur_bbox, levels, current_level=0, all_subtiles=False):
"""
:param cur_bbox: the bbox to seed in this call
:param levels: list of levels to seed
:param all_subtiles: seed all subtiles and do not check for
intersections with bbox/geom
"""
bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(cur_bbox, current_level)
total_subtiles = tiles[0] * tiles[1]
if len(levels) < self.skip_geoms_for_last_levels:
# do not filter in last levels
all_subtiles = True
subtiles = self._filter_subtiles(subtiles, all_subtiles)
if current_level in levels and current_level <= self.report_till_level:
self.report_progress(current_level, cur_bbox)
if not self.seed_progress.running():
if current_level in levels:
self.report_progress(current_level, cur_bbox)
self.tile_mgr.cleanup()
raise StopProcess()
process = False;
if current_level in levels:
levels = levels[1:]
process = True
for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
if subtile is None: # no intersection
self.seed_progress.step_forward(total_subtiles)
continue
if levels: # recurse to next level
sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
if intersection == CONTAINS:
all_subtiles = True
else:
all_subtiles = False
with self.seed_progress.step_down(i, total_subtiles):
if self.seed_progress.already_processed():
self.seed_progress.step_forward()
else:
self._walk(sub_bbox, levels, current_level=current_level+1,
all_subtiles=all_subtiles)
if not process:
continue
# check if subtile was already processed. see comment in __init__
if subtile in self.seeded_tiles[current_level]:
if not levels:
self.seed_progress.step_forward(total_subtiles)
continue
self.seeded_tiles[current_level].appendleft(subtile)
if not self.work_on_metatiles:
# collect actual tiles
handle_tiles = self.grid.tile_list(subtile)
else:
handle_tiles = [subtile]
if self.handle_uncached:
handle_tiles = [t for t in handle_tiles if
t is not None and
not self.tile_mgr.is_cached(t)]
elif self.handle_stale:
handle_tiles = [t for t in handle_tiles if
t is not None and
self.tile_mgr.is_stale(t)]
if handle_tiles:
self.count += 1
self.worker_pool.process(handle_tiles, self.seed_progress)
if not levels:
self.seed_progress.step_forward(total_subtiles)
if len(levels) >= 4:
# call cleanup to close open caches
# for connection based caches
self.tile_mgr.cleanup()
def report_progress(self, level, bbox):
if self.progress_logger:
self.progress_logger.log_progress(self.seed_progress, level, bbox,
self.count * self.tiles_per_metatile)
def _filter_subtiles(self, subtiles, all_subtiles):
"""
Return an iterator with all sub tiles.
Yields (None, None, None) for non-intersecting tiles,
otherwise (subtile, subtile_bbox, intersection).
"""
for subtile in subtiles:
if subtile is None:
yield None, None, None
else:
sub_bbox = self.grid.meta_tile(subtile).bbox
if all_subtiles:
intersection = CONTAINS
else:
intersection = self.task.intersects(sub_bbox)
if intersection:
yield subtile, sub_bbox, intersection
else:
yield None, None, None
