def test_seed_with_res_list(self):
if not load_wkt: raise SkipTest('no shapely installed')
# box from 10 10 to 80 80 with small spike/corner to -10 60 (upper left)
geom = load_wkt(
"POLYGON((10 10, 10 50, -10 60, 10 80, 80 80, 80 10, 10 10))")
self.grid = TileGrid(
SRS(4326),
bbox=[-180, -90, 180, 90],
res=[360 / 256, 360 / 720, 360 / 2000, 360 / 5000, 360 / 8000])
self.tile_mgr = TileManager(self.grid,
MockCache(), [self.source],
'png',
locker=DummyLocker())
task = self.make_geom_task(geom, SRS(4326), [0, 1, 2, 3, 4])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 5)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.grid.grid_sizes[1], (3, 2))
eq_(self.seed_pool.seeded_tiles[1],
set([(1, 0), (1, 1), (2, 0), (2, 1)]))
eq_(self.grid.grid_sizes[2], (8, 4))
eq_(self.seed_pool.seeded_tiles[2],
set([(4, 2), (5, 2), (4, 3), (5, 3), (3, 3)]))
eq_(self.grid.grid_sizes[3], (20, 10))
eq_(len(self.seed_pool.seeded_tiles[3]), 5 * 5 + 2)
def process(self):
log.debug('Start raster import process. Task %d' % self.task_id)
try:
with self.task() as task:
seed_task = self.create_seed_task(task)
progress_logger = self.create_progress_logger(task)
start_progress = parse_progress_identifier(task.seed_progress)
self.terminate_event = threading.Event()
seed_progress = SeedProgress(self.terminate_event, start_progress)
if seed_task.refresh_timestamp is not None:
seed_task.tile_manager._expire_timestamp = seed_task.refresh_timestamp
seed_worker_factory = partial(TileSeedWorker, self.terminate_event)
self.tile_worker_pool = TileWorkerPool(seed_task, seed_worker_factory,
size=4, progress_logger=progress_logger)
tile_walker = TileWalker(seed_task, self.tile_worker_pool, handle_uncached=True,
progress_logger=progress_logger,
seed_progress=seed_progress)
tile_walker.walk()
if not self.terminate_event.is_set():
self.tile_worker_pool.stop()
if not self.terminate_event.is_set():
self.task_done()
except Exception, e:
self.task_failed(e)
def process(self):
log.debug('Start raster import process. Task %d' % self.task_id)
try:
with self.task() as task:
seed_task = self.create_seed_task(task)
progress_logger = self.create_progress_logger(task)
start_progress = parse_progress_identifier(task.seed_progress)
self.terminate_event = threading.Event()
seed_progress = SeedProgress(self.terminate_event,
start_progress)
if seed_task.refresh_timestamp is not None:
seed_task.tile_manager._expire_timestamp = seed_task.refresh_timestamp
seed_worker_factory = partial(TileSeedWorker,
self.terminate_event)
self.tile_worker_pool = TileWorkerPool(
seed_task,
seed_worker_factory,
size=4,
progress_logger=progress_logger)
tile_walker = TileWalker(seed_task,
self.tile_worker_pool,
handle_uncached=True,
progress_logger=progress_logger,
seed_progress=seed_progress)
tile_walker.walk()
if not self.terminate_event.is_set():
self.tile_worker_pool.stop()
if not self.terminate_event.is_set():
self.task_done()
except Exception, e:
self.task_failed(e)
def tilewalker_cleanup(task,
dry_run,
concurrency,
skip_geoms_for_last_levels,
progress_logger=None,
seed_progress=None):
"""
Cleanup tiles with tile traversal.
"""
task.tile_manager._expire_timestamp = task.remove_timestamp
task.tile_manager.minimize_meta_requests = False
tile_worker_pool = TileWorkerPool(task,
TileCleanupWorker,
progress_logger=progress_logger,
dry_run=dry_run,
size=concurrency)
tile_walker = TileWalker(
task,
tile_worker_pool,
handle_stale=True,
work_on_metatiles=False,
progress_logger=progress_logger,
skip_geoms_for_last_levels=skip_geoms_for_last_levels,
seed_progress=seed_progress)
try:
tile_walker.walk()
except KeyboardInterrupt:
tile_worker_pool.stop(force=True)
raise
finally:
tile_worker_pool.stop()
def test_seed_with_res_list(self):
# box from 10 10 to 80 80 with small spike/corner to -10 60 (upper left)
geom = load_wkt(
"POLYGON((10 10, 10 50, -10 60, 10 80, 80 80, 80 10, 10 10))")
self.grid = TileGrid(
SRS(4326),
bbox=[-180, -90, 180, 90],
res=[360 / 256, 360 / 720, 360 / 2000, 360 / 5000, 360 / 8000],
)
self.tile_mgr = TileManager(self.grid,
MockCache(), [self.source],
"png",
locker=DummyLocker())
task = self.make_geom_task(geom, SRS(4326), [0, 1, 2, 3, 4])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
assert len(self.seed_pool.seeded_tiles) == 5
assert self.seed_pool.seeded_tiles[0] == set([(0, 0)])
assert self.grid.grid_sizes[1] == (3, 2)
assert self.seed_pool.seeded_tiles[1] == set([(1, 0), (1, 1), (2, 0),
(2, 1)])
assert self.grid.grid_sizes[2] == (8, 4)
assert self.seed_pool.seeded_tiles[2] == set([(4, 2), (5, 2), (4, 3),
(5, 3), (3, 3)])
assert self.grid.grid_sizes[3] == (20, 10)
assert len(self.seed_pool.seeded_tiles[3]) == 5 * 5 + 2
def test_seed_small_bbox_iregular_levels(self):
task = self.make_bbox_task([-45, 0, 180, 90], SRS(4326), [0, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
assert len(self.seed_pool.seeded_tiles) == 2
assert self.seed_pool.seeded_tiles[0] == set([(0, 0)])
assert self.seed_pool.seeded_tiles[2] == set([(1, 1), (2, 1), (3, 1)])
def test_seed_small_bbox_iregular_levels(self):
task = self.make_bbox_task([-45, 0, 180, 90], SRS(4326), [0, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 2)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(1, 1), (2, 1), (3, 1)]))
def test_seed_full_bbox(self):
task = self.make_bbox_task([-180, -90, 180, 90], SRS(4326), [0, 1, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 3)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[1], set([(0, 0), (1, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(0, 0), (1, 0), (2, 0), (3, 0), (0, 1), (1, 1), (2, 1), (3, 1)]))
def test_seed_small_bbox(self):
task = self.make_bbox_task([-45, 0, 180, 90], SRS(4326), [0, 1, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 3)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[1], set([(0, 0), (1, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(1, 1), (2, 1), (3, 1)]))
def test_seed_small_bbox_transformed(self):
bbox = SRS(4326).transform_bbox_to(SRS(900913), [-45, 0, 179, 80])
task = self.make_bbox_task(bbox, SRS(900913), [0, 1, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
assert len(self.seed_pool.seeded_tiles) == 3
assert self.seed_pool.seeded_tiles[0] == set([(0, 0)])
assert self.seed_pool.seeded_tiles[1] == set([(0, 0), (1, 0)])
assert self.seed_pool.seeded_tiles[2] == set([(1, 1), (2, 1), (3, 1)])
def test_seed_small_bbox_transformed(self):
bbox = SRS(4326).transform_bbox_to(SRS(900913), [-45, 0, 179, 80])
task = self.make_bbox_task(bbox, SRS(900913), [0, 1, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 3)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[1], set([(0, 0), (1, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(1, 1), (2, 1), (3, 1)]))
def test_seed_full_bbox(self):
task = self.make_bbox_task([-180, -90, 180, 90], SRS(4326), [0, 1, 2])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
assert len(self.seed_pool.seeded_tiles) == 3
assert self.seed_pool.seeded_tiles[0] == set([(0, 0)])
assert self.seed_pool.seeded_tiles[1] == set([(0, 0), (1, 0)])
assert self.seed_pool.seeded_tiles[2] == set([(0, 0), (1, 0), (2, 0),
(3, 0), (0, 1), (1, 1),
(2, 1), (3, 1)])
def test_seed_full_bbox_continue(self):
task = self.make_bbox_task([-180, -90, 180, 90], SRS(4326), [0, 1, 2])
seed_progress = SeedProgress([(0, 1), (0, 2)])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True, seed_progress=seed_progress)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 3)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[1], set([(0, 0), (1, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(2, 0), (3, 0),
(2, 1), (3, 1)]))
def tilewalker_cleanup(task, dry_run, concurrency, skip_geoms_for_last_levels,
progress_logger=None):
"""
Cleanup tiles with tile traversal.
"""
task.tile_manager._expire_timestamp = task.remove_timestamp
task.tile_manager.minimize_meta_requests = False
tile_worker_pool = TileWorkerPool(task, TileCleanupWorker, progress_logger=progress_logger,
dry_run=dry_run, size=concurrency)
tile_walker = TileWalker(task, tile_worker_pool, handle_stale=True,
work_on_metatiles=False, progress_logger=progress_logger,
skip_geoms_for_last_levels=skip_geoms_for_last_levels)
tile_walker.walk()
tile_worker_pool.stop()
def test_seed_with_geom(self):
if not load_wkt: raise SkipTest('no shapely installed')
# box from 10 10 to 80 80 with small spike/corner to -10 60 (upper left)
geom = load_wkt("POLYGON((10 10, 10 50, -10 60, 10 80, 80 80, 80 10, 10 10))")
task = self.make_geom_task(geom, SRS(4326), [0, 1, 2, 3, 4])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 5)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[1], set([(0, 0), (1, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(1, 1), (2, 1)]))
eq_(self.seed_pool.seeded_tiles[3], set([(4, 2), (5, 2), (4, 3), (5, 3), (3, 3)]))
eq_(len(self.seed_pool.seeded_tiles[4]), 4*4+2)
def test_seed_with_geom(self):
# box from 10 10 to 80 80 with small spike/corner to -10 60 (upper left)
geom = load_wkt(
"POLYGON((10 10, 10 50, -10 60, 10 80, 80 80, 80 10, 10 10))")
task = self.make_geom_task(geom, SRS(4326), [0, 1, 2, 3, 4])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
assert len(self.seed_pool.seeded_tiles) == 5
assert self.seed_pool.seeded_tiles[0] == set([(0, 0)])
assert self.seed_pool.seeded_tiles[1] == set([(0, 0), (1, 0)])
assert self.seed_pool.seeded_tiles[2] == set([(1, 1), (2, 1)])
assert self.seed_pool.seeded_tiles[3] == set([(4, 2), (5, 2), (4, 3),
(5, 3), (3, 3)])
assert len(self.seed_pool.seeded_tiles[4]) == 4 * 4 + 2
def test_seed_with_res_list(self):
if not load_wkt: raise SkipTest('no shapely installed')
# box from 10 10 to 80 80 with small spike/corner to -10 60 (upper left)
geom = load_wkt("POLYGON((10 10, 10 50, -10 60, 10 80, 80 80, 80 10, 10 10))")
self.grid = TileGrid(SRS(4326), bbox=[-180, -90, 180, 90],
res=[360/256, 360/720, 360/2000, 360/5000, 360/8000])
self.tile_mgr = TileManager(self.grid, MockCache(), [self.source], 'png')
task = self.make_geom_task(geom, SRS(4326), [0, 1, 2, 3, 4])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 5)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.grid.grid_sizes[1], (3, 2))
eq_(self.seed_pool.seeded_tiles[1], set([(1, 0), (1, 1), (2, 0), (2, 1)]))
eq_(self.grid.grid_sizes[2], (8, 4))
eq_(self.seed_pool.seeded_tiles[2], set([(4, 2), (5, 2), (4, 3), (5, 3), (3, 3)]))
eq_(self.grid.grid_sizes[3], (20, 10))
eq_(len(self.seed_pool.seeded_tiles[3]), 5*5+2)