def test_seed_full_bbox_continue(self):
task = self.make_bbox_task([-180, -90, 180, 90], SRS(4326), [0, 1, 2])
seed_progress = SeedProgress([(0, 1), (0, 2)])
seeder = TileWalker(task, self.seed_pool, handle_uncached=True, seed_progress=seed_progress)
seeder.walk()
eq_(len(self.seed_pool.seeded_tiles), 3)
eq_(self.seed_pool.seeded_tiles[0], set([(0, 0)]))
eq_(self.seed_pool.seeded_tiles[1], set([(0, 0), (1, 0)]))
eq_(self.seed_pool.seeded_tiles[2], set([(2, 0), (3, 0),
(2, 1), (3, 1)]))
def cleanup(tasks,
concurrency=2,
dry_run=False,
skip_geoms_for_last_levels=0,
verbose=True,
progress_logger=None):
for task in tasks:
print(format_cleanup_task(task))
if task.coverage is False:
continue
# seed_progress for tilewalker cleanup
seed_progress = None
# cleanup_progress for os.walk based cleanup
cleanup_progress = None
if progress_logger and progress_logger.progress_store:
progress_logger.current_task_id = task.id
start_progress = progress_logger.progress_store.get(task.id)
seed_progress = SeedProgress(
old_progress_identifier=start_progress)
cleanup_progress = DirectoryCleanupProgress(old_dir=start_progress)
if task.complete_extent:
if callable(
getattr(task.tile_manager.cache, 'level_location', None)):
simple_cleanup(task,
dry_run=dry_run,
progress_logger=progress_logger,
cleanup_progress=cleanup_progress)
continue
elif callable(
getattr(task.tile_manager.cache,
'remove_level_tiles_before', None)):
cache_cleanup(task,
dry_run=dry_run,
progress_logger=progress_logger)
continue
tilewalker_cleanup(
task,
dry_run=dry_run,
concurrency=concurrency,
skip_geoms_for_last_levels=skip_geoms_for_last_levels,
progress_logger=progress_logger,
seed_progress=seed_progress,
)
def __init__(self, terminate_event, *args, **kw):
SeedProgress_.__init__(self, *args, **kw)
self.terminate_event = terminate_event
def test_already_processed(self):
new = SeedProgress([(0, 2)])
with new.step_down(0, 2):
assert not new.already_processed()
with new.step_down(0, 2):
assert not new.already_processed()
new = SeedProgress([(1, 2)])
with new.step_down(0, 2):
assert new.already_processed()
with new.step_down(0, 2):
assert new.already_processed()
new = SeedProgress([(0, 2), (1, 4), (2, 4)])
with new.step_down(0, 2):
assert not new.already_processed()
with new.step_down(0, 4):
assert new.already_processed()
with new.step_down(1, 4):
assert not new.already_processed()
with new.step_down(1, 4):
assert new.already_processed()
with new.step_down(2, 4):
assert not new.already_processed()
with new.step_down(3, 4):
assert not new.already_processed()
with new.step_down(2, 4):
assert not new.already_processed()
def test_progress_identifier(self):
old = SeedProgress()
with old.step_down(0, 2):
with old.step_down(0, 4):
assert old.current_progress_identifier() == [(0, 2), (0, 4)]
# previous leafs are still present
assert old.current_progress_identifier() == [(0, 2), (0, 4)]
with old.step_down(1, 4):
assert old.current_progress_identifier() == [(0, 2), (1, 4)]
assert old.current_progress_identifier() == [(0, 2), (1, 4)]
assert old.current_progress_identifier() == [] # empty list after seed
with old.step_down(1, 2):
assert old.current_progress_identifier() == [(1, 2)]
with old.step_down(0, 4):
with old.step_down(1, 4):
assert old.current_progress_identifier() == [(1, 2),
(0, 4),
(1, 4)]
def test_progress_identifier(self):
old = SeedProgress()
with old.step_down(0, 2):
with old.step_down(0, 4):
eq_(old.current_progress_identifier(), [(0, 2), (0, 4)])
# previous leafs are still present
eq_(old.current_progress_identifier(), [(0, 2), (0, 4)])
with old.step_down(1, 4):
eq_(old.current_progress_identifier(), [(0, 2), (1, 4)])
eq_(old.current_progress_identifier(), [(0, 2), (1, 4)])
eq_(old.current_progress_identifier(), []) # empty list after seed
with old.step_down(1, 2):
eq_(old.current_progress_identifier(), [(1, 2)])
with old.step_down(0, 4):
with old.step_down(1, 4):
eq_(old.current_progress_identifier(), [(1, 2), (0, 4),
(1, 4)])
def test_already_processed(self):
new = SeedProgress([(0, 2)])
with new.step_down(0, 2):
assert not new.already_processed()
with new.step_down(0, 2):
assert not new.already_processed()
new = SeedProgress([(1, 2)])
with new.step_down(0, 2):
assert new.already_processed()
with new.step_down(0, 2):
assert new.already_processed()
new = SeedProgress([(0, 2), (1, 4), (2, 4)])
with new.step_down(0, 2):
assert not new.already_processed()
with new.step_down(0, 4):
assert new.already_processed()
with new.step_down(1, 4):
assert not new.already_processed()
with new.step_down(1, 4):
assert new.already_processed()
with new.step_down(2, 4):
assert not new.already_processed()
with new.step_down(3, 4):
assert not new.already_processed()
with new.step_down(2, 4):
assert not new.already_processed()
def test_progress_identifier(self):
old = SeedProgress()
with old.step_down(0, 2):
with old.step_down(0, 4):
eq_(old.current_progress_identifier(), [(0, 2), (0, 4)])
# previous leafs are still present
eq_(old.current_progress_identifier(), [(0, 2), (0, 4)])
with old.step_down(1, 4):
eq_(old.current_progress_identifier(), [(0, 2), (1, 4)])
eq_(old.current_progress_identifier(), [(0, 2), (1, 4)])
eq_(old.current_progress_identifier(), []) # empty list after seed
with old.step_down(1, 2):
eq_(old.current_progress_identifier(), [(1, 2)])
with old.step_down(0, 4):
with old.step_down(1, 4):
eq_(old.current_progress_identifier(), [(1, 2), (0, 4), (1, 4)])