class RasterProcess(ProcessBase):
terminate_event = None
def process(self):
log.debug('Start raster import process. Task %d' % self.task_id)
try:
with self.task() as task:
seed_task = self.create_seed_task(task)
progress_logger = self.create_progress_logger(task)
start_progress = parse_progress_identifier(task.seed_progress)
self.terminate_event = threading.Event()
seed_progress = SeedProgress(self.terminate_event, start_progress)
if seed_task.refresh_timestamp is not None:
seed_task.tile_manager._expire_timestamp = seed_task.refresh_timestamp
seed_worker_factory = partial(TileSeedWorker, self.terminate_event)
self.tile_worker_pool = TileWorkerPool(seed_task, seed_worker_factory,
size=4, progress_logger=progress_logger)
tile_walker = TileWalker(seed_task, self.tile_worker_pool, handle_uncached=True,
progress_logger=progress_logger,
seed_progress=seed_progress)
tile_walker.walk()
if not self.terminate_event.is_set():
self.tile_worker_pool.stop()
if not self.terminate_event.is_set():
self.task_done()
except Exception, e:
self.task_failed(e)
def process(self):
log.debug('Start raster import process. Task %d' % self.task_id)
try:
with self.task() as task:
seed_task = self.create_seed_task(task)
progress_logger = self.create_progress_logger(task)
start_progress = parse_progress_identifier(task.seed_progress)
self.terminate_event = threading.Event()
seed_progress = SeedProgress(self.terminate_event,
start_progress)
if seed_task.refresh_timestamp is not None:
seed_task.tile_manager._expire_timestamp = seed_task.refresh_timestamp
seed_worker_factory = partial(TileSeedWorker,
self.terminate_event)
self.tile_worker_pool = TileWorkerPool(
seed_task,
seed_worker_factory,
size=4,
progress_logger=progress_logger)
tile_walker = TileWalker(seed_task,
self.tile_worker_pool,
handle_uncached=True,
progress_logger=progress_logger,
seed_progress=seed_progress)
tile_walker.walk()
if not self.terminate_event.is_set():
self.tile_worker_pool.stop()
if not self.terminate_event.is_set():
self.task_done()
except Exception, e:
self.task_failed(e)
def tilewalker_cleanup(task, dry_run, concurrency, skip_geoms_for_last_levels,
progress_logger=None):
"""
Cleanup tiles with tile traversal.
"""
task.tile_manager._expire_timestamp = task.remove_timestamp
task.tile_manager.minimize_meta_requests = False
tile_worker_pool = TileWorkerPool(task, TileCleanupWorker, progress_logger=progress_logger,
dry_run=dry_run, size=concurrency)
tile_walker = TileWalker(task, tile_worker_pool, handle_stale=True,
work_on_metatiles=False, progress_logger=progress_logger,
skip_geoms_for_last_levels=skip_geoms_for_last_levels)
tile_walker.walk()
tile_worker_pool.stop()
def tilewalker_cleanup(task,
dry_run,
concurrency,
skip_geoms_for_last_levels,
progress_logger=None,
seed_progress=None):
"""
Cleanup tiles with tile traversal.
"""
task.tile_manager._expire_timestamp = task.remove_timestamp
task.tile_manager.minimize_meta_requests = False
tile_worker_pool = TileWorkerPool(task,
TileCleanupWorker,
progress_logger=progress_logger,
dry_run=dry_run,
size=concurrency)
tile_walker = TileWalker(
task,
tile_worker_pool,
handle_stale=True,
work_on_metatiles=False,
progress_logger=progress_logger,
skip_geoms_for_last_levels=skip_geoms_for_last_levels,
seed_progress=seed_progress)
try:
tile_walker.walk()
except KeyboardInterrupt:
tile_worker_pool.stop(force=True)
raise
finally:
tile_worker_pool.stop()