def __init__(self,
task,
worker_pool,
handle_stale=False,
handle_uncached=False,
work_on_metatiles=True,
skip_geoms_for_last_levels=0,
progress_logger=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
self.report_till_level = task.levels[int(num_seed_levels * 0.8)]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (
1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid,
meta_size=meta_size,
meta_buffer=0)
self.progress = 0.0
self.eta = ETA()
self.count = 0
def __init__(self, old_progress_identifier=None):
self.progress = 0.0
self.eta = ETA()
self.level_progress_percentages = [1.0]
self.level_progresses = []
self.progress_str_parts = []
self.old_level_progresses = None
if old_progress_identifier is not None:
self.old_level_progresses = old_progress_identifier
def __init__(self, old_progress_identifier=None):
self.progress = 0.0
self.eta = ETA()
self.level_progress_percentages = [1.0]
self.level_progresses = []
self.progress_str_parts = []
self.old_level_progresses = None
if old_progress_identifier is not None:
self.old_level_progresses = old_progress_identifier
def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False,
work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
self.report_till_level = task.levels[int(num_seed_levels * 0.8)]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0)
self.progress = 0.0
self.eta = ETA()
self.count = 0
class SeedProgress(object):
def __init__(self, old_progress_identifier=None):
self.progress = 0.0
self.eta = ETA()
self.level_progress_percentages = [1.0]
self.level_progresses = []
self.progress_str_parts = []
self.old_level_progresses = None
if old_progress_identifier is not None:
self.old_level_progresses = old_progress_identifier
def step_forward(self, subtiles=1):
self.progress += self.level_progress_percentages[-1] / subtiles
self.eta.update(self.progress)
@property
def progress_str(self):
return ''.join(self.progress_str_parts)
@contextmanager
def step_down(self, i, subtiles):
self.level_progresses.append((i, subtiles))
self.progress_str_parts.append(status_symbol(i, subtiles))
self.level_progress_percentages.append(self.level_progress_percentages[-1] / subtiles)
yield
self.level_progress_percentages.pop()
self.progress_str_parts.pop()
self.level_progresses.pop()
def already_processed(self):
if self.old_level_progresses == []:
return True
if self.old_level_progresses is None:
return False
if self.progress_is_behind(self.old_level_progresses, self.level_progresses):
return True
else:
return False
def current_progress_identifier(self):
return self.level_progresses
@staticmethod
def progress_is_behind(old_progress, current_progress):
"""
Return True if the `current_progress` is behind the `old_progress` -
when it isn't as far as the old progress.
>>> SeedProgress.progress_is_behind([], [(0, 1)])
True
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1)])
False
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (0, 4)])
True
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (1, 4)])
True
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (3, 4)])
False
"""
for old, current in izip_longest(old_progress, current_progress, fillvalue=(9e15, 9e15)):
if old < current:
return False
if old > current:
return True
return True
def running(self):
return True
class TileWalker(object):
def __init__(self,
task,
worker_pool,
handle_stale=False,
handle_uncached=False,
work_on_metatiles=True,
skip_geoms_for_last_levels=0,
progress_logger=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
self.report_till_level = task.levels[int(num_seed_levels * 0.8)]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (
1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid,
meta_size=meta_size,
meta_buffer=0)
self.progress = 0.0
self.eta = ETA()
self.count = 0
def walk(self):
assert self.handle_stale or self.handle_uncached
bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs)
self._walk(bbox, self.task.levels)
self.report_progress(self.task.levels[0], self.task.coverage.bbox)
def _walk(self,
cur_bbox,
levels,
progess_str='',
progress=1.0,
all_subtiles=False):
"""
:param cur_bbox: the bbox to seed in this call
:param levels: list of levels to seed
:param all_subtiles: seed all subtiles and do not check for
intersections with bbox/geom
"""
current_level, levels = levels[0], levels[1:]
bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(
cur_bbox, current_level)
total_subtiles = tiles[0] * tiles[1]
if len(levels) < self.skip_geoms_for_last_levels:
# do not filter in last levels
all_subtiles = True
subtiles = self._filter_subtiles(subtiles, all_subtiles)
if current_level <= self.report_till_level:
self.report_progress(current_level, cur_bbox)
progress = progress / total_subtiles
for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
if subtile is None: # no intersection
self.progress += progress
continue
if levels: # recurse to next level
sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
cur_progess_str = progess_str + status_symbol(
i, total_subtiles)
if intersection == CONTAINS:
all_subtiles = True
else:
all_subtiles = False
self._walk(sub_bbox,
levels,
cur_progess_str,
all_subtiles=all_subtiles,
progress=progress)
if not self.work_on_metatiles:
# collect actual tiles
handle_tiles = self.grid.tile_list(subtile)
else:
handle_tiles = [subtile]
if self.handle_uncached:
handle_tiles = [
t for t in handle_tiles
if t is not None and not self.tile_mgr.is_cached(t)
]
elif self.handle_stale:
handle_tiles = [
t for t in handle_tiles
if t is not None and self.tile_mgr.is_stale(t)
]
if handle_tiles:
self.count += 1
self.worker_pool.process(
handle_tiles, (progess_str, self.progress, self.eta))
if not levels:
self.progress += progress
if len(levels) >= 4:
# call cleanup to close open caches
# for connection based caches
self.tile_mgr.cleanup()
self.eta.update(self.progress)
def report_progress(self, level, bbox):
if self.progress_logger:
self.progress_logger.log_progress(
self.progress, level, bbox,
self.count * self.tiles_per_metatile, self.eta)
def _filter_subtiles(self, subtiles, all_subtiles):
"""
Return an iterator with all sub tiles.
Yields (None, None, None) for non-intersecting tiles,
otherwise (subtile, subtile_bbox, intersection).
"""
for subtile in subtiles:
if subtile is None:
yield None, None, None
else:
sub_bbox = self.grid.meta_tile(subtile).bbox
if all_subtiles:
intersection = CONTAINS
else:
intersection = self.task.intersects(sub_bbox)
if intersection:
yield subtile, sub_bbox, intersection
else:
yield None, None, None
class TileWalker(object):
def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False,
work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None):
self.tile_mgr = task.tile_manager
self.task = task
self.worker_pool = worker_pool
self.handle_stale = handle_stale
self.handle_uncached = handle_uncached
self.work_on_metatiles = work_on_metatiles
self.skip_geoms_for_last_levels = skip_geoms_for_last_levels
self.progress_logger = progress_logger
num_seed_levels = len(task.levels)
self.report_till_level = task.levels[int(num_seed_levels * 0.8)]
meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (1, 1)
self.tiles_per_metatile = meta_size[0] * meta_size[1]
self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0)
self.progress = 0.0
self.eta = ETA()
self.count = 0
def walk(self):
assert self.handle_stale or self.handle_uncached
bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs)
self._walk(bbox, self.task.levels)
self.report_progress(self.task.levels[0], self.task.coverage.bbox)
def _walk(self, cur_bbox, levels, progess_str='', progress=1.0, all_subtiles=False):
"""
:param cur_bbox: the bbox to seed in this call
:param levels: list of levels to seed
:param all_subtiles: seed all subtiles and do not check for
intersections with bbox/geom
"""
current_level, levels = levels[0], levels[1:]
bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(cur_bbox, current_level)
total_subtiles = tiles[0] * tiles[1]
if len(levels) < self.skip_geoms_for_last_levels:
# do not filter in last levels
all_subtiles = True
subtiles = self._filter_subtiles(subtiles, all_subtiles)
if current_level <= self.report_till_level:
self.report_progress(current_level, cur_bbox)
progress = progress / total_subtiles
for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
if subtile is None: # no intersection
self.progress += progress
continue
if levels: # recurse to next level
sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
cur_progess_str = progess_str + status_symbol(i, total_subtiles)
if intersection == CONTAINS:
all_subtiles = True
else:
all_subtiles = False
self._walk(sub_bbox, levels, cur_progess_str,
all_subtiles=all_subtiles, progress=progress)
if not self.work_on_metatiles:
# collect actual tiles
handle_tiles = self.grid.tile_list(subtile)
else:
handle_tiles = [subtile]
if self.handle_uncached:
handle_tiles = [t for t in handle_tiles if
t is not None and
not self.tile_mgr.is_cached(t)]
elif self.handle_stale:
handle_tiles = [t for t in handle_tiles if
t is not None and
self.tile_mgr.is_stale(t)]
if handle_tiles:
self.count += 1
self.worker_pool.process(handle_tiles,
(progess_str, self.progress, self.eta))
if not levels:
self.progress += progress
if len(levels) >= 4:
# call cleanup to close open caches
# for connection based caches
self.tile_mgr.cleanup()
self.eta.update(self.progress)
def report_progress(self, level, bbox):
if self.progress_logger:
self.progress_logger.log_progress(self.progress, level, bbox,
self.count * self.tiles_per_metatile, self.eta)
def _filter_subtiles(self, subtiles, all_subtiles):
"""
Return an iterator with all sub tiles.
Yields (None, None, None) for non-intersecting tiles,
otherwise (subtile, subtile_bbox, intersection).
"""
for subtile in subtiles:
if subtile is None:
yield None, None, None
else:
sub_bbox = self.grid.meta_tile(subtile).bbox
if all_subtiles:
intersection = CONTAINS
else:
intersection = self.task.intersects(sub_bbox)
if intersection:
yield subtile, sub_bbox, intersection
else:
yield None, None, None
class SeedProgress(object):
def __init__(self, old_progress_identifier=None):
self.progress = 0.0
self.eta = ETA()
self.level_progress_percentages = [1.0]
self.level_progresses = []
self.progress_str_parts = []
self.old_level_progresses = None
if old_progress_identifier is not None:
self.old_level_progresses = old_progress_identifier
def step_forward(self, subtiles=1):
self.progress += self.level_progress_percentages[-1] / subtiles
self.eta.update(self.progress)
@property
def progress_str(self):
return ''.join(self.progress_str_parts)
@contextmanager
def step_down(self, i, subtiles):
self.level_progresses.append((i, subtiles))
self.progress_str_parts.append(status_symbol(i, subtiles))
self.level_progress_percentages.append(self.level_progress_percentages[-1] / subtiles)
yield
self.level_progress_percentages.pop()
self.progress_str_parts.pop()
self.level_progresses.pop()
def already_processed(self):
if self.old_level_progresses == []:
return True
if self.old_level_progresses is None:
return False
if self.progress_is_behind(self.old_level_progresses, self.level_progresses):
return True
else:
return False
def current_progress_identifier(self):
return self.level_progresses
@staticmethod
def progress_is_behind(old_progress, current_progress):
"""
Return True if the `current_progress` is behind the `old_progress` -
when it isn't as far as the old progress.
>>> SeedProgress.progress_is_behind([], [(0, 1)])
True
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1)])
False
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (0, 4)])
True
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (1, 4)])
True
>>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (3, 4)])
False
"""
for old, current in izip_longest(old_progress, current_progress, fillvalue=(9e15, 9e15)):
if old < current:
return False
if old > current:
return True
return True
def running(self):
return True
