def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False, work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None): self.tile_mgr = task.tile_manager self.task = task self.worker_pool = worker_pool self.handle_stale = handle_stale self.handle_uncached = handle_uncached self.work_on_metatiles = work_on_metatiles self.skip_geoms_for_last_levels = skip_geoms_for_last_levels self.progress_logger = progress_logger num_seed_levels = len(task.levels) self.report_till_level = task.levels[int(num_seed_levels * 0.8)] meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else ( 1, 1) self.tiles_per_metatile = meta_size[0] * meta_size[1] self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0) self.progress = 0.0 self.eta = ETA() self.count = 0
def __init__(self, old_progress_identifier=None): self.progress = 0.0 self.eta = ETA() self.level_progress_percentages = [1.0] self.level_progresses = [] self.progress_str_parts = [] self.old_level_progresses = None if old_progress_identifier is not None: self.old_level_progresses = old_progress_identifier
def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False, work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None): self.tile_mgr = task.tile_manager self.task = task self.worker_pool = worker_pool self.handle_stale = handle_stale self.handle_uncached = handle_uncached self.work_on_metatiles = work_on_metatiles self.skip_geoms_for_last_levels = skip_geoms_for_last_levels self.progress_logger = progress_logger num_seed_levels = len(task.levels) self.report_till_level = task.levels[int(num_seed_levels * 0.8)] meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (1, 1) self.tiles_per_metatile = meta_size[0] * meta_size[1] self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0) self.progress = 0.0 self.eta = ETA() self.count = 0
class SeedProgress(object): def __init__(self, old_progress_identifier=None): self.progress = 0.0 self.eta = ETA() self.level_progress_percentages = [1.0] self.level_progresses = [] self.progress_str_parts = [] self.old_level_progresses = None if old_progress_identifier is not None: self.old_level_progresses = old_progress_identifier def step_forward(self, subtiles=1): self.progress += self.level_progress_percentages[-1] / subtiles self.eta.update(self.progress) @property def progress_str(self): return ''.join(self.progress_str_parts) @contextmanager def step_down(self, i, subtiles): self.level_progresses.append((i, subtiles)) self.progress_str_parts.append(status_symbol(i, subtiles)) self.level_progress_percentages.append(self.level_progress_percentages[-1] / subtiles) yield self.level_progress_percentages.pop() self.progress_str_parts.pop() self.level_progresses.pop() def already_processed(self): if self.old_level_progresses == []: return True if self.old_level_progresses is None: return False if self.progress_is_behind(self.old_level_progresses, self.level_progresses): return True else: return False def current_progress_identifier(self): return self.level_progresses @staticmethod def progress_is_behind(old_progress, current_progress): """ Return True if the `current_progress` is behind the `old_progress` - when it isn't as far as the old progress. >>> SeedProgress.progress_is_behind([], [(0, 1)]) True >>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1)]) False >>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (0, 4)]) True >>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (1, 4)]) True >>> SeedProgress.progress_is_behind([(0, 1), (1, 4)], [(0, 1), (3, 4)]) False """ for old, current in izip_longest(old_progress, current_progress, fillvalue=(9e15, 9e15)): if old < current: return False if old > current: return True return True def running(self): return True
class TileWalker(object): def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False, work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None): self.tile_mgr = task.tile_manager self.task = task self.worker_pool = worker_pool self.handle_stale = handle_stale self.handle_uncached = handle_uncached self.work_on_metatiles = work_on_metatiles self.skip_geoms_for_last_levels = skip_geoms_for_last_levels self.progress_logger = progress_logger num_seed_levels = len(task.levels) self.report_till_level = task.levels[int(num_seed_levels * 0.8)] meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else ( 1, 1) self.tiles_per_metatile = meta_size[0] * meta_size[1] self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0) self.progress = 0.0 self.eta = ETA() self.count = 0 def walk(self): assert self.handle_stale or self.handle_uncached bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs) self._walk(bbox, self.task.levels) self.report_progress(self.task.levels[0], self.task.coverage.bbox) def _walk(self, cur_bbox, levels, progess_str='', progress=1.0, all_subtiles=False): """ :param cur_bbox: the bbox to seed in this call :param levels: list of levels to seed :param all_subtiles: seed all subtiles and do not check for intersections with bbox/geom """ current_level, levels = levels[0], levels[1:] bbox_, tiles, subtiles = self.grid.get_affected_level_tiles( cur_bbox, current_level) total_subtiles = tiles[0] * tiles[1] if len(levels) < self.skip_geoms_for_last_levels: # do not filter in last levels all_subtiles = True subtiles = self._filter_subtiles(subtiles, all_subtiles) if current_level <= self.report_till_level: self.report_progress(current_level, cur_bbox) progress = progress / total_subtiles for i, (subtile, sub_bbox, intersection) in enumerate(subtiles): if subtile is None: # no intersection self.progress += progress continue if levels: # recurse to next level sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox) cur_progess_str = progess_str + status_symbol( i, total_subtiles) if intersection == CONTAINS: all_subtiles = True else: all_subtiles = False self._walk(sub_bbox, levels, cur_progess_str, all_subtiles=all_subtiles, progress=progress) if not self.work_on_metatiles: # collect actual tiles handle_tiles = self.grid.tile_list(subtile) else: handle_tiles = [subtile] if self.handle_uncached: handle_tiles = [ t for t in handle_tiles if t is not None and not self.tile_mgr.is_cached(t) ] elif self.handle_stale: handle_tiles = [ t for t in handle_tiles if t is not None and self.tile_mgr.is_stale(t) ] if handle_tiles: self.count += 1 self.worker_pool.process( handle_tiles, (progess_str, self.progress, self.eta)) if not levels: self.progress += progress if len(levels) >= 4: # call cleanup to close open caches # for connection based caches self.tile_mgr.cleanup() self.eta.update(self.progress) def report_progress(self, level, bbox): if self.progress_logger: self.progress_logger.log_progress( self.progress, level, bbox, self.count * self.tiles_per_metatile, self.eta) def _filter_subtiles(self, subtiles, all_subtiles): """ Return an iterator with all sub tiles. Yields (None, None, None) for non-intersecting tiles, otherwise (subtile, subtile_bbox, intersection). """ for subtile in subtiles: if subtile is None: yield None, None, None else: sub_bbox = self.grid.meta_tile(subtile).bbox if all_subtiles: intersection = CONTAINS else: intersection = self.task.intersects(sub_bbox) if intersection: yield subtile, sub_bbox, intersection else: yield None, None, None
class TileWalker(object): def __init__(self, task, worker_pool, handle_stale=False, handle_uncached=False, work_on_metatiles=True, skip_geoms_for_last_levels=0, progress_logger=None): self.tile_mgr = task.tile_manager self.task = task self.worker_pool = worker_pool self.handle_stale = handle_stale self.handle_uncached = handle_uncached self.work_on_metatiles = work_on_metatiles self.skip_geoms_for_last_levels = skip_geoms_for_last_levels self.progress_logger = progress_logger num_seed_levels = len(task.levels) self.report_till_level = task.levels[int(num_seed_levels * 0.8)] meta_size = self.tile_mgr.meta_grid.meta_size if self.tile_mgr.meta_grid else (1, 1) self.tiles_per_metatile = meta_size[0] * meta_size[1] self.grid = MetaGrid(self.tile_mgr.grid, meta_size=meta_size, meta_buffer=0) self.progress = 0.0 self.eta = ETA() self.count = 0 def walk(self): assert self.handle_stale or self.handle_uncached bbox = self.task.coverage.extent.bbox_for(self.tile_mgr.grid.srs) self._walk(bbox, self.task.levels) self.report_progress(self.task.levels[0], self.task.coverage.bbox) def _walk(self, cur_bbox, levels, progess_str='', progress=1.0, all_subtiles=False): """ :param cur_bbox: the bbox to seed in this call :param levels: list of levels to seed :param all_subtiles: seed all subtiles and do not check for intersections with bbox/geom """ current_level, levels = levels[0], levels[1:] bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(cur_bbox, current_level) total_subtiles = tiles[0] * tiles[1] if len(levels) < self.skip_geoms_for_last_levels: # do not filter in last levels all_subtiles = True subtiles = self._filter_subtiles(subtiles, all_subtiles) if current_level <= self.report_till_level: self.report_progress(current_level, cur_bbox) progress = progress / total_subtiles for i, (subtile, sub_bbox, intersection) in enumerate(subtiles): if subtile is None: # no intersection self.progress += progress continue if levels: # recurse to next level sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox) cur_progess_str = progess_str + status_symbol(i, total_subtiles) if intersection == CONTAINS: all_subtiles = True else: all_subtiles = False self._walk(sub_bbox, levels, cur_progess_str, all_subtiles=all_subtiles, progress=progress) if not self.work_on_metatiles: # collect actual tiles handle_tiles = self.grid.tile_list(subtile) else: handle_tiles = [subtile] if self.handle_uncached: handle_tiles = [t for t in handle_tiles if t is not None and not self.tile_mgr.is_cached(t)] elif self.handle_stale: handle_tiles = [t for t in handle_tiles if t is not None and self.tile_mgr.is_stale(t)] if handle_tiles: self.count += 1 self.worker_pool.process(handle_tiles, (progess_str, self.progress, self.eta)) if not levels: self.progress += progress if len(levels) >= 4: # call cleanup to close open caches # for connection based caches self.tile_mgr.cleanup() self.eta.update(self.progress) def report_progress(self, level, bbox): if self.progress_logger: self.progress_logger.log_progress(self.progress, level, bbox, self.count * self.tiles_per_metatile, self.eta) def _filter_subtiles(self, subtiles, all_subtiles): """ Return an iterator with all sub tiles. Yields (None, None, None) for non-intersecting tiles, otherwise (subtile, subtile_bbox, intersection). """ for subtile in subtiles: if subtile is None: yield None, None, None else: sub_bbox = self.grid.meta_tile(subtile).bbox if all_subtiles: intersection = CONTAINS else: intersection = self.task.intersects(sub_bbox) if intersection: yield subtile, sub_bbox, intersection else: yield None, None, None