Beispiel #1
0
    def _walk(self,
              cur_bbox,
              levels,
              progess_str='',
              progress=1.0,
              all_subtiles=False):
        """
        :param cur_bbox: the bbox to seed in this call
        :param levels: list of levels to seed
        :param all_subtiles: seed all subtiles and do not check for
                             intersections with bbox/geom
        """
        current_level, levels = levels[0], levels[1:]
        bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(
            cur_bbox, current_level)
        total_subtiles = tiles[0] * tiles[1]

        if len(levels) < self.skip_geoms_for_last_levels:
            # do not filter in last levels
            all_subtiles = True
        subtiles = self._filter_subtiles(subtiles, all_subtiles)

        if current_level <= self.report_till_level:
            self.report_progress(current_level, cur_bbox)

        progress = progress / total_subtiles
        for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
            if subtile is None:  # no intersection
                self.progress += progress
                continue
            if levels:  # recurse to next level
                sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
                cur_progess_str = progess_str + status_symbol(
                    i, total_subtiles)
                if intersection == CONTAINS:
                    all_subtiles = True
                else:
                    all_subtiles = False
                self._walk(sub_bbox,
                           levels,
                           cur_progess_str,
                           all_subtiles=all_subtiles,
                           progress=progress)

            if not self.work_on_metatiles:
                # collect actual tiles
                handle_tiles = self.grid.tile_list(subtile)
            else:
                handle_tiles = [subtile]

            if self.handle_uncached:
                handle_tiles = [
                    t for t in handle_tiles
                    if t is not None and not self.tile_mgr.is_cached(t)
                ]
            elif self.handle_stale:
                handle_tiles = [
                    t for t in handle_tiles
                    if t is not None and self.tile_mgr.is_stale(t)
                ]
            if handle_tiles:
                self.count += 1
                self.worker_pool.process(
                    handle_tiles, (progess_str, self.progress, self.eta))

            if not levels:
                self.progress += progress

        if len(levels) >= 4:
            # call cleanup to close open caches
            # for connection based caches
            self.tile_mgr.cleanup()
        self.eta.update(self.progress)
Beispiel #2
0
    def _walk(self, cur_bbox, levels, all_subtiles=False):
        """
        :param cur_bbox: the bbox to seed in this call
        :param levels: list of levels to seed
        :param all_subtiles: seed all subtiles and do not check for
                             intersections with bbox/geom
        """
        current_level, levels = levels[0], levels[1:]
        bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(cur_bbox, current_level)
        total_subtiles = tiles[0] * tiles[1]

        if len(levels) < self.skip_geoms_for_last_levels:
            # do not filter in last levels
            all_subtiles = True
        subtiles = self._filter_subtiles(subtiles, all_subtiles)

        if current_level <= self.report_till_level:
            self.report_progress(current_level, cur_bbox)

        if not self.seed_progress.running():
            self.report_progress(current_level, cur_bbox)
            self.tile_mgr.cleanup()
            raise StopProcess()

        for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
            if subtile is None: # no intersection
                self.seed_progress.step_forward(total_subtiles)
                continue
            if levels: # recurse to next level
                sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
                if intersection == CONTAINS:
                    all_subtiles = True
                else:
                    all_subtiles = False

                with self.seed_progress.step_down(i, total_subtiles):
                    if self.seed_progress.already_processed():
                        self.seed_progress.step_forward()
                    else:
                        self._walk(sub_bbox, levels,
                            all_subtiles=all_subtiles)

            if not self.work_on_metatiles:
                # collect actual tiles
                handle_tiles = self.grid.tile_list(subtile)
            else:
                handle_tiles = [subtile]

            if self.handle_uncached:
                handle_tiles = [t for t in handle_tiles if
                                    t is not None and
                                    not self.tile_mgr.is_cached(t)]
            elif self.handle_stale:
                handle_tiles = [t for t in handle_tiles if
                                    t is not None and
                                    self.tile_mgr.is_stale(t)]
            if handle_tiles:
                self.count += 1
                self.worker_pool.process(handle_tiles, self.seed_progress)

            if not levels:
                self.seed_progress.step_forward(total_subtiles)

        if len(levels) >= 4:
            # call cleanup to close open caches
            # for connection based caches
            self.tile_mgr.cleanup()
Beispiel #3
0
    def _walk(self, cur_bbox, levels, current_level=0, all_subtiles=False):
        """
        :param cur_bbox: the bbox to seed in this call
        :param levels: list of levels to seed
        :param all_subtiles: seed all subtiles and do not check for
                             intersections with bbox/geom
        """
        bbox_, tiles, subtiles = self.grid.get_affected_level_tiles(
            cur_bbox, current_level)
        total_subtiles = tiles[0] * tiles[1]
        if len(levels) < self.skip_geoms_for_last_levels:
            # do not filter in last levels
            all_subtiles = True
        subtiles = self._filter_subtiles(subtiles, all_subtiles)

        if current_level in levels and current_level <= self.report_till_level:
            self.report_progress(current_level, cur_bbox)

        if not self.seed_progress.running():
            if current_level in levels:
                self.report_progress(current_level, cur_bbox)
            self.tile_mgr.cleanup()
            raise StopProcess()

        process = False
        if current_level in levels:
            levels = levels[1:]
            process = True

        for i, (subtile, sub_bbox, intersection) in enumerate(subtiles):
            if subtile is None:  # no intersection
                self.seed_progress.step_forward(total_subtiles)
                continue
            if levels:  # recurse to next level
                sub_bbox = limit_sub_bbox(cur_bbox, sub_bbox)
                if intersection == CONTAINS:
                    all_subtiles = True
                else:
                    all_subtiles = False

                with self.seed_progress.step_down(i, total_subtiles):
                    if self.seed_progress.already_processed():
                        self.seed_progress.step_forward()
                    else:
                        self._walk(sub_bbox,
                                   levels,
                                   current_level=current_level + 1,
                                   all_subtiles=all_subtiles)

            if not process:
                continue

            # check if subtile was already processed. see comment in __init__
            if subtile in self.seeded_tiles[current_level]:
                if not levels:
                    self.seed_progress.step_forward(total_subtiles)
                continue
            self.seeded_tiles[current_level].appendleft(subtile)

            if not self.work_on_metatiles:
                # collect actual tiles
                handle_tiles = self.grid.tile_list(subtile)
            else:
                handle_tiles = [subtile]

            if self.handle_uncached:
                handle_tiles = [
                    t for t in handle_tiles
                    if t is not None and not self.tile_mgr.is_cached(t)
                ]
            elif self.handle_stale:
                handle_tiles = [
                    t for t in handle_tiles
                    if t is not None and self.tile_mgr.is_stale(t)
                ]
            if handle_tiles:
                self.count += 1
                self.worker_pool.process(handle_tiles, self.seed_progress)

            if not levels:
                self.seed_progress.step_forward(total_subtiles)

        if len(levels) >= 4:
            # call cleanup to close open caches
            # for connection based caches
            self.tile_mgr.cleanup()