Пример #1
0
class TileManager(object):
    """
    Manages tiles for a single grid.
    Loads tiles from the cache, creates new tiles from sources and stores them
    into the cache, or removes tiles.

    :param pre_store_filter: a list with filter. each filter will be called
        with a tile before it will be stored to disc. the filter should
        return this or a new tile object.
    """
    def __init__(self, grid, cache, sources, format, image_opts=None, request_format=None,
        meta_buffer=None, meta_size=None, minimize_meta_requests=False,
        pre_store_filter=None, concurrent_tile_creators=1,max_age=None):
        self.grid = grid
        self.cache = cache
        self.meta_grid = None
        self.format = format
        self.image_opts = image_opts
        self.request_format = request_format or format
        self.sources = sources
        self.minimize_meta_requests = minimize_meta_requests
        self._max_age = max_age
        self._expire_timestamp = None
        self.transparent = self.sources[0].transparent
        self.pre_store_filter = pre_store_filter or []
        self.concurrent_tile_creators = concurrent_tile_creators

        if meta_buffer or (meta_size and not meta_size == [1, 1]):
            if all(source.supports_meta_tiles for source in sources):
                self.meta_grid = MetaGrid(grid, meta_size=meta_size, meta_buffer=meta_buffer)
            elif any(source.supports_meta_tiles for source in sources):
                raise ValueError('meta tiling configured but not supported by all sources')

    @contextmanager
    def session(self):
        """
        Context manager for access to the cache. Cleans up after usage
        for connection based caches.

        >>> with tile_manager.session(): #doctest: +SKIP
        ...    tile_manager.load_tile_coords(tile_coords)

        """
        yield
        self.cleanup()

    def cleanup(self):
        if hasattr(self.cache, 'cleanup'):
            self.cache.cleanup()

    def load_tile_coord(self, tile_coord, with_metadata=False):
        tile = Tile(tile_coord)
        self.cache.load_tile(tile, with_metadata)

        if tile.coord is not None and not self.is_cached(tile):
            # missing or staled
            creator = self.creator()
            created_tiles = creator.create_tiles([tile])
            for created_tile in created_tiles:
                if created_tile.coord == tile_coord:
                    return created_tile

        return tile

    def load_tile_coords(self, tile_coords, with_metadata=False):
        tiles = TileCollection(tile_coords)
        uncached_tiles = []

        # load all in batch
        self.cache.load_tiles(tiles, with_metadata)

        for tile in tiles:
            if tile.coord is not None and not self.is_cached(tile):
                # missing or staled
                uncached_tiles.append(tile)

        if uncached_tiles:
            creator = self.creator()
            created_tiles = creator.create_tiles(uncached_tiles)
            for created_tile in created_tiles:
                if created_tile.coord in tiles:
                    tiles[created_tile.coord].source = created_tile.source

        return tiles

    def remove_tile_coords(self, tile_coords):
        tiles = TileCollection(tile_coords)
        self.cache.remove_tiles(tiles)

    def creator(self):
        return TileCreator(self.cache, self.sources, self.grid, self.meta_grid, self)

    def lock(self, tile):
        if self.meta_grid:
            tile = Tile(self.meta_grid.main_tile(tile.coord))
        return self.cache.lock(tile)

    def is_cached(self, tile):
        """
        Return True if the tile is cached.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if tile.coord is None:
            return True
        cached = self.cache.is_cached(tile)
        max_mtime = self.expire_timestamp(tile)
        if cached and max_mtime is not None:
            self.cache.load_tile_metadata(tile)
            stale = max_mtime < time.time()
            """ #### Debug #### 
            log.info('Expires:')
            log.info(max_mtime)
            log.info('Current Time:')
            log.info(time.time())
            """
            if stale:
                cached = False
                """ log.info('Tile is passed its used by date and starting to smell - will request a new version.') """   
        return cached

    def is_stale(self, tile):
        """
        Return True if tile exists _and_ is expired.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if self.cache.is_cached(tile):
            # tile exists
            if not self.is_cached(tile):
                # expired
                return True
            return False
        return False

    def expire_timestamp(self, tile=None):
        """
        Return the timestamp until which a tile should be accepted as up-to-date,
        or ``None`` if the tiles should not expire.

        :note: Returns _expire_timestamp by default.
        """
        
        """
        If we have got a max_age variable from the YAML, work out the timestamp that this tile should expire
        """
        if self._max_age is not None and tile is not None:
            if 'time' in self._max_age:
                try:
                    return timestamp_from_isodate(self._max_age['time'])
                except ValueError:
                    log.warn("Could not parse time '%s'. should be ISO time string" % (self._max_age["time"]))
            deltas = {}
            for delta_type in ('weeks', 'days', 'hours', 'minutes'):
                deltas[delta_type] = self._max_age.get(delta_type, 0)
                
            self.cache.load_tile_metadata(tile)
            return tile.timestamp + datetime.timedelta(**deltas).total_seconds()
    
        return self._expire_timestamp

    def apply_tile_filter(self, tile):
        """
        Apply all `pre_store_filter` to this tile.
        Returns filtered tile.
        """
        if tile.stored:
            return tile

        for img_filter in self.pre_store_filter:
            tile = img_filter(tile)
        return tile
Пример #2
0
class TileManager(object):
    """
    Manages tiles for a single grid.
    Loads tiles from the cache, creates new tiles from sources and stores them
    into the cache, or removes tiles.

    :param pre_store_filter: a list with filter. each filter will be called
        with a tile before it will be stored to disc. the filter should
        return this or a new tile object.
    """
    def __init__(self, grid, cache, sources, format, image_opts=None, request_format=None,
        meta_buffer=None, meta_size=None, minimize_meta_requests=False,
        pre_store_filter=None, concurrent_tile_creators=1):
        self.grid = grid
        self.cache = cache
        self.meta_grid = None
        self.format = format
        self.image_opts = image_opts
        self.request_format = request_format or format
        self.sources = sources
        self.minimize_meta_requests = minimize_meta_requests
        self._expire_timestamp = None
        self.transparent = self.sources[0].transparent
        self.pre_store_filter = pre_store_filter or []
        self.concurrent_tile_creators = concurrent_tile_creators

        if meta_buffer or (meta_size and not meta_size == [1, 1]):
            if all(source.supports_meta_tiles for source in sources):
                self.meta_grid = MetaGrid(grid, meta_size=meta_size, meta_buffer=meta_buffer)
            elif any(source.supports_meta_tiles for source in sources):
                raise ValueError('meta tiling configured but not supported by all sources')

    @contextmanager
    def session(self):
        """
        Context manager for access to the cache. Cleans up after usage
        for connection based caches.

        >>> with tile_manager.session(): #doctest: +SKIP
        ...    tile_manager.load_tile_coords(tile_coords)

        """
        yield
        self.cleanup()

    def cleanup(self):
        if hasattr(self.cache, 'cleanup'):
            self.cache.cleanup()

    def load_tile_coord(self, tile_coord, dimensions=None, with_metadata=False):
        tile = Tile(tile_coord)
        self.cache.load_tile(tile, with_metadata)

        if tile.coord is not None and not self.is_cached(tile, dimensions=dimensions):
            # missing or staled
            creator = self.creator(dimensions=dimensions)
            created_tiles = creator.create_tiles([tile])
            for created_tile in created_tiles:
                if created_tile.coord == tile_coord:
                    return created_tile

        return tile

    def load_tile_coords(self, tile_coords, dimensions=None, with_metadata=False):
        tiles = TileCollection(tile_coords)
        uncached_tiles = []

        # load all in batch
        self.cache.load_tiles(tiles, with_metadata)

        for tile in tiles:
            if tile.coord is not None and not self.is_cached(tile, dimensions=dimensions):
                # missing or staled
                uncached_tiles.append(tile)

        if uncached_tiles:
            creator = self.creator(dimensions=dimensions)
            created_tiles = creator.create_tiles(uncached_tiles)
            for created_tile in created_tiles:
                if created_tile.coord in tiles:
                    tiles[created_tile.coord].source = created_tile.source

        return tiles

    def remove_tile_coords(self, tile_coords, dimensions=None):
        tiles = TileCollection(tile_coords)
        self.cache.remove_tiles(tiles)

    def creator(self, dimensions=None):
        return TileCreator(self.cache, self.sources, self.grid, self.meta_grid,
            self, dimensions=dimensions)

    def lock(self, tile):
        if self.meta_grid:
            tile = Tile(self.meta_grid.main_tile(tile.coord))
        return self.cache.lock(tile)

    def is_cached(self, tile, dimensions=None):
        """
        Return True if the tile is cached.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if tile.coord is None:
            return True
        cached = self.cache.is_cached(tile)
        max_mtime = self.expire_timestamp(tile)
        if cached and max_mtime is not None:
            self.cache.load_tile_metadata(tile)
            stale = tile.timestamp < max_mtime
            if stale:
                cached = False
        return cached

    def is_stale(self, tile, dimensions=None):
        """
        Return True if tile exists _and_ is expired.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if self.cache.is_cached(tile):
            # tile exists
            if not self.is_cached(tile):
                # expired
                return True
            return False
        return False

    def expire_timestamp(self, tile=None):
        """
        Return the timestamp until which a tile should be accepted as up-to-date,
        or ``None`` if the tiles should not expire.

        :note: Returns _expire_timestamp by default.
        """
        return self._expire_timestamp

    def apply_tile_filter(self, tile):
        """
        Apply all `pre_store_filter` to this tile.
        Returns filtered tile.
        """
        if tile.stored:
            return tile

        for img_filter in self.pre_store_filter:
            tile = img_filter(tile)
        return tile
Пример #3
0
class TileManager(object):
    """
    Manages tiles for a single grid.
    Loads tiles from the cache, creates new tiles from sources and stores them
    into the cache, or removes tiles.

    :param pre_store_filter: a list with filter. each filter will be called
        with a tile before it will be stored to disc. the filter should
        return this or a new tile object.
    """
    def __init__(self, grid, cache, sources, format, locker, image_opts=None, request_format=None,
            meta_buffer=None, meta_size=None, minimize_meta_requests=False, identifier=None,
            pre_store_filter=None, concurrent_tile_creators=1, tile_creator_class=None,
            bulk_meta_tiles=False,
            rescale_tiles=0,
            cache_rescaled_tiles=False,
        ):
        self.grid = grid
        self.cache = cache
        self.locker = locker
        self.identifier = identifier
        self.meta_grid = None
        self.format = format
        self.image_opts = image_opts
        self.request_format = request_format or format
        self.sources = sources
        self.minimize_meta_requests = minimize_meta_requests
        self._expire_timestamp = None
        self.pre_store_filter = pre_store_filter or []
        self.concurrent_tile_creators = concurrent_tile_creators
        self.tile_creator_class = tile_creator_class or TileCreator

        self.rescale_tiles = rescale_tiles
        self.cache_rescaled_tiles = cache_rescaled_tiles

        if meta_buffer or (meta_size and not meta_size == [1, 1]):
            if all(source.supports_meta_tiles for source in sources):
                self.meta_grid = MetaGrid(grid, meta_size=meta_size, meta_buffer=meta_buffer)
            elif any(source.supports_meta_tiles for source in sources):
                raise ValueError('meta tiling configured but not supported by all sources')
            elif meta_size and not meta_size == [1, 1] and bulk_meta_tiles:
                # meta tiles configured but all sources are tiled
                # use bulk_meta_tile mode that download tiles in parallel
                self.meta_grid = MetaGrid(grid, meta_size=meta_size, meta_buffer=0)
                self.tile_creator_class = partial(self.tile_creator_class, bulk_meta_tiles=True)

    @contextmanager
    def session(self):
        """
        Context manager for access to the cache. Cleans up after usage
        for connection based caches.

        >>> with tile_manager.session(): #doctest: +SKIP
        ...    tile_manager.load_tile_coords(tile_coords)

        """
        yield
        self.cleanup()

    def cleanup(self):
        if hasattr(self.cache, 'cleanup'):
            self.cache.cleanup()

    def load_tile_coord(self, tile_coord, dimensions=None, with_metadata=False):
        return self.load_tile_coords(
            [tile_coord], dimensions=dimensions, with_metadata=with_metadata,
        )[0]


    def load_tile_coords(self, tile_coords, dimensions=None, with_metadata=False):
        tiles = TileCollection(tile_coords)
        rescale_till_zoom = 0
        if self.rescale_tiles:
            rescaled_tiles = {}

            for t in tiles.tiles:
                # Use zoom level from first None tile.
                if t.coord is not None:
                    rescale_till_zoom = t.coord[2] + self.rescale_tiles
                    break
            else:
                return tiles

            if rescale_till_zoom < 0:
                rescale_till_zoom = 0
            if rescale_till_zoom > self.grid.levels:
                rescale_till_zoom = self.grid.levels

        tiles = self._load_tile_coords(
            tiles, dimensions=dimensions, with_metadata=with_metadata,
            rescale_till_zoom=rescale_till_zoom, rescaled_tiles={},
        )

        for t in tiles.tiles:
            # Remove our internal marker source, for missing tiles.
            if t.source is RESCALE_TILE_MISSING:
                t.source = None

        return tiles

    def _load_tile_coords(self, tiles, dimensions=None, with_metadata=False,
                          rescale_till_zoom=None, rescaled_tiles=None,
        ):
        uncached_tiles = []

        if rescaled_tiles:
            for t in tiles:
                if t.coord in rescaled_tiles:
                    t.source = rescaled_tiles[t.coord].source

        # load all in batch
        self.cache.load_tiles(tiles, with_metadata)

        for tile in tiles:
            if tile.coord is not None and not self.is_cached(tile, dimensions=dimensions):
                # missing or staled
                uncached_tiles.append(tile)

        if uncached_tiles:
            creator = self.creator(dimensions=dimensions)
            created_tiles = creator.create_tiles(uncached_tiles)
            if not created_tiles and self.rescale_tiles:
                created_tiles = [self._scaled_tile(t, rescale_till_zoom, rescaled_tiles) for t in uncached_tiles]

            for created_tile in created_tiles:
                if created_tile.coord in tiles:
                    tiles[created_tile.coord].source = created_tile.source

        return tiles

    def remove_tile_coords(self, tile_coords, dimensions=None):
        tiles = TileCollection(tile_coords)
        self.cache.remove_tiles(tiles)

    def creator(self, dimensions=None):
        return self.tile_creator_class(self, dimensions=dimensions)

    def lock(self, tile):
        if self.meta_grid:
            tile = Tile(self.meta_grid.main_tile(tile.coord))
        return self.locker.lock(tile)

    def is_cached(self, tile, dimensions=None):
        """
        Return True if the tile is cached.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if tile.coord is None:
            return True
        cached = self.cache.is_cached(tile)
        max_mtime = self.expire_timestamp(tile)
        if cached and max_mtime is not None:
            self.cache.load_tile_metadata(tile)
            stale = tile.timestamp < max_mtime
            if stale:
                cached = False
        return cached

    def is_stale(self, tile, dimensions=None):
        """
        Return True if tile exists _and_ is expired.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if self.cache.is_cached(tile):
            # tile exists
            if not self.is_cached(tile):
                # expired
                return True
            return False
        return False

    def expire_timestamp(self, tile=None):
        """
        Return the timestamp until which a tile should be accepted as up-to-date,
        or ``None`` if the tiles should not expire.

        :note: Returns _expire_timestamp by default.
        """
        return self._expire_timestamp

    def apply_tile_filter(self, tile):
        """
        Apply all `pre_store_filter` to this tile.
        Returns filtered tile.
        """
        if tile.stored:
            return tile

        for img_filter in self.pre_store_filter:
            tile = img_filter(tile)
        return tile

    def _scaled_tile(self, tile, stop_zoom, rescaled_tiles):
        """
        Try to load tile by loading, scaling and clipping tiles from zoom levels above or
        below. stop_zoom determines if tiles from above should be scaled up, or if tiles
        from below should be scaled down.
        Returns an empty Tile if tile zoom level is stop_zoom.
        """
        if tile.coord in rescaled_tiles:
            return rescaled_tiles[tile.coord]

        # Cache tile in rescaled_tiles. We initially set source to a fixed
        # BlankImageSource and overwrite it if we actually rescaled the tile.
        tile.source = RESCALE_TILE_MISSING
        rescaled_tiles[tile.coord] = tile

        tile_bbox = self.grid.tile_bbox(tile.coord)
        current_zoom = tile.coord[2]
        if stop_zoom == current_zoom:
            return tile
        if stop_zoom > current_zoom:
            src_level = current_zoom + 1
        else:
            src_level = current_zoom - 1

        src_bbox, src_tile_grid, affected_tile_coords = self.grid.get_affected_level_tiles(tile_bbox, src_level)

        affected_tiles = TileCollection(affected_tile_coords)
        for t in affected_tiles:
            # Add sources of cached tiles, to avoid loading same tile multiple times
            # loading recursive.
            if t.coord in rescaled_tiles:
                t.source = rescaled_tiles[t.coord].source

        tile_collection = self._load_tile_coords(
            affected_tiles,
            rescale_till_zoom=stop_zoom,
            rescaled_tiles=rescaled_tiles,
        )

        if tile_collection.blank:
            return tile

        tile_sources = []
        for t in tile_collection:
            # Replace RESCALE_TILE_MISSING with None, before transforming tiles.
            tile_sources.append(t.source if t.source is not RESCALE_TILE_MISSING else None)

        tiled_image = TiledImage(tile_sources, src_bbox=src_bbox, src_srs=self.grid.srs,
                            tile_grid=src_tile_grid, tile_size=self.grid.tile_size)
        tile.source = tiled_image.transform(tile_bbox, self.grid.srs, self.grid.tile_size, self.image_opts)

        if self.cache_rescaled_tiles:
            self.cache.store_tile(tile)
        return tile
Пример #4
0
class TileManager(object):
    """
    Manages tiles for a single grid.
    Loads tiles from the cache, creates new tiles from sources and stores them
    into the cache, or removes tiles.

    :param pre_store_filter: a list with filter. each filter will be called
        with a tile before it will be stored to disc. the filter should
        return this or a new tile object.
    """
    def __init__(
        self,
        grid,
        cache,
        sources,
        format,
        locker,
        image_opts=None,
        request_format=None,
        meta_buffer=None,
        meta_size=None,
        minimize_meta_requests=False,
        identifier=None,
        pre_store_filter=None,
        concurrent_tile_creators=1,
        tile_creator_class=None,
        bulk_meta_tiles=False,
    ):
        self.grid = grid
        self.cache = cache
        self.locker = locker
        self.identifier = identifier
        self.meta_grid = None
        self.format = format
        self.image_opts = image_opts
        self.request_format = request_format or format
        self.sources = sources
        self.minimize_meta_requests = minimize_meta_requests
        self._expire_timestamp = None
        self.transparent = self.sources[0].transparent
        self.pre_store_filter = pre_store_filter or []
        self.concurrent_tile_creators = concurrent_tile_creators
        self.tile_creator_class = tile_creator_class or TileCreator

        if meta_buffer or (meta_size and not meta_size == [1, 1]):
            if all(source.supports_meta_tiles for source in sources):
                self.meta_grid = MetaGrid(grid,
                                          meta_size=meta_size,
                                          meta_buffer=meta_buffer)
            elif any(source.supports_meta_tiles for source in sources):
                raise ValueError(
                    'meta tiling configured but not supported by all sources')
            elif meta_size and not meta_size == [1, 1] and bulk_meta_tiles:
                # meta tiles configured but all sources are tiled
                # use bulk_meta_tile mode that download tiles in parallel
                self.meta_grid = MetaGrid(grid,
                                          meta_size=meta_size,
                                          meta_buffer=0)
                self.tile_creator_class = partial(self.tile_creator_class,
                                                  bulk_meta_tiles=True)

    @contextmanager
    def session(self):
        """
        Context manager for access to the cache. Cleans up after usage
        for connection based caches.

        >>> with tile_manager.session(): #doctest: +SKIP
        ...    tile_manager.load_tile_coords(tile_coords)

        """
        yield
        self.cleanup()

    def cleanup(self):
        if hasattr(self.cache, 'cleanup'):
            self.cache.cleanup()

    def load_tile_coord(self,
                        tile_coord,
                        dimensions=None,
                        with_metadata=False):
        tile = Tile(tile_coord)
        self.cache.load_tile(tile, with_metadata)

        if tile.coord is not None and not self.is_cached(
                tile, dimensions=dimensions):
            # missing or staled
            creator = self.creator(dimensions=dimensions)
            created_tiles = creator.create_tiles([tile])
            for created_tile in created_tiles:
                if created_tile.coord == tile_coord:
                    return created_tile

        return tile

    def load_tile_coords(self,
                         tile_coords,
                         dimensions=None,
                         with_metadata=False):
        tiles = TileCollection(tile_coords)
        uncached_tiles = []

        # load all in batch
        self.cache.load_tiles(tiles, with_metadata)

        for tile in tiles:
            if tile.coord is not None and not self.is_cached(
                    tile, dimensions=dimensions):
                # missing or staled
                uncached_tiles.append(tile)

        if uncached_tiles:
            creator = self.creator(dimensions=dimensions)
            created_tiles = creator.create_tiles(uncached_tiles)
            for created_tile in created_tiles:
                if created_tile.coord in tiles:
                    tiles[created_tile.coord].source = created_tile.source

        return tiles

    def remove_tile_coords(self, tile_coords, dimensions=None):
        tiles = TileCollection(tile_coords)
        self.cache.remove_tiles(tiles)

    def creator(self, dimensions=None):
        return self.tile_creator_class(self, dimensions=dimensions)

    def lock(self, tile):
        if self.meta_grid:
            tile = Tile(self.meta_grid.main_tile(tile.coord))
        return self.locker.lock(tile)

    def is_cached(self, tile, dimensions=None):
        """
        Return True if the tile is cached.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if tile.coord is None:
            return True
        cached = self.cache.is_cached(tile)
        max_mtime = self.expire_timestamp(tile)
        if cached and max_mtime is not None:
            self.cache.load_tile_metadata(tile)
            stale = tile.timestamp < max_mtime
            if stale:
                cached = False
        return cached

    def is_stale(self, tile, dimensions=None):
        """
        Return True if tile exists _and_ is expired.
        """
        if isinstance(tile, tuple):
            tile = Tile(tile)
        if self.cache.is_cached(tile):
            # tile exists
            if not self.is_cached(tile):
                # expired
                return True
            return False
        return False

    def expire_timestamp(self, tile=None):
        """
        Return the timestamp until which a tile should be accepted as up-to-date,
        or ``None`` if the tiles should not expire.

        :note: Returns _expire_timestamp by default.
        """
        return self._expire_timestamp

    def apply_tile_filter(self, tile):
        """
        Apply all `pre_store_filter` to this tile.
        Returns filtered tile.
        """
        if tile.stored:
            return tile

        for img_filter in self.pre_store_filter:
            tile = img_filter(tile)
        return tile