def main():
# Create our RenderingTheWorld tile store that will manage the queue and subdivision.
# We pass it the function that decides whether a tile should be subdivided, and an initial tile.
rendering_the_world_tilestore = RenderingTheWorldTileStore(subdivide, seeds=(Tile(TileCoord(0, 0, 0)),))
# Start the tilestream by getting a list of all tiles to be generated.
tilestream = rendering_the_world_tilestore.list()
tilestream = imap(Logger(logger, logging.INFO, 'get %(tilecoord)s'), tilestream)
# Create the tile store that will generate our tiles, in this case it's a demo WMTS server at OpenGeo.
# Getting tiles from this store will either return the tile as a PNG file, or set an error on the tile
# if there are no features in this tile.
generate_tilestore = WMTSTileStore(
url='http://v2.suite.opengeo.org/geoserver/gwc/service/wmts/',
layer='medford:buildings',
style='_null',
format='image/png',
tile_matrix_set='EPSG:900913',
tile_matrix=lambda z: 'EPSG:900913:{0:d}'.format(z))
tilestream = generate_tilestore.get(tilestream)
tilestream = imap(Logger(logger, logging.INFO, 'got %(tilecoord)s, error=%(error)s'), tilestream)
# Put the tile back into the RenderingTheWorld tile store. This check whether the tile should be
# subdivided, and, if so, adds the tile's children to the list of tiles to be generated.
tilestream = rendering_the_world_tilestore.put(tilestream)
# Get rid of tiles that returned an error (i.e. where there was no data).
tilestream = imap(DropErrors(), tilestream)
# Store the generated tiles in the output tile store, in our case a local MBTiles file.
output_tilestore = MBTilesTileStore(sqlite3.connect('medford_buildings.mbtiles'))
tilestream = output_tilestore.put(tilestream)
tilestream = imap(Logger(logger, logging.INFO, 'saved %(tilecoord)s'), tilestream)
# Go!
consume(tilestream, None)
def test_metadata(self) -> None:
tilestore = MBTilesTileStore(sqlite3.connect(":memory:"))
tilestore.put_one(Tile(TileCoord(1, 0, 0)))
tilestore.put_one(Tile(TileCoord(2, 0, 0)))
tilestore.set_metadata_zooms()
self.assertEqual(int(tilestore.metadata["minzoom"]), 1)
self.assertEqual(int(tilestore.metadata["maxzoom"]), 2)
self.assertEqual(sorted(tilestore.metadata.itervalues()), ["1", "2"])
self.assertEqual(sorted(tilestore.metadata.keys()), ["maxzoom", "minzoom"])
def test_metadata(self):
tilestore = MBTilesTileStore(sqlite3.connect(':memory:'))
tilestore.put_one(Tile(TileCoord(1, 0, 0)))
tilestore.put_one(Tile(TileCoord(2, 0, 0)))
tilestore.set_metadata_zooms()
self.assertEqual(int(tilestore.metadata['minzoom']), 1)
self.assertEqual(int(tilestore.metadata['maxzoom']), 2)
self.assertEqual(sorted(tilestore.metadata.itervalues()), ['1', '2'])
self.assertEqual(sorted(tilestore.metadata.keys()),
['maxzoom', 'minzoom'])
def load(cls, name):
root, ext = os.path.splitext(name)
if ext == '.bsddb':
import bsddb
from tilecloud.store.bsddb import BSDDBTileStore
return BSDDBTileStore(bsddb.hashopen(name))
if ext == '.mbtiles':
import sqlite3
from tilecloud.store.mbtiles import MBTilesTileStore
return MBTilesTileStore(sqlite3.connect(name))
if ext == '.zip':
import zipfile
from tilecloud.store.zip import ZipTileStore
return ZipTileStore(zipfile.ZipFile(name, 'a'))
module = __import__(name)
components = name.split('.')
module = reduce(lambda module, attr: getattr(module, attr),
components[1:], module)
return getattr(module, 'tile_store')
def test_one(self):
tilestore = MBTilesTileStore(sqlite3.connect(':memory:'),
content_type='image/png')
self.assertEqual(len(tilestore), 0)
tilestream = [
Tile(TileCoord(1, 0, 0), data=b'data'), None,
Tile(TileCoord(1, 0, 1), error=True)
]
tilestream = tilestore.put(tilestream)
tiles = list(tilestream)
self.assertEqual(len(tilestore), 2)
self.assertEqual(len(tiles), 2)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
self.assertEqual(tiles[0].data, b'data')
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
self.assertEqual(tiles[1].error, True)
self.assertTrue(Tile(TileCoord(1, 0, 0)) in tilestore)
self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
tilestream = [Tile(TileCoord(1, 0, 0)), Tile(TileCoord(1, 0, 1))]
tilestream = tilestore.get(tilestream)
consume(tilestream, None)
self.assertEqual(tilestore.get_cheap_bounding_pyramid(),
BoundingPyramid({1: (Bounds(0, 1), Bounds(0, 2))}))
self.assertEqual(len(tilestore), 2)
tiles = list(tilestore.list())
self.assertEqual(len(tiles), 2)
tiles = sorted(tilestore.get_all())
self.assertEqual(len(tiles), 2)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
self.assertEqual(bytes(tiles[0].data), b'data')
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
self.assertEqual(tiles[1].data, None)
tilestream = [Tile(TileCoord(1, 0, 0))]
tilestream = tilestore.delete(tilestream)
consume(tilestream, None)
self.assertEqual(len(tilestore), 1)
tiles = list(tilestore.get_all())
self.assertEqual(len(tiles), 1)
self.assertFalse(Tile(TileCoord(1, 0, 0)) in tilestore)
self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
def load(cls, name): # pragma: no cover
"""
Construct a :class:`TileStore` from a name.
:param name: Name
:type name: string
:rtype: :class:`TileStore`
The following shortcuts are available:
bounds://<bounding-pyramid>
file://<template>
http://<template> and https://<template>
memcached://<server>:<port>/<template>
s3://<bucket>/<template>
sqs://<region>/<queue>
<filename>.bsddb
<filename>.mbtiles
<filename>.zip
<module>
"""
if name == 'null://':
from tilecloud.store.null import NullTileStore
return NullTileStore()
if name.startswith('bounds://'):
from tilecloud.store.boundingpyramid import BoundingPyramidTileStore
return BoundingPyramidTileStore(
BoundingPyramid.from_string(name[9:]))
if name.startswith('file://'):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.filesystem import FilesystemTileStore
return FilesystemTileStore(TemplateTileLayout(name[7:]), )
if name.startswith('http://') or name.startswith('https://'):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore
return URLTileStore((TemplateTileLayout(name), ))
if name.startswith('memcached://'):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.memcached import MemcachedTileStore
from tilecloud.lib.memcached import MemcachedClient
server, template = name[12:].split('/', 1)
host, port = server.split(':', 1)
client = MemcachedClient(host, int(port))
return MemcachedTileStore(client, TemplateTileLayout(template))
if name.startswith('s3://'):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.s3 import S3TileStore
bucket, template = name[5:].split('/', 1)
return S3TileStore(bucket, TemplateTileLayout(template))
if name.startswith('sqs://'):
from tilecloud.store.sqs import SQSTileStore
import boto.sqs
from boto.sqs.jsonmessage import JSONMessage
region_name, queue_name = name[6:].split('/', 1)
connection = boto.sqs.connect_to_region(region_name)
queue = connection.create_queue(queue_name)
queue.set_message_class(JSONMessage)
return SQSTileStore(queue)
root, ext = os.path.splitext(name)
if ext == '.bsddb':
import bsddb
from tilecloud.store.bsddb import BSDDBTileStore
return BSDDBTileStore(bsddb.hashopen(name))
if ext == '.mbtiles':
import sqlite3
from tilecloud.store.mbtiles import MBTilesTileStore
return MBTilesTileStore(sqlite3.connect(name))
if ext == '.zip':
import zipfile
from tilecloud.store.zip import ZipTileStore
return ZipTileStore(zipfile.ZipFile(name, 'a'))
module = __import__(name)
components = name.split('.')
module = reduce(lambda module, attr: getattr(module, attr),
components[1:], module)
return getattr(module, 'tilestore')
def test_empty(self):
connection = sqlite3.connect(':memory:')
tilestore = MBTilesTileStore(connection)
self.assertEqual(len(tilestore), 0)
self.assertEqual(tilestore.get_one(Tile(TileCoord(0, 0, 0))), None)
def test_content_type(self):
connection = sqlite3.connect(':memory:')
tilestore1 = MBTilesTileStore(connection)
tilestore1.metadata['format'] = 'png'
tilestore2 = MBTilesTileStore(connection)
self.assertEqual(tilestore2.content_type, 'image/png')
def get_store(self, cache, layer, read_only=False):
grid = layer['grid_ref'] if 'grid_ref' in layer else None
layout = WMTSTileLayout(
layer=layer['name'],
url=cache['folder'],
style=layer['wmts_style'],
format='.' + layer['extension'],
dimensions_name=[
dimension['name'] for dimension in layer['dimensions']
],
tile_matrix_set=layer['grid'],
tile_matrix=lambda z: get_tile_matrix_identifier(grid, zoom=z),
request_encoding='REST',
)
# store
if cache['type'] == 's3':
# on s3
cache_tilestore = S3TileStore(
cache['bucket'],
layout,
s3_host=cache.get('host'),
cache_control=cache.get('cache_control')) # pragma: no cover
elif cache['type'] == 'mbtiles':
metadata = {}
for dimension in layer['dimensions']:
metadata['dimension_' +
dimension['name']] = dimension['default']
# on mbtiles file
filename = layout.filename(TileCoord(
0, 0, 0), metadata=metadata).replace('/0/0/0', '') + '.mbtiles'
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
cache_tilestore = MBTilesTileStore(
sqlite3.connect(filename),
content_type=layer['mime_type'],
tilecoord_in_topleft=True,
)
elif cache['type'] == 'bsddb':
metadata = {}
for dimension in layer['dimensions']:
metadata['dimension_' +
dimension['name']] = dimension['default']
import bsddb3 as bsddb
from tilecloud.store.bsddb import BSDDBTileStore
# on bsddb file
filename = layout.filename(TileCoord(
0, 0, 0), metadata=metadata).replace('/0/0/0', '') + '.bsddb'
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
db = bsddb.hashopen(
filename,
# and os.path.exists(filename) to avoid error on non existing file
'r' if read_only and os.path.exists(filename) else 'c')
class Close:
def __init__(self, db):
self.db = db
def __call__(self):
self.db.close()
self._close_actions.append(Close(db))
cache_tilestore = BSDDBTileStore(
db,
content_type=layer['mime_type'],
)
elif cache['type'] == 'filesystem':
# on filesystem
cache_tilestore = FilesystemTileStore(
layout,
content_type=layer['mime_type'],
)
else:
exit('unknown cache type: ' + cache['type']) # pragma: no cover
return cache_tilestore
def load(name): # pragma: no cover
"""
Construct a :class:`TileStore` from a name.
:param name: Name
:type name: string
:rtype: :class:`TileStore`
The following shortcuts are available:
bounds://<bounding-pyramid>
file://<template>
http://<template> and https://<template>
memcached://<server>:<port>/<template>
s3://<bucket>/<template>
sqs://<region>/<queue>
<filename>.bsddb
<filename>.mbtiles
<filename>.zip
<module>
"""
if name == "null://":
from tilecloud.store.null import NullTileStore
return NullTileStore()
if name.startswith("bounds://"):
from tilecloud.store.boundingpyramid import BoundingPyramidTileStore
return BoundingPyramidTileStore(
BoundingPyramid.from_string(name[9:]))
if name.startswith("file://"):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.filesystem import FilesystemTileStore
return FilesystemTileStore(TemplateTileLayout(name[7:]), )
if name.startswith("http://") or name.startswith("https://"):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.url import URLTileStore
return URLTileStore((TemplateTileLayout(name), ))
if name.startswith("memcached://"):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.lib.memcached import MemcachedClient
from tilecloud.store.memcached import MemcachedTileStore
server, template = name[12:].split("/", 1)
host, port = server.split(":", 1)
client = MemcachedClient(host, int(port))
return MemcachedTileStore(client, TemplateTileLayout(template))
if name.startswith("s3://"):
from tilecloud.layout.template import TemplateTileLayout
from tilecloud.store.s3 import S3TileStore
bucket, template = name[5:].split("/", 1)
return S3TileStore(bucket, TemplateTileLayout(template))
if name.startswith("sqs://"):
import boto.sqs # pylint: disable=import-error
from boto.sqs.jsonmessage import JSONMessage # pylint: disable=import-error
from tilecloud.store.sqs import SQSTileStore
region_name, queue_name = name[6:].split("/", 1)
connection = boto.sqs.connect_to_region(region_name)
queue = connection.create_queue(queue_name)
queue.set_message_class(JSONMessage)
return SQSTileStore(queue)
if name.startswith("redis://"):
from tilecloud.store.redis import RedisTileStore
return RedisTileStore(name)
_, ext = os.path.splitext(name)
if ext == ".bsddb":
import bsddb # pylint: disable=import-error
from tilecloud.store.bsddb import BSDDBTileStore
return BSDDBTileStore(bsddb.hashopen(name))
if ext == ".mbtiles":
import sqlite3
from tilecloud.store.mbtiles import MBTilesTileStore
return MBTilesTileStore(sqlite3.connect(name))
if ext == ".zip":
import zipfile
from tilecloud.store.zip import ZipTileStore
return ZipTileStore(zipfile.ZipFile(name, "a"))
module = __import__(name)
components = name.split(".")
module = reduce(getattr, components[1:], module)
return getattr(module, "tilestore")
def test_content_type(self) -> None:
connection = sqlite3.connect(":memory:")
tilestore1 = MBTilesTileStore(connection)
tilestore1.metadata["format"] = "png"
tilestore2 = MBTilesTileStore(connection)
self.assertEqual(tilestore2.content_type, "image/png")
