def get_one(self, tile: Tile) -> Optional[Tile]:
bbox = self.tilegrid.extent(tile.tilecoord, self.buffer)
bbox2d = mapnik.Box2d(bbox[0], bbox[1], bbox[2], bbox[3])
size = tile.tilecoord.n * self.tilegrid.tile_size + 2 * self.buffer
self.mapnik.resize(size, size)
self.mapnik.zoom_to_box(bbox2d)
if self.output_format == "grid":
grid = mapnik.Grid(self.tilegrid.tile_size,
self.tilegrid.tile_size)
for n, l in enumerate(self.mapnik.layers):
if l.name in self.layers_fields:
mapnik.render_layer(self.mapnik,
grid,
layer=n,
fields=self.layers_fields[l.name])
encode = grid.encode("utf", resolution=self.resolution)
if self.drop_empty_utfgrid and len(encode["data"].keys()) == 0:
return None
tile.data = dumps(encode).encode()
else:
# Render image with default Agg renderer
im = mapnik.Image(size, size)
mapnik.render(self.mapnik, im)
tile.data = im.tostring(self.output_format)
return tile
def _decode_tile(data: bytes, tile: Tile) -> None:
"""Decode a tile."""
image_len = struct.unpack("q", data[:8])[0]
tile.data = data[8:(image_len + 8)]
other = json.loads((data[(8 + image_len):]).decode("utf-8"))
tile.content_encoding = other["content_encoding"]
tile.content_type = other["content_type"]
def get(self, tiles: Iterable[Tile]) -> Iterator[Tile]:
for metatile in tiles:
if isinstance(metatile.data, bytes):
metaimage = Image.open(BytesIO(metatile.data))
for tilecoord in metatile.tilecoord:
if metatile.error:
yield Tile(tilecoord,
metadata=metatile.metadata,
error=metatile.error,
metatile=metatile)
continue
if metatile.data is None:
yield Tile(
tilecoord,
metadata=metatile.metadata,
error="Metatile data is None",
metatile=metatile,
)
continue
x = self.border + (tilecoord.x -
metatile.tilecoord.x) * self.tile_size
y = self.border + (tilecoord.y -
metatile.tilecoord.y) * self.tile_size
image = metaimage.crop(
(x, y, x + self.tile_size, y + self.tile_size))
bytes_io = BytesIO()
image.save(bytes_io, FORMAT_BY_CONTENT_TYPE[self.format])
yield Tile(
tilecoord,
data=bytes_io.getvalue(),
content_type=self.format,
metadata=metatile.metadata,
metatile=metatile,
)
def get_one(self, tile: Tile) -> Optional[Tile]:
try:
tile.content_type = self.content_type
tile.data = self.db[str(tile.tilecoord).encode("utf-8")]
return tile
except KeyError:
return None
def get(self, tiles):
for metatile in tiles:
metaimage = None if metatile.data is None else Image.open(BytesIO(metatile.data))
for tilecoord in metatile.tilecoord:
if metatile.error:
yield Tile(
tilecoord, metadata=metatile.metadata, error=metatile.error, metatile=metatile
)
continue
if metatile.data is None:
yield Tile(
tilecoord, metadata=metatile.metadata,
error="Metatile data is None", metatile=metatile
)
continue
x = self.border + (tilecoord.x - metatile.tilecoord.x) * self.tile_size
y = self.border + (tilecoord.y - metatile.tilecoord.y) * self.tile_size
image = metaimage.crop((x, y, x + self.tile_size, y + self.tile_size))
string_io = BytesIO()
image.save(string_io, FORMAT_BY_CONTENT_TYPE[self.format])
yield Tile(
tilecoord, data=string_io.getvalue(), content_type=self.format,
metadata=metatile.metadata, metatile=metatile
)
def __call__(self, tile: Tile) -> Tile:
if tile.content_type != self.content_type:
assert tile.data is not None
bytes_io = BytesIO()
PIL.Image.open(BytesIO(tile.data)).save(bytes_io, self.format, **self.kwargs)
tile.content_type = self.content_type
tile.data = bytes_io.getvalue()
return tile
def get_one(self, tile: Tile) -> Optional[Tile]:
try:
tile.data = self.tiles[tile.tilecoord]
except KeyError:
return None
if self.content_type is not None:
tile.content_type = self.content_type
return tile
def test_metadata(self) -> None:
tilestore = MBTilesTileStore(sqlite3.connect(":memory:"))
tilestore.put_one(Tile(TileCoord(1, 0, 0)))
tilestore.put_one(Tile(TileCoord(2, 0, 0)))
tilestore.set_metadata_zooms()
self.assertEqual(int(tilestore.metadata["minzoom"]), 1)
self.assertEqual(int(tilestore.metadata["maxzoom"]), 2)
self.assertEqual(sorted(tilestore.metadata.itervalues()), ["1", "2"])
self.assertEqual(sorted(tilestore.metadata.keys()), ["maxzoom", "minzoom"])
def get_one(self, tile: Tile) -> Optional[Tile]:
if tile is None:
return None
if hasattr(tile, "zipinfo"):
tile.data = self.zipfile.read(tile.zipinfo) # type: ignore
else:
filename = self.layout.filename(tile.tilecoord, tile.metadata)
tile.data = self.zipfile.read(filename)
return tile
def test_metadata(self):
tilestore = MBTilesTileStore(sqlite3.connect(':memory:'))
tilestore.put_one(Tile(TileCoord(1, 0, 0)))
tilestore.put_one(Tile(TileCoord(2, 0, 0)))
tilestore.set_metadata_zooms()
self.assertEqual(int(tilestore.metadata['minzoom']), 1)
self.assertEqual(int(tilestore.metadata['maxzoom']), 2)
self.assertEqual(sorted(tilestore.metadata.itervalues()), ['1', '2'])
self.assertEqual(sorted(tilestore.metadata.keys()),
['maxzoom', 'minzoom'])
def test_one(self):
tilestore = DictTileStore()
self.assertEqual(len(tilestore), 0)
tilestream = [
Tile(TileCoord(1, 0, 0), data='data'), None,
Tile(TileCoord(1, 0, 1), error=True)
]
tilestream = tilestore.put(tilestream)
tiles = list(tilestream)
self.assertEqual(len(tiles), 2)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
self.assertEqual(tiles[0].data, 'data')
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
self.assertEqual(tiles[1].error, True)
self.assertTrue(Tile(TileCoord(1, 0, 0)) in tilestore)
self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
tilestream = [Tile(TileCoord(1, 0, 0)), Tile(TileCoord(1, 0, 1))]
tilestream = tilestore.get(tilestream)
consume(tilestream, None)
tiles = list(tilestore.get_all())
self.assertEqual(len(tiles), 2)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
self.assertEqual(tiles[0].data, 'data')
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
self.assertEqual(tiles[1].error, True)
tilestream = [Tile(TileCoord(1, 0, 0))]
tilestream = tilestore.delete(tilestream)
consume(tilestream, None)
tiles = list(tilestore.get_all())
self.assertEqual(len(tiles), 1)
self.assertFalse(Tile(TileCoord(1, 0, 0)) in tilestore)
self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
def get_one(self, tile):
if self.bounding_pyramid is not None:
if tile.tilecoord not in self.bounding_pyramid:
return None
try:
tile = Tile(tile.tilecoord, data=self.tiles[tile.tilecoord])
except KeyError:
return None
if self.content_type is not None:
tile.content_type = self.content_type
return tile
def get_one(self, tile):
if self.bounding_pyramid is not None:
if tile.tilecoord not in self.bounding_pyramid:
return None
try:
tile = Tile(tile.tilecoord, data=self.tiles[tile.tilecoord])
except KeyError:
return None
if self.content_type is not None:
tile.content_type = self.content_type
return tile
def __call__(self, tile: Tile) -> Tile:
if self.content_type is None and tile.content_encoding is None and tile.data is not None:
data = str(tile.data)
if data.startswith("{"):
tile.content_type = "application/json"
elif data.startswith("\x89PNG\x0d\x0a\x1a\x0a"):
tile.content_type = "image/png"
elif data.startswith("\xff\xd8"):
tile.content_type = "image/jpeg"
else:
tile.content_type = self.content_type
return tile
def __call__(self, tile: Tile) -> Tile:
assert tile.data is not None
assert tile.content_encoding is None
bytes_io = BytesIO()
gzip_file = GzipFile(compresslevel=self.compresslevel,
fileobj=bytes_io,
mode="w")
gzip_file.write(tile.data)
gzip_file.close()
tile.content_encoding = "gzip"
tile.data = bytes_io.getvalue()
return tile
def get_one(self, tile: Tile) -> Optional[Tile]:
key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata)
try:
response = self.client.get_object(Bucket=self.bucket, Key=key_name)
tile.data = response["Body"].read()
tile.content_encoding = response.get("ContentEncoding")
tile.content_type = response.get("ContentType")
except botocore.exceptions.ClientError as exc:
if _get_status(exc) == 404:
return None
else:
tile.error = exc
return tile
def get_one(self, tile: Tile) -> Optional[Tile]:
key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata)
try:
blob = self.container_client.get_blob_client(blob=key_name)
tile.data = blob.download_blob().readall()
properties = blob.get_blob_properties()
tile.content_encoding = properties.content_settings.content_encoding
tile.content_type = properties.content_settings.content_type
except ResourceNotFoundError:
return None
except Exception as exc:
LOGGER.exception(exc)
tile.error = exc
return tile
def get_one(self, tile):
if not tile:
return None
test_tile = Tile(tile.tilecoord)
while test_tile.tilecoord:
if test_tile in self.tilestore:
tmp_tilecoord = tile.tilecoord
tile.tilecoord = test_tile.tilecoord
tile = self.tilestore.get_one(tile)
if tile:
tile.tilecoord = tmp_tilecoord
return tile
else:
test_tile.tilecoord = self.tile_structure.parent(test_tile.tilecoord)
return None
def get_one(self, tile):
if not tile:
return None
test_tile = Tile(tile.tilecoord)
while test_tile.tilecoord:
if test_tile in self.tilestore:
tmp_tilecoord = tile.tilecoord
tile.tilecoord = test_tile.tilecoord
tile = self.tilestore.get_one(tile)
if tile:
tile.tilecoord = tmp_tilecoord
return tile
else:
test_tile.tilecoord = self.tilegrid.parent(test_tile.tilecoord)
return None
def list(self):
# FIXME warn that this consumes file
filename_re = re.compile(self.tilelayout.pattern)
for line in self.file:
match = filename_re.search(line)
if match:
yield Tile(self.tilelayout.tilecoord(match.group()), line=line)
def get_one(self, tile):
try:
return Tile(tile.tilecoord,
content_type=self.content_type,
data=self.db[str(tile.tilecoord)])
except KeyError:
return None
def __call__(self, tile):
if len(tile.data) != self.size or \
sha1(tile.data).hexdigest() != self.sha1code:
return tile
else:
if self.store is not None:
if tile.tilecoord.n != 1:
for tilecoord in tile.tilecoord:
self.store.delete_one(
Tile(tilecoord, metadata=tile.metadata))
else:
self.store.delete_one(tile)
logger.info("The tile {} {} is dropped".format(
tile.tilecoord, tile.formated_metadata))
if hasattr(tile, 'metatile'):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0 and self.queue_store is not None:
self.queue_store.delete_one(
tile.metatile) # pragma: no cover
elif self.queue_store is not None: # pragma: no cover
self.queue_store.delete_one(tile)
if self.count:
self.count()
return None
def test_empty(self) -> None:
ts = TileStore()
self.assertEqual(ts.bounding_pyramid, None)
self.assertEqual(ts.content_type, None)
self.assertEqual(len(ts), 0)
self.assertRaises(NotImplementedError, next, ts.delete((Tile(TileCoord(0, 0, 0)),)))
self.assertRaises(NotImplementedError, ts.delete_one, None)
self.assertEqual(ts.get_cheap_bounding_pyramid(), None)
self.assertRaises(NotImplementedError, next, ts.get((Tile(TileCoord(0, 0, 0)),)))
self.assertEqual(list(ts.get_all()), [])
self.assertRaises(NotImplementedError, ts.get_one, None)
self.assertEqual(list(ts.list()), [])
self.assertRaises(NotImplementedError, next, ts.put((Tile(TileCoord(0, 0, 0)),)))
self.assertRaises(NotImplementedError, ts.put_one, None)
self.assertFalse(None in ts)
self.assertEqual(ts.get_bounding_pyramid(), BoundingPyramid())
def main():
# Create our RenderingTheWorld tile store that will manage the queue and subdivision.
# We pass it the function that decides whether a tile should be subdivided, and an initial tile.
rendering_the_world_tilestore = RenderingTheWorldTileStore(subdivide, seeds=(Tile(TileCoord(0, 0, 0)),))
# Start the tilestream by getting a list of all tiles to be generated.
tilestream = rendering_the_world_tilestore.list()
tilestream = imap(Logger(logger, logging.INFO, 'get %(tilecoord)s'), tilestream)
# Create the tile store that will generate our tiles, in this case it's a demo WMTS server at OpenGeo.
# Getting tiles from this store will either return the tile as a PNG file, or set an error on the tile
# if there are no features in this tile.
generate_tilestore = WMTSTileStore(
url='http://v2.suite.opengeo.org/geoserver/gwc/service/wmts/',
layer='medford:buildings',
style='_null',
format='image/png',
tile_matrix_set='EPSG:900913',
tile_matrix=lambda z: 'EPSG:900913:{0:d}'.format(z))
tilestream = generate_tilestore.get(tilestream)
tilestream = imap(Logger(logger, logging.INFO, 'got %(tilecoord)s, error=%(error)s'), tilestream)
# Put the tile back into the RenderingTheWorld tile store. This check whether the tile should be
# subdivided, and, if so, adds the tile's children to the list of tiles to be generated.
tilestream = rendering_the_world_tilestore.put(tilestream)
# Get rid of tiles that returned an error (i.e. where there was no data).
tilestream = imap(DropErrors(), tilestream)
# Store the generated tiles in the output tile store, in our case a local MBTiles file.
output_tilestore = MBTilesTileStore(sqlite3.connect('medford_buildings.mbtiles'))
tilestream = output_tilestore.put(tilestream)
tilestream = imap(Logger(logger, logging.INFO, 'saved %(tilecoord)s'), tilestream)
# Go!
consume(tilestream, None)
def test_dropping_too_many_retries(store):
if REDIS_VERSION < [5, 0, 4]:
# Bug in redis x5.0.3 that doesn't increment the retry counter.
pytest.skip("Bug in redis")
for y in range(10):
store.put_one(Tile(TileCoord(0, 0, y)))
count = 0
nb_tries = 0
# max_retries=2 => 2 iterations to have the error two times and a third one to drop the message
for _ in range(3):
try:
for tile in store.list():
if tile.tilecoord.y == 0: # this tile always fails and will be dropped after two tries
nb_tries += 1
assert nb_tries <= 2
raise SlaveException
count += 1
store.delete_one(tile)
except SlaveException:
pass
assert 9 == count
# test we see the tile in the list of errors
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 0
assert messages["Approximate number of generating tiles"] == 0
assert messages["Tiles in error"] == "0/0/0"
# test old errors deleting
time.sleep(1.1)
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 0
assert messages["Approximate number of generating tiles"] == 0
assert messages["Tiles in error"] == ""
def list(self):
for zipinfo in self.zipfile.infolist():
try:
yield Tile(self.layout.tilecoord(zipinfo.filename),
zipinfo=zipinfo)
except ValueError:
pass
def list(self):
for x in range(0, self.width):
for y in range(0, self.height):
if self.pixels[x, y]:
yield Tile(
TileCoord(self.z, self.xbounds.start + x,
self.ybounds.stop - y - 1))
def test_init_boundingpyramid(self):
ts = TileStore(
bounding_pyramid=BoundingPyramid.from_string('1/0/0:1/1'))
self.assertTrue(Tile(TileCoord(1, 0, 0)) in ts)
tiles = list(ts.list())
self.assertEqual(len(tiles), 1)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
def list(self):
while True:
sqs_messages = self.queue.receive_messages(
MaxNumberOfMessages=BATCH_SIZE)
if not sqs_messages:
try:
self.on_empty(self.queue)
except StopIteration:
break
else:
for sqs_message in sqs_messages:
try:
body = json.loads(
base64.b64decode(sqs_message.body.encode(
'utf-8')).decode('utf-8'))
z = body.get('z')
x = body.get('x')
y = body.get('y')
n = body.get('n')
metadata = body.get('metadata', {})
# FIXME deserialize other attributes
tile = Tile(TileCoord(z, x, y, n),
sqs_message=sqs_message,
metadata=metadata)
yield tile
except Exception:
logger.warning('Failed decoding the SQS message',
exc_info=True)
sqs_message.delete()
def test_get(self):
image = Image.new("RGBA", (8, 8))
image.paste((255, 0, 0, 0), (0, 0, 4, 4))
image.paste((0, 255, 0, 0), (0, 4, 4, 8))
image.paste((0, 0, 255, 0), (4, 0, 8, 4))
image.paste((0, 0, 0, 255), (4, 4, 8, 8))
string_io = BytesIO()
image.save(string_io, "PNG")
tile = Tile(TileCoord(1, 0, 0, 2), data=string_io.getvalue())
tiles = list(self.mtsts.get([tile]))
self.assertEqual(len(tiles), 4)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
image = Image.open(BytesIO(tiles[0].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (255, 0, 0, 0))])
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
image = Image.open(BytesIO(tiles[1].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (0, 255, 0, 0))])
self.assertEqual(tiles[2].tilecoord, TileCoord(1, 1, 0))
image = Image.open(BytesIO(tiles[2].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (0, 0, 255, 0))])
self.assertEqual(tiles[3].tilecoord, TileCoord(1, 1, 1))
image = Image.open(BytesIO(tiles[3].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (0, 0, 0, 255))])
def list(self) -> Iterator[Tile]:
top = getattr(self.tilelayout, "prefix", ".")
for dirpath, _, filenames in os.walk(top):
for filename in filenames:
path = os.path.join(dirpath, filename)
tilecoord = self.tilelayout.tilecoord(path)
if tilecoord:
yield Tile(tilecoord, path=path)
def get_one(self, tile: Tile) -> Optional[Tile]:
try:
filename = self.tilelayout.filename(tile.tilecoord, tile.metadata)
except Exception as e:
tile.error = e
return tile
try:
with open(filename, "rb") as file:
tile.data = file.read()
if self.content_type is not None:
tile.content_type = self.content_type
return tile
except OSError as e:
if e.errno == errno.ENOENT:
return None
else:
raise
def get_all(self):
for tilecoord, data in self.tiles.iteritems():
tile = Tile(tilecoord, data=data)
if self.content_type is not None:
tile.content_type = self.content_type
yield tile