def get(self, tiles):
for metatile in tiles:
metaimage = None if metatile.data is None else Image.open(BytesIO(metatile.data))
for tilecoord in metatile.tilecoord:
if metatile.error:
yield Tile(
tilecoord, metadata=metatile.metadata, error=metatile.error, metatile=metatile
)
continue
if metatile.data is None:
yield Tile(
tilecoord, metadata=metatile.metadata,
error="Metatile data is None", metatile=metatile
)
continue
x = self.border + (tilecoord.x - metatile.tilecoord.x) * self.tile_size
y = self.border + (tilecoord.y - metatile.tilecoord.y) * self.tile_size
image = metaimage.crop((x, y, x + self.tile_size, y + self.tile_size))
string_io = BytesIO()
image.save(string_io, FORMAT_BY_CONTENT_TYPE[self.format])
yield Tile(
tilecoord, data=string_io.getvalue(), content_type=self.format,
metadata=metatile.metadata, metatile=metatile
)
def get(self, tiles: Iterable[Tile]) -> Iterator[Tile]:
for metatile in tiles:
if isinstance(metatile.data, bytes):
metaimage = Image.open(BytesIO(metatile.data))
for tilecoord in metatile.tilecoord:
if metatile.error:
yield Tile(tilecoord,
metadata=metatile.metadata,
error=metatile.error,
metatile=metatile)
continue
if metatile.data is None:
yield Tile(
tilecoord,
metadata=metatile.metadata,
error="Metatile data is None",
metatile=metatile,
)
continue
x = self.border + (tilecoord.x -
metatile.tilecoord.x) * self.tile_size
y = self.border + (tilecoord.y -
metatile.tilecoord.y) * self.tile_size
image = metaimage.crop(
(x, y, x + self.tile_size, y + self.tile_size))
bytes_io = BytesIO()
image.save(bytes_io, FORMAT_BY_CONTENT_TYPE[self.format])
yield Tile(
tilecoord,
data=bytes_io.getvalue(),
content_type=self.format,
metadata=metatile.metadata,
metatile=metatile,
)
def test_metadata(self) -> None:
tilestore = MBTilesTileStore(sqlite3.connect(":memory:"))
tilestore.put_one(Tile(TileCoord(1, 0, 0)))
tilestore.put_one(Tile(TileCoord(2, 0, 0)))
tilestore.set_metadata_zooms()
self.assertEqual(int(tilestore.metadata["minzoom"]), 1)
self.assertEqual(int(tilestore.metadata["maxzoom"]), 2)
self.assertEqual(sorted(tilestore.metadata.itervalues()), ["1", "2"])
self.assertEqual(sorted(tilestore.metadata.keys()), ["maxzoom", "minzoom"])
def test_metadata(self):
tilestore = MBTilesTileStore(sqlite3.connect(':memory:'))
tilestore.put_one(Tile(TileCoord(1, 0, 0)))
tilestore.put_one(Tile(TileCoord(2, 0, 0)))
tilestore.set_metadata_zooms()
self.assertEqual(int(tilestore.metadata['minzoom']), 1)
self.assertEqual(int(tilestore.metadata['maxzoom']), 2)
self.assertEqual(sorted(tilestore.metadata.itervalues()), ['1', '2'])
self.assertEqual(sorted(tilestore.metadata.keys()),
['maxzoom', 'minzoom'])
def test_one(self):
tilestore = DictTileStore()
self.assertEqual(len(tilestore), 0)
tilestream = [
Tile(TileCoord(1, 0, 0), data='data'), None,
Tile(TileCoord(1, 0, 1), error=True)
]
tilestream = tilestore.put(tilestream)
tiles = list(tilestream)
self.assertEqual(len(tiles), 2)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
self.assertEqual(tiles[0].data, 'data')
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
self.assertEqual(tiles[1].error, True)
self.assertTrue(Tile(TileCoord(1, 0, 0)) in tilestore)
self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
tilestream = [Tile(TileCoord(1, 0, 0)), Tile(TileCoord(1, 0, 1))]
tilestream = tilestore.get(tilestream)
consume(tilestream, None)
tiles = list(tilestore.get_all())
self.assertEqual(len(tiles), 2)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
self.assertEqual(tiles[0].data, 'data')
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
self.assertEqual(tiles[1].error, True)
tilestream = [Tile(TileCoord(1, 0, 0))]
tilestream = tilestore.delete(tilestream)
consume(tilestream, None)
tiles = list(tilestore.get_all())
self.assertEqual(len(tiles), 1)
self.assertFalse(Tile(TileCoord(1, 0, 0)) in tilestore)
self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
def test_init_boundingpyramid(self):
ts = TileStore(
bounding_pyramid=BoundingPyramid.from_string('1/0/0:1/1'))
self.assertTrue(Tile(TileCoord(1, 0, 0)) in ts)
tiles = list(ts.list())
self.assertEqual(len(tiles), 1)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
def test_get(self):
image = Image.new("RGBA", (8, 8))
image.paste((255, 0, 0, 0), (0, 0, 4, 4))
image.paste((0, 255, 0, 0), (0, 4, 4, 8))
image.paste((0, 0, 255, 0), (4, 0, 8, 4))
image.paste((0, 0, 0, 255), (4, 4, 8, 8))
string_io = BytesIO()
image.save(string_io, "PNG")
tile = Tile(TileCoord(1, 0, 0, 2), data=string_io.getvalue())
tiles = list(self.mtsts.get([tile]))
self.assertEqual(len(tiles), 4)
self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
image = Image.open(BytesIO(tiles[0].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (255, 0, 0, 0))])
self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1))
image = Image.open(BytesIO(tiles[1].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (0, 255, 0, 0))])
self.assertEqual(tiles[2].tilecoord, TileCoord(1, 1, 0))
image = Image.open(BytesIO(tiles[2].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (0, 0, 255, 0))])
self.assertEqual(tiles[3].tilecoord, TileCoord(1, 1, 1))
image = Image.open(BytesIO(tiles[3].data))
self.assertEqual(image.size, (2, 2))
self.assertEqual(image.getcolors(), [(4, (0, 0, 0, 255))])
def list(self):
while True:
sqs_messages = self.queue.receive_messages(
MaxNumberOfMessages=BATCH_SIZE)
if not sqs_messages:
try:
self.on_empty(self.queue)
except StopIteration:
break
else:
for sqs_message in sqs_messages:
try:
body = json.loads(
base64.b64decode(sqs_message.body.encode(
'utf-8')).decode('utf-8'))
z = body.get('z')
x = body.get('x')
y = body.get('y')
n = body.get('n')
metadata = body.get('metadata', {})
# FIXME deserialize other attributes
tile = Tile(TileCoord(z, x, y, n),
sqs_message=sqs_message,
metadata=metadata)
yield tile
except Exception:
logger.warning('Failed decoding the SQS message',
exc_info=True)
sqs_message.delete()
def list(self):
for zipinfo in self.zipfile.infolist():
try:
yield Tile(self.layout.tilecoord(zipinfo.filename),
zipinfo=zipinfo)
except ValueError:
pass
def test_empty(self) -> None:
ts = TileStore()
self.assertEqual(ts.bounding_pyramid, None)
self.assertEqual(ts.content_type, None)
self.assertEqual(len(ts), 0)
self.assertRaises(NotImplementedError, next, ts.delete((Tile(TileCoord(0, 0, 0)),)))
self.assertRaises(NotImplementedError, ts.delete_one, None)
self.assertEqual(ts.get_cheap_bounding_pyramid(), None)
self.assertRaises(NotImplementedError, next, ts.get((Tile(TileCoord(0, 0, 0)),)))
self.assertEqual(list(ts.get_all()), [])
self.assertRaises(NotImplementedError, ts.get_one, None)
self.assertEqual(list(ts.list()), [])
self.assertRaises(NotImplementedError, next, ts.put((Tile(TileCoord(0, 0, 0)),)))
self.assertRaises(NotImplementedError, ts.put_one, None)
self.assertFalse(None in ts)
self.assertEqual(ts.get_bounding_pyramid(), BoundingPyramid())
def list(self):
# FIXME warn that this consumes file
filename_re = re.compile(self.tilelayout.pattern)
for line in self.file:
match = filename_re.search(line)
if match:
yield Tile(self.tilelayout.tilecoord(match.group()), line=line)
def __call__(self, tile):
if len(tile.data) != self.size or \
sha1(tile.data).hexdigest() != self.sha1code:
return tile
else:
if self.store is not None:
if tile.tilecoord.n != 1:
for tilecoord in tile.tilecoord:
self.store.delete_one(
Tile(tilecoord, metadata=tile.metadata))
else:
self.store.delete_one(tile)
logger.info("The tile {} {} is dropped".format(
tile.tilecoord, tile.formated_metadata))
if hasattr(tile, 'metatile'):
tile.metatile.elapsed_togenerate -= 1
if tile.metatile.elapsed_togenerate == 0 and self.queue_store is not None:
self.queue_store.delete_one(
tile.metatile) # pragma: no cover
elif self.queue_store is not None: # pragma: no cover
self.queue_store.delete_one(tile)
if self.count:
self.count()
return None
def main():
# Create our RenderingTheWorld tile store that will manage the queue and subdivision.
# We pass it the function that decides whether a tile should be subdivided, and an initial tile.
rendering_the_world_tilestore = RenderingTheWorldTileStore(subdivide, seeds=(Tile(TileCoord(0, 0, 0)),))
# Start the tilestream by getting a list of all tiles to be generated.
tilestream = rendering_the_world_tilestore.list()
tilestream = imap(Logger(logger, logging.INFO, 'get %(tilecoord)s'), tilestream)
# Create the tile store that will generate our tiles, in this case it's a demo WMTS server at OpenGeo.
# Getting tiles from this store will either return the tile as a PNG file, or set an error on the tile
# if there are no features in this tile.
generate_tilestore = WMTSTileStore(
url='http://v2.suite.opengeo.org/geoserver/gwc/service/wmts/',
layer='medford:buildings',
style='_null',
format='image/png',
tile_matrix_set='EPSG:900913',
tile_matrix=lambda z: 'EPSG:900913:{0:d}'.format(z))
tilestream = generate_tilestore.get(tilestream)
tilestream = imap(Logger(logger, logging.INFO, 'got %(tilecoord)s, error=%(error)s'), tilestream)
# Put the tile back into the RenderingTheWorld tile store. This check whether the tile should be
# subdivided, and, if so, adds the tile's children to the list of tiles to be generated.
tilestream = rendering_the_world_tilestore.put(tilestream)
# Get rid of tiles that returned an error (i.e. where there was no data).
tilestream = imap(DropErrors(), tilestream)
# Store the generated tiles in the output tile store, in our case a local MBTiles file.
output_tilestore = MBTilesTileStore(sqlite3.connect('medford_buildings.mbtiles'))
tilestream = output_tilestore.put(tilestream)
tilestream = imap(Logger(logger, logging.INFO, 'saved %(tilecoord)s'), tilestream)
# Go!
consume(tilestream, None)
def list(self):
for x in range(0, self.width):
for y in range(0, self.height):
if self.pixels[x, y]:
yield Tile(
TileCoord(self.z, self.xbounds.start + x,
self.ybounds.stop - y - 1))
def test_dropping_too_many_retries(store):
if REDIS_VERSION < [5, 0, 4]:
# Bug in redis x5.0.3 that doesn't increment the retry counter.
pytest.skip("Bug in redis")
for y in range(10):
store.put_one(Tile(TileCoord(0, 0, y)))
count = 0
nb_tries = 0
# max_retries=2 => 2 iterations to have the error two times and a third one to drop the message
for _ in range(3):
try:
for tile in store.list():
if tile.tilecoord.y == 0: # this tile always fails and will be dropped after two tries
nb_tries += 1
assert nb_tries <= 2
raise SlaveException
count += 1
store.delete_one(tile)
except SlaveException:
pass
assert 9 == count
# test we see the tile in the list of errors
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 0
assert messages["Approximate number of generating tiles"] == 0
assert messages["Tiles in error"] == "0/0/0"
# test old errors deleting
time.sleep(1.1)
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 0
assert messages["Approximate number of generating tiles"] == 0
assert messages["Tiles in error"] == ""
def get_one(self, tile):
try:
return Tile(tile.tilecoord,
content_type=self.content_type,
data=self.db[str(tile.tilecoord)])
except KeyError:
return None
def test(self):
tilestore = NullTileStore()
tile = Tile(TileCoord(0, 0, 0))
self.assertFalse(tile in tilestore)
self.assertEqual(list(tilestore.delete([tile])), [tile])
self.assertEqual(list(tilestore.list()), [])
self.assertEqual(list(tilestore.get([tile])), [tile])
self.assertEqual(list(tilestore.put([tile])), [tile])
def decode_message(text, **kwargs):
body = json.loads(base64.b64decode(text).decode("utf-8"))
z = body.get("z")
x = body.get("x")
y = body.get("y")
n = body.get("n")
metadata = body.get("metadata", {})
return Tile(TileCoord(z, x, y, n), metadata=metadata, **kwargs)
def list(self):
prefix = getattr(self.tilelayout, 'prefix', '')
for s3_key in self.s3bucket.list_objects(prefix=prefix):
try:
tilecoord = self.tilelayout.tilecoord(s3_key.name)
except ValueError:
continue
yield Tile(tilecoord, s3_key=s3_key)
def decode_message(text, **kwargs):
body = json.loads(base64.b64decode(text).decode('utf-8'))
z = body.get('z')
x = body.get('x')
y = body.get('y')
n = body.get('n')
metadata = body.get('metadata', {})
return Tile(TileCoord(z, x, y, n), metadata=metadata, **kwargs)
def list(self):
top = getattr(self.tilelayout, 'prefix', '.')
for dirpath, _, filenames in os.walk(top):
for filename in filenames:
path = os.path.join(dirpath, filename)
tilecoord = self.tilelayout.tilecoord(path)
if tilecoord:
yield Tile(tilecoord, path=path)
def list(self) -> Iterator[Tile]:
top = getattr(self.tilelayout, "prefix", ".")
for dirpath, _, filenames in os.walk(top):
for filename in filenames:
path = os.path.join(dirpath, filename)
tilecoord = self.tilelayout.tilecoord(path)
if tilecoord:
yield Tile(tilecoord, path=path)
def popleft(self):
sqs_message = self.queue.read(self.visibility_timeout)
if sqs_message is None:
return self.on_empty()
z = sqs_message.get('z')
x = sqs_message.get('x')
y = sqs_message.get('y')
return Tile(TileCoord(z, x, y), sqs_message=sqs_message)
def list(self):
prefix = getattr(self.tilelayout, 'prefix', '')
for s3_key in self.client.list_objects(Bucket=self.bucket,
Prefix=prefix):
try:
tilecoord = self.tilelayout.tilecoord(s3_key['Key'])
except ValueError:
continue
yield Tile(tilecoord)
def list(self) -> Iterator[Tile]:
prefix = getattr(self.tilelayout, "prefix", "")
for s3_key in self.client.list_objects(Bucket=self.bucket,
Prefix=prefix):
try:
tilecoord = self.tilelayout.tilecoord(s3_key["Key"])
except ValueError:
continue
yield Tile(tilecoord)
def list(self):
prefix = getattr(self.tilelayout, "prefix", "")
for blob in self.container.list_blobs(name_starts_with=prefix):
try:
tilecoord = self.tilelayout.tilecoord(blob.name)
except ValueError:
continue
yield Tile(tilecoord)
def _tilestream(tilecoords, default_metadata, all_dimensions):
for tilecoord in tilecoords:
for dimensions in all_dimensions:
metadata = {}
if default_metadata is not None:
metadata.update(default_metadata)
for k, v in dimensions.items():
metadata["dimension_" + k] = v
yield Tile(tilecoord, metadata=metadata)
def list(self) -> Iterator[Tile]:
assert self.xbounds.start is not None
assert self.ybounds.stop is not None
for x in range(0, self.width):
for y in range(0, self.height):
if self.pixels[x, y]:
yield Tile(
TileCoord(self.z, self.xbounds.start + x,
self.ybounds.stop - y - 1))
def test_list(store):
for y in range(10):
store.put_one(Tile(TileCoord(0, 0, y)))
count = 0
for y, tile in enumerate(store.list()):
print(repr(tile))
assert y == tile.tilecoord.y
count += 1
assert 10 == count
def get(self, tiles):
for metatile in tiles:
metaimage = Image.open(StringIO(metatile.data))
for tilecoord in metatile.tilecoord:
x = self.border + (tilecoord.x - metatile.tilecoord.x) * self.tile_size
y = self.border + (tilecoord.y - metatile.tilecoord.y) * self.tile_size
image = metaimage.crop((x, y, x + self.tile_size, y + self.tile_size))
string_io = StringIO()
image.save(string_io, FORMAT_BY_CONTENT_TYPE[self.format])
yield Tile(tilecoord, data=string_io.getvalue(), content_type=self.format)
