def get(self, tiles): for metatile in tiles: metaimage = None if metatile.data is None else Image.open(BytesIO(metatile.data)) for tilecoord in metatile.tilecoord: if metatile.error: yield Tile( tilecoord, metadata=metatile.metadata, error=metatile.error, metatile=metatile ) continue if metatile.data is None: yield Tile( tilecoord, metadata=metatile.metadata, error="Metatile data is None", metatile=metatile ) continue x = self.border + (tilecoord.x - metatile.tilecoord.x) * self.tile_size y = self.border + (tilecoord.y - metatile.tilecoord.y) * self.tile_size image = metaimage.crop((x, y, x + self.tile_size, y + self.tile_size)) string_io = BytesIO() image.save(string_io, FORMAT_BY_CONTENT_TYPE[self.format]) yield Tile( tilecoord, data=string_io.getvalue(), content_type=self.format, metadata=metatile.metadata, metatile=metatile )
def get(self, tiles: Iterable[Tile]) -> Iterator[Tile]: for metatile in tiles: if isinstance(metatile.data, bytes): metaimage = Image.open(BytesIO(metatile.data)) for tilecoord in metatile.tilecoord: if metatile.error: yield Tile(tilecoord, metadata=metatile.metadata, error=metatile.error, metatile=metatile) continue if metatile.data is None: yield Tile( tilecoord, metadata=metatile.metadata, error="Metatile data is None", metatile=metatile, ) continue x = self.border + (tilecoord.x - metatile.tilecoord.x) * self.tile_size y = self.border + (tilecoord.y - metatile.tilecoord.y) * self.tile_size image = metaimage.crop( (x, y, x + self.tile_size, y + self.tile_size)) bytes_io = BytesIO() image.save(bytes_io, FORMAT_BY_CONTENT_TYPE[self.format]) yield Tile( tilecoord, data=bytes_io.getvalue(), content_type=self.format, metadata=metatile.metadata, metatile=metatile, )
def test_metadata(self) -> None: tilestore = MBTilesTileStore(sqlite3.connect(":memory:")) tilestore.put_one(Tile(TileCoord(1, 0, 0))) tilestore.put_one(Tile(TileCoord(2, 0, 0))) tilestore.set_metadata_zooms() self.assertEqual(int(tilestore.metadata["minzoom"]), 1) self.assertEqual(int(tilestore.metadata["maxzoom"]), 2) self.assertEqual(sorted(tilestore.metadata.itervalues()), ["1", "2"]) self.assertEqual(sorted(tilestore.metadata.keys()), ["maxzoom", "minzoom"])
def test_metadata(self): tilestore = MBTilesTileStore(sqlite3.connect(':memory:')) tilestore.put_one(Tile(TileCoord(1, 0, 0))) tilestore.put_one(Tile(TileCoord(2, 0, 0))) tilestore.set_metadata_zooms() self.assertEqual(int(tilestore.metadata['minzoom']), 1) self.assertEqual(int(tilestore.metadata['maxzoom']), 2) self.assertEqual(sorted(tilestore.metadata.itervalues()), ['1', '2']) self.assertEqual(sorted(tilestore.metadata.keys()), ['maxzoom', 'minzoom'])
def test_one(self): tilestore = DictTileStore() self.assertEqual(len(tilestore), 0) tilestream = [ Tile(TileCoord(1, 0, 0), data='data'), None, Tile(TileCoord(1, 0, 1), error=True) ] tilestream = tilestore.put(tilestream) tiles = list(tilestream) self.assertEqual(len(tiles), 2) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0)) self.assertEqual(tiles[0].data, 'data') self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1)) self.assertEqual(tiles[1].error, True) self.assertTrue(Tile(TileCoord(1, 0, 0)) in tilestore) self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore) tilestream = [Tile(TileCoord(1, 0, 0)), Tile(TileCoord(1, 0, 1))] tilestream = tilestore.get(tilestream) consume(tilestream, None) tiles = list(tilestore.get_all()) self.assertEqual(len(tiles), 2) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0)) self.assertEqual(tiles[0].data, 'data') self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1)) self.assertEqual(tiles[1].error, True) tilestream = [Tile(TileCoord(1, 0, 0))] tilestream = tilestore.delete(tilestream) consume(tilestream, None) tiles = list(tilestore.get_all()) self.assertEqual(len(tiles), 1) self.assertFalse(Tile(TileCoord(1, 0, 0)) in tilestore) self.assertTrue(Tile(TileCoord(1, 0, 1)) in tilestore)
def test_init_boundingpyramid(self): ts = TileStore( bounding_pyramid=BoundingPyramid.from_string('1/0/0:1/1')) self.assertTrue(Tile(TileCoord(1, 0, 0)) in ts) tiles = list(ts.list()) self.assertEqual(len(tiles), 1) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0))
def test_get(self): image = Image.new("RGBA", (8, 8)) image.paste((255, 0, 0, 0), (0, 0, 4, 4)) image.paste((0, 255, 0, 0), (0, 4, 4, 8)) image.paste((0, 0, 255, 0), (4, 0, 8, 4)) image.paste((0, 0, 0, 255), (4, 4, 8, 8)) string_io = BytesIO() image.save(string_io, "PNG") tile = Tile(TileCoord(1, 0, 0, 2), data=string_io.getvalue()) tiles = list(self.mtsts.get([tile])) self.assertEqual(len(tiles), 4) self.assertEqual(tiles[0].tilecoord, TileCoord(1, 0, 0)) image = Image.open(BytesIO(tiles[0].data)) self.assertEqual(image.size, (2, 2)) self.assertEqual(image.getcolors(), [(4, (255, 0, 0, 0))]) self.assertEqual(tiles[1].tilecoord, TileCoord(1, 0, 1)) image = Image.open(BytesIO(tiles[1].data)) self.assertEqual(image.size, (2, 2)) self.assertEqual(image.getcolors(), [(4, (0, 255, 0, 0))]) self.assertEqual(tiles[2].tilecoord, TileCoord(1, 1, 0)) image = Image.open(BytesIO(tiles[2].data)) self.assertEqual(image.size, (2, 2)) self.assertEqual(image.getcolors(), [(4, (0, 0, 255, 0))]) self.assertEqual(tiles[3].tilecoord, TileCoord(1, 1, 1)) image = Image.open(BytesIO(tiles[3].data)) self.assertEqual(image.size, (2, 2)) self.assertEqual(image.getcolors(), [(4, (0, 0, 0, 255))])
def list(self): while True: sqs_messages = self.queue.receive_messages( MaxNumberOfMessages=BATCH_SIZE) if not sqs_messages: try: self.on_empty(self.queue) except StopIteration: break else: for sqs_message in sqs_messages: try: body = json.loads( base64.b64decode(sqs_message.body.encode( 'utf-8')).decode('utf-8')) z = body.get('z') x = body.get('x') y = body.get('y') n = body.get('n') metadata = body.get('metadata', {}) # FIXME deserialize other attributes tile = Tile(TileCoord(z, x, y, n), sqs_message=sqs_message, metadata=metadata) yield tile except Exception: logger.warning('Failed decoding the SQS message', exc_info=True) sqs_message.delete()
def list(self): for zipinfo in self.zipfile.infolist(): try: yield Tile(self.layout.tilecoord(zipinfo.filename), zipinfo=zipinfo) except ValueError: pass
def test_empty(self) -> None: ts = TileStore() self.assertEqual(ts.bounding_pyramid, None) self.assertEqual(ts.content_type, None) self.assertEqual(len(ts), 0) self.assertRaises(NotImplementedError, next, ts.delete((Tile(TileCoord(0, 0, 0)),))) self.assertRaises(NotImplementedError, ts.delete_one, None) self.assertEqual(ts.get_cheap_bounding_pyramid(), None) self.assertRaises(NotImplementedError, next, ts.get((Tile(TileCoord(0, 0, 0)),))) self.assertEqual(list(ts.get_all()), []) self.assertRaises(NotImplementedError, ts.get_one, None) self.assertEqual(list(ts.list()), []) self.assertRaises(NotImplementedError, next, ts.put((Tile(TileCoord(0, 0, 0)),))) self.assertRaises(NotImplementedError, ts.put_one, None) self.assertFalse(None in ts) self.assertEqual(ts.get_bounding_pyramid(), BoundingPyramid())
def list(self): # FIXME warn that this consumes file filename_re = re.compile(self.tilelayout.pattern) for line in self.file: match = filename_re.search(line) if match: yield Tile(self.tilelayout.tilecoord(match.group()), line=line)
def __call__(self, tile): if len(tile.data) != self.size or \ sha1(tile.data).hexdigest() != self.sha1code: return tile else: if self.store is not None: if tile.tilecoord.n != 1: for tilecoord in tile.tilecoord: self.store.delete_one( Tile(tilecoord, metadata=tile.metadata)) else: self.store.delete_one(tile) logger.info("The tile {} {} is dropped".format( tile.tilecoord, tile.formated_metadata)) if hasattr(tile, 'metatile'): tile.metatile.elapsed_togenerate -= 1 if tile.metatile.elapsed_togenerate == 0 and self.queue_store is not None: self.queue_store.delete_one( tile.metatile) # pragma: no cover elif self.queue_store is not None: # pragma: no cover self.queue_store.delete_one(tile) if self.count: self.count() return None
def main(): # Create our RenderingTheWorld tile store that will manage the queue and subdivision. # We pass it the function that decides whether a tile should be subdivided, and an initial tile. rendering_the_world_tilestore = RenderingTheWorldTileStore(subdivide, seeds=(Tile(TileCoord(0, 0, 0)),)) # Start the tilestream by getting a list of all tiles to be generated. tilestream = rendering_the_world_tilestore.list() tilestream = imap(Logger(logger, logging.INFO, 'get %(tilecoord)s'), tilestream) # Create the tile store that will generate our tiles, in this case it's a demo WMTS server at OpenGeo. # Getting tiles from this store will either return the tile as a PNG file, or set an error on the tile # if there are no features in this tile. generate_tilestore = WMTSTileStore( url='http://v2.suite.opengeo.org/geoserver/gwc/service/wmts/', layer='medford:buildings', style='_null', format='image/png', tile_matrix_set='EPSG:900913', tile_matrix=lambda z: 'EPSG:900913:{0:d}'.format(z)) tilestream = generate_tilestore.get(tilestream) tilestream = imap(Logger(logger, logging.INFO, 'got %(tilecoord)s, error=%(error)s'), tilestream) # Put the tile back into the RenderingTheWorld tile store. This check whether the tile should be # subdivided, and, if so, adds the tile's children to the list of tiles to be generated. tilestream = rendering_the_world_tilestore.put(tilestream) # Get rid of tiles that returned an error (i.e. where there was no data). tilestream = imap(DropErrors(), tilestream) # Store the generated tiles in the output tile store, in our case a local MBTiles file. output_tilestore = MBTilesTileStore(sqlite3.connect('medford_buildings.mbtiles')) tilestream = output_tilestore.put(tilestream) tilestream = imap(Logger(logger, logging.INFO, 'saved %(tilecoord)s'), tilestream) # Go! consume(tilestream, None)
def list(self): for x in range(0, self.width): for y in range(0, self.height): if self.pixels[x, y]: yield Tile( TileCoord(self.z, self.xbounds.start + x, self.ybounds.stop - y - 1))
def test_dropping_too_many_retries(store): if REDIS_VERSION < [5, 0, 4]: # Bug in redis x5.0.3 that doesn't increment the retry counter. pytest.skip("Bug in redis") for y in range(10): store.put_one(Tile(TileCoord(0, 0, y))) count = 0 nb_tries = 0 # max_retries=2 => 2 iterations to have the error two times and a third one to drop the message for _ in range(3): try: for tile in store.list(): if tile.tilecoord.y == 0: # this tile always fails and will be dropped after two tries nb_tries += 1 assert nb_tries <= 2 raise SlaveException count += 1 store.delete_one(tile) except SlaveException: pass assert 9 == count # test we see the tile in the list of errors messages = store.get_status() assert messages["Approximate number of tiles to generate"] == 0 assert messages["Approximate number of generating tiles"] == 0 assert messages["Tiles in error"] == "0/0/0" # test old errors deleting time.sleep(1.1) messages = store.get_status() assert messages["Approximate number of tiles to generate"] == 0 assert messages["Approximate number of generating tiles"] == 0 assert messages["Tiles in error"] == ""
def get_one(self, tile): try: return Tile(tile.tilecoord, content_type=self.content_type, data=self.db[str(tile.tilecoord)]) except KeyError: return None
def test(self): tilestore = NullTileStore() tile = Tile(TileCoord(0, 0, 0)) self.assertFalse(tile in tilestore) self.assertEqual(list(tilestore.delete([tile])), [tile]) self.assertEqual(list(tilestore.list()), []) self.assertEqual(list(tilestore.get([tile])), [tile]) self.assertEqual(list(tilestore.put([tile])), [tile])
def decode_message(text, **kwargs): body = json.loads(base64.b64decode(text).decode("utf-8")) z = body.get("z") x = body.get("x") y = body.get("y") n = body.get("n") metadata = body.get("metadata", {}) return Tile(TileCoord(z, x, y, n), metadata=metadata, **kwargs)
def list(self): prefix = getattr(self.tilelayout, 'prefix', '') for s3_key in self.s3bucket.list_objects(prefix=prefix): try: tilecoord = self.tilelayout.tilecoord(s3_key.name) except ValueError: continue yield Tile(tilecoord, s3_key=s3_key)
def decode_message(text, **kwargs): body = json.loads(base64.b64decode(text).decode('utf-8')) z = body.get('z') x = body.get('x') y = body.get('y') n = body.get('n') metadata = body.get('metadata', {}) return Tile(TileCoord(z, x, y, n), metadata=metadata, **kwargs)
def list(self): top = getattr(self.tilelayout, 'prefix', '.') for dirpath, _, filenames in os.walk(top): for filename in filenames: path = os.path.join(dirpath, filename) tilecoord = self.tilelayout.tilecoord(path) if tilecoord: yield Tile(tilecoord, path=path)
def list(self) -> Iterator[Tile]: top = getattr(self.tilelayout, "prefix", ".") for dirpath, _, filenames in os.walk(top): for filename in filenames: path = os.path.join(dirpath, filename) tilecoord = self.tilelayout.tilecoord(path) if tilecoord: yield Tile(tilecoord, path=path)
def popleft(self): sqs_message = self.queue.read(self.visibility_timeout) if sqs_message is None: return self.on_empty() z = sqs_message.get('z') x = sqs_message.get('x') y = sqs_message.get('y') return Tile(TileCoord(z, x, y), sqs_message=sqs_message)
def list(self): prefix = getattr(self.tilelayout, 'prefix', '') for s3_key in self.client.list_objects(Bucket=self.bucket, Prefix=prefix): try: tilecoord = self.tilelayout.tilecoord(s3_key['Key']) except ValueError: continue yield Tile(tilecoord)
def list(self) -> Iterator[Tile]: prefix = getattr(self.tilelayout, "prefix", "") for s3_key in self.client.list_objects(Bucket=self.bucket, Prefix=prefix): try: tilecoord = self.tilelayout.tilecoord(s3_key["Key"]) except ValueError: continue yield Tile(tilecoord)
def list(self): prefix = getattr(self.tilelayout, "prefix", "") for blob in self.container.list_blobs(name_starts_with=prefix): try: tilecoord = self.tilelayout.tilecoord(blob.name) except ValueError: continue yield Tile(tilecoord)
def _tilestream(tilecoords, default_metadata, all_dimensions): for tilecoord in tilecoords: for dimensions in all_dimensions: metadata = {} if default_metadata is not None: metadata.update(default_metadata) for k, v in dimensions.items(): metadata["dimension_" + k] = v yield Tile(tilecoord, metadata=metadata)
def list(self) -> Iterator[Tile]: assert self.xbounds.start is not None assert self.ybounds.stop is not None for x in range(0, self.width): for y in range(0, self.height): if self.pixels[x, y]: yield Tile( TileCoord(self.z, self.xbounds.start + x, self.ybounds.stop - y - 1))
def test_list(store): for y in range(10): store.put_one(Tile(TileCoord(0, 0, y))) count = 0 for y, tile in enumerate(store.list()): print(repr(tile)) assert y == tile.tilecoord.y count += 1 assert 10 == count
def get(self, tiles): for metatile in tiles: metaimage = Image.open(StringIO(metatile.data)) for tilecoord in metatile.tilecoord: x = self.border + (tilecoord.x - metatile.tilecoord.x) * self.tile_size y = self.border + (tilecoord.y - metatile.tilecoord.y) * self.tile_size image = metaimage.crop((x, y, x + self.tile_size, y + self.tile_size)) string_io = StringIO() image.save(string_io, FORMAT_BY_CONTENT_TYPE[self.format]) yield Tile(tilecoord, data=string_io.getvalue(), content_type=self.format)