def tilecoord_from_quadcode(quadcode):
z, x, y = len(quadcode), 0, 0
for i, c in enumerate(quadcode):
mask = 1 << (z - i - 1)
if c == '1' or c == '3':
x |= mask
if c == '2' or c == '3':
y |= mask
return TileCoord(z, x, y)
def popleft(self):
sqs_message = self.queue.read(self.visibility_timeout)
if sqs_message is None:
return self.on_empty()
z = sqs_message.get('z')
x = sqs_message.get('x')
y = sqs_message.get('y')
metadata = sqs_message.get('metadata', {})
return Tile(TileCoord(z, x, y), sqs_message=sqs_message, **metadata)
def children(self, tilecoord):
if tilecoord.z < len(self.resolutions):
for child_z in self.child_zs[tilecoord.z]:
factor = self.resolutions[tilecoord.z] / self.resolutions[child_z]
for i in xrange(0, int(factor)):
x = factor * tilecoord.x + i
for j in xrange(0, int(factor)):
y = factor * tilecoord.y + j
yield TileCoord(child_z, x, y)
def list(self) -> Iterator[Tile]:
assert self.xbounds.start is not None
assert self.ybounds.stop is not None
for x in range(0, self.width):
for y in range(0, self.height):
if self.pixels[x, y]:
yield Tile(
TileCoord(self.z, self.xbounds.start + x,
self.ybounds.stop - y - 1))
def tilecoord_from_quadcode(quadcode):
z, x, y = len(quadcode), 0, 0
for i, c in enumerate(quadcode):
mask = 1 << (z - i - 1)
if c in ["1", "3"]:
x |= mask
if c in ["2", "3"]:
y |= mask
return TileCoord(z, x, y)
def children(self, tilecoord: TileCoord) -> Iterator[TileCoord]:
if tilecoord.z < len(self.resolutions):
for child_z in self.child_zs[tilecoord.z]:
factor = self.resolutions[tilecoord.z] / self.resolutions[child_z]
for i in range(0, int(factor)):
x = round(factor * tilecoord.x + i)
for j in range(0, int(factor)):
y = round(factor * tilecoord.y + j)
yield TileCoord(child_z, x, y)
def test_list(store):
for y in range(10):
store.put_one(Tile(TileCoord(0, 0, y)))
count = 0
for y, tile in enumerate(store.list()):
print(repr(tile))
assert y == tile.tilecoord.y
count += 1
assert 10 == count
def test_empty(self):
ts = TileStore()
self.assertEqual(ts.bounding_pyramid, None)
self.assertEqual(ts.content_type, None)
self.assertEqual(len(ts), 0)
self.assertRaises(NotImplementedError, next,
ts.delete((Tile(TileCoord(0, 0, 0)), )))
self.assertRaises(NotImplementedError, ts.delete_one, None)
self.assertEqual(ts.get_cheap_bounding_pyramid(), None)
self.assertRaises(NotImplementedError, next,
ts.get((Tile(TileCoord(0, 0, 0)), )))
self.assertEqual(list(ts.get_all()), [])
self.assertRaises(NotImplementedError, ts.get_one, None)
self.assertEqual(list(ts.list()), [])
self.assertRaises(NotImplementedError, next,
ts.put((Tile(TileCoord(0, 0, 0)), )))
self.assertRaises(NotImplementedError, ts.put_one, None)
self.assertFalse(None in ts)
self.assertEqual(ts.get_bounding_pyramid(), BoundingPyramid())
def tilecoord(self, z, x, y):
tx = self.scale * (x - self.max_extent[0]) / (self.resolutions[z] * self.tile_size)
ty = self.scale * (y - self.max_extent[1]) / float(self.resolutions[z] * self.tile_size)
if not self.flip_y:
n = self.scale * (self.max_extent[3] - self.max_extent[1]) / \
float(self.tile_size * self.resolutions[z])
ty = n - ty
return TileCoord(z, int(floor(tx)), int(floor(ty)))
def roots(self):
for z, parent_z in enumerate(self.parent_zs):
if parent_z is None:
x, s = 0, 0
while s < self.resolutions[0]:
y, t = 0, 0
while t < self.resolutions[0]:
yield TileCoord(z, x, y)
y += 1
t += self.resolutions[z]
x += 1
s += self.resolutions[z]
def test_extent(self):
self.assertEqual(self.ftg.extent(TileCoord(0, 0, 0)), (420000, 275000, 495000, 350000))
self.assertEqual(self.ftg.extent(TileCoord(1, 0, 0)), (420000, 348000, 422000, 350000))
self.assertEqual(self.ftg.extent(TileCoord(2, 0, 0)), (420000, 349000, 421000, 350000))
self.assertEqual(self.ftg.extent(TileCoord(3, 0, 0)), (420000, 349500, 420500, 350000))
self.assertEqual(self.ftg.extent(TileCoord(1, 4, 6)), (428000, 336000, 430000, 338000))
self.assertEqual(self.ftg.extent(TileCoord(1, 5, 7)), (430000, 334000, 432000, 336000))
def test_tilecoord(self):
self.assertEqual(self.ftg.tilecoord(0, 420000.75, 349999.25), TileCoord(0, 0, 0))
self.assertEqual(self.ftg.tilecoord(1, 420000.25, 349999.75), TileCoord(1, 0, 0))
self.assertEqual(self.ftg2.tilecoord(0, 420000.01, 349999.99), TileCoord(0, 0, 0))
self.assertEqual(self.ftg2.tilecoord(0, 420010, 349990), TileCoord(0, 0, 0))
self.assertEqual(self.ftg2.tilecoord(0, 420051.19, 349948.81), TileCoord(0, 0, 0))
self.assertEqual(self.ftg2.tilecoord(0, 420051.21, 349948.79), TileCoord(0, 1, 1))
def _generate_queue(self, layer_name: Optional[str]) -> None:
if self._options.tiles:
self._gene.set_store(TilesFileStore(self._options.tiles))
return
assert layer_name is not None
assert self._gene.config_file is not None
config = self._gene.get_config(self._gene.config_file)
layer = config.config["layers"][layer_name]
if self._options.get_bbox:
try:
tilecoord = parse_tilecoord(self._options.get_bbox)
bounds = default_int(self._gene.get_grid(config, layer["grid"]).extent(tilecoord))
print(f"Tile bounds: [{','.join([str(b) for b in bounds])}]")
sys.exit()
except ValueError:
logger.exception(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n",
self._options.get_bbox,
)
sys.exit(1)
if self._options.role in ("local", "master"):
# Generate a stream of metatiles
self._gene.init_tilecoords(config, layer_name)
elif self._options.role == "hash":
layer = config.config["layers"][layer_name]
try:
z, x, y = (int(v) for v in self._options.get_hash.split("/"))
if layer.get("meta"):
self._gene.set_tilecoords(config, [TileCoord(z, x, y, layer["meta_size"])], layer_name)
else:
self._gene.set_tilecoords(config, [TileCoord(z, x, y)], layer_name)
except ValueError:
logger.exception("Tile '%s' is not in the format 'z/x/y'", self._options.get_hash)
sys.exit(1)
def test_put(store):
tiles = [Tile(TileCoord(0, 0, y)) for y in range(20)]
count = 0
for y, tile in enumerate(store.put(tiles)):
assert y == tile.tilecoord.y
count += 1
assert 20 == count
count = 0
for y, tile in enumerate(store.list()):
assert y == tile.tilecoord.y
count += 1
assert 20 == count
def _generate_queue(self, layer):
assert layer is not None
if self._options.get_bbox:
try:
tilecoord = parse_tilecoord(self._options.get_bbox)
print("Tile bounds: [{},{},{},{}]".format(
*default_int(layer["grid_ref"]["obj"].extent(tilecoord))))
sys.exit()
except ValueError: # pragma: no cover
logger.error(
"Tile '%s' is not in the format 'z/x/y' or z/x/y:+n/+n",
self._options.get_bbox,
exc_info=True,
)
sys.exit(1)
if self._options.tiles:
self._gene.set_store(
TilesFileStore(self._options.tiles, layer["name"],
self._gene.get_all_dimensions(layer)))
elif self._options.role in ("local", "master"):
# Generate a stream of metatiles
self._gene.init_tilecoords(layer)
elif self._options.role == "hash":
try:
z, x, y = (int(v) for v in self._options.get_hash.split("/"))
if layer.get("meta"):
self._gene.set_tilecoords(
[TileCoord(z, x, y, layer["meta_size"])], layer)
else:
self._gene.set_tilecoords([TileCoord(z, x, y)], layer)
except ValueError as e: # pragma: no cover
sys.exit("Tile '{}' is not in the format 'z/x/y'\n{}".format(
self._options.get_hash, repr(e)))
def test_add(self):
bp = BoundingPyramid()
bp.add(TileCoord(1, 0, 0))
self.assertEqual(len(bp), 1)
self.assertTrue(TileCoord(1, 0, 0) in bp)
self.assertFalse(TileCoord(1, 0, 1) in bp)
self.assertFalse(TileCoord(1, 1, 0) in bp)
self.assertFalse(TileCoord(1, 1, 1) in bp)
self.assertEqual(list(bp), [TileCoord(1, 0, 0)])
def test_tilecoord(self):
self.assertEqual(self.ftg.tilecoord(1, 428000, 336000),
TileCoord(1, 4, 7))
self.assertEqual(self.ftg.tilecoord(1, 428000.1, 336000.1),
TileCoord(1, 4, 6))
self.assertEqual(self.ftg.tilecoord(1, 429999.9, 337999.9),
TileCoord(1, 4, 6))
self.assertEqual(self.ftg.tilecoord(1, 430000, 338000),
TileCoord(1, 5, 6))
self.assertEqual(self.ftg.tilecoord(1, 430000, 334000),
TileCoord(1, 5, 8))
self.assertEqual(self.ftg.tilecoord(1, 431000, 335000),
TileCoord(1, 5, 7))
self.assertEqual(self.ftg.tilecoord(1, 432000, 336000),
TileCoord(1, 6, 7))
self.assertEqual(self.ftg.tilecoord(1, 432000, 333000),
TileCoord(1, 6, 8))
def test_tilecoord(self):
assert self.ftg.tilecoord(0, 420000.75,
349999.25) == TileCoord(0, 0, 0)
assert self.ftg.tilecoord(1, 420000.25,
349999.75) == TileCoord(1, 0, 0)
assert self.ftg2.tilecoord(0, 420000.01,
349999.99) == TileCoord(0, 0, 0)
assert self.ftg2.tilecoord(0, 420010, 349990) == TileCoord(0, 0, 0)
assert self.ftg2.tilecoord(0, 420051.19,
349948.81) == TileCoord(0, 0, 0)
assert self.ftg2.tilecoord(0, 420051.21,
349948.79) == TileCoord(0, 1, 1)
def test_extent(self) -> None:
assert self.ftg.extent(TileCoord(0, 0, 0)) == (420000, 275000, 495000,
350000)
assert self.ftg.extent(TileCoord(1, 0, 0)) == (420000, 348000, 422000,
350000)
assert self.ftg.extent(TileCoord(2, 0, 0)) == (420000, 349000, 421000,
350000)
assert self.ftg.extent(TileCoord(3, 0, 0)) == (420000, 349500, 420500,
350000)
assert self.ftg.extent(TileCoord(1, 4, 6)) == (428000, 336000, 430000,
338000)
assert self.ftg.extent(TileCoord(1, 5, 7)) == (430000, 334000, 432000,
336000)
def list(self):
while True:
try:
sqs_message = self.queue.read()
if sqs_message is None:
try:
self.on_empty(self.queue)
except StopIteration:
break
else:
z = sqs_message.get('z')
x = sqs_message.get('x')
y = sqs_message.get('y')
n = sqs_message.get('n')
# FIXME deserialize other attributes
tile = Tile(TileCoord(z, x, y, n), sqs_message=sqs_message)
yield tile
except SQSDecodeError as e:
logger.warning(str(e))
sqs_message.delete()
def test_png(self):
layout = WMSTileLayout(url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
assert result.netloc == 'example.com'
assert result.path == '/folder'
query = parse_qs(result.query)
assert query['LAYERS'] == ['l1,l2']
assert query['FORMAT'] == ['image/png']
assert query['TRANSPARENT'] == ['TRUE']
assert query['SERVICE'] == ['WMS']
assert query['VERSION'] == ['1.1.1']
assert query['REQUEST'] == ['GetMap']
assert query['SRS'] == ['EPSG:1000']
assert query['WIDTH'] == ['100']
assert query['HEIGHT'] == ['100']
bbox = [float(i) for i in query['BBOX'][0].split(',')]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def test_recovery_from_failing_slave(store):
for y in range(10):
store.put_one(Tile(TileCoord(0, 0, y)))
with pytest.raises(SlaveException):
for _ in store.list():
raise SlaveException # fail the processing of the first tile
count = 0
for y, tile in enumerate(store.list()):
print(repr(tile))
# weird computation for the expected value because the first tile is read last since its processing failed
assert (y + 1) % 10 == tile.tilecoord.y
count += 1
store.delete_one(tile)
assert 10 == count
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 0
assert messages["Approximate number of generating tiles"] == 0
assert messages["Tiles in error"] == ""
def test_metatile(self):
layout = WMSTileLayout(
url="http://example.com/folder",
layers="l1,l2",
srs="EPSG:1000",
format="image/png",
tilegrid=self.tilegrid,
)
result = urlparse(layout.filename(TileCoord(1, 0, 0, 2)))
assert result.netloc == "example.com"
assert result.path == "/folder"
query = parse_qs(result.query)
assert query["LAYERS"] == ["l1,l2"]
assert query["FORMAT"] == ["image/png"]
assert query["TRANSPARENT"] == ["TRUE"]
assert query["SERVICE"] == ["WMS"]
assert query["VERSION"] == ["1.1.1"]
assert query["REQUEST"] == ["GetMap"]
assert query["SRS"] == ["EPSG:1000"]
assert query["WIDTH"] == ["200"]
assert query["HEIGHT"] == ["200"]
bbox = [float(i) for i in query["BBOX"][0].split(",")]
assert bbox == [420000.0, 340000.0, 430000.0, 350000.0]
def test_list(store):
for y in range(10):
store.put_one(Tile(TileCoord(0, 0, y)))
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 10
assert messages["Approximate number of generating tiles"] == 0
count = 0
for y, tile in enumerate(store.list()):
print(repr(tile))
assert y == tile.tilecoord.y
count += 1
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 10 - y
assert messages["Approximate number of generating tiles"] == 1
store.delete_one(tile)
assert 10 == count
messages = store.get_status()
assert messages["Approximate number of tiles to generate"] == 0
assert messages["Approximate number of generating tiles"] == 0
def main():
# Create our RenderingTheWorld tile store that will manage the queue and subdivision.
# We pass it the function that decides whether a tile should be subdivided, and an initial tile.
rendering_the_world_tilestore = RenderingTheWorldTileStore(
subdivide, seeds=(Tile(TileCoord(0, 0, 0)), ))
# Start the tilestream by getting a list of all tiles to be generated.
tilestream = rendering_the_world_tilestore.list()
tilestream = imap(Logger(logger, logging.INFO, "get %(tilecoord)s"),
tilestream)
# Create the tile store that will generate our tiles, in this case it's a demo WMTS server at OpenGeo.
# Getting tiles from this store will either return the tile as a PNG file, or set an error on the tile
# if there are no features in this tile.
generate_tilestore = WMTSTileStore(
url="http://v2.suite.opengeo.org/geoserver/gwc/service/wmts/",
layer="medford:buildings",
style="_null",
format="image/png",
tile_matrix_set="EPSG:900913",
tile_matrix=lambda z: f"EPSG:900913:{z:d}",
)
tilestream = generate_tilestore.get(tilestream)
tilestream = imap(
Logger(logger, logging.INFO, "got %(tilecoord)s, error=%(error)s"),
tilestream)
# Put the tile back into the RenderingTheWorld tile store. This check whether the tile should be
# subdivided, and, if so, adds the tile's children to the list of tiles to be generated.
tilestream = rendering_the_world_tilestore.put(tilestream)
# Get rid of tiles that returned an error (i.e. where there was no data).
tilestream = imap(DropErrors(), tilestream)
# Store the generated tiles in the output tile store, in our case a local MBTiles file.
output_tilestore = MBTilesTileStore(
sqlite3.connect("medford_buildings.mbtiles"))
tilestream = output_tilestore.put(tilestream)
tilestream = imap(Logger(logger, logging.INFO, "saved %(tilecoord)s"),
tilestream)
# Go!
consume(tilestream, None)
def test_params(self):
layout = WMSTileLayout(
url='http://example.com/folder',
layers='l1,l2',
srs='EPSG:1000',
format='image/png',
tilegrid=self.tilegrid,
params={
'TRANSPARENT': 'FALSE',
'PARAM': 'Value',
'FILTER': 'l1:"field" = '
'{PARAM}'
''
},
)
result = urlparse(layout.filename(TileCoord(0, 0, 0)))
self.assertEqual(result.netloc, 'example.com')
self.assertEqual(result.path, '/folder')
query = parse_qs(result.query)
self.assertEqual(query['PARAM'], ['Value'])
self.assertEqual(query['FILTER'], ['l1:"field" = ' 'Value' ''])
self.assertEqual(query['LAYERS'], ['l1,l2'])
self.assertEqual(query['FORMAT'], ['image/png'])
self.assertEqual(query['TRANSPARENT'], ['FALSE'])
self.assertEqual(query['SERVICE'], ['WMS'])
self.assertEqual(query['VERSION'], ['1.1.1'])
self.assertEqual(query['REQUEST'], ['GetMap'])
self.assertEqual(query['SRS'], ['EPSG:1000'])
self.assertEqual(query['WIDTH'], ['100'])
self.assertEqual(query['HEIGHT'], ['100'])
bbox = [float(i) for i in query['BBOX'][0].split(',')]
self.assertEqual(len(bbox), 4)
self.assertEqual(bbox[0], 420000.0)
self.assertEqual(bbox[1], 340000.0)
self.assertEqual(bbox[2], 430000.0)
self.assertEqual(bbox[3], 350000.0)
def test_extent(self):
self.assertEqual(self.ftg.extent(TileCoord(1, 4, 6)), (428000, 336000, 430000, 338000))
self.assertEqual(self.ftg.extent(TileCoord(1, 5, 7)), (430000, 334000, 432000, 336000))
def test_extent_metatile_border(self):
self.assertEqual(self.ftg.extent(TileCoord(1, 4, 6, 2), 5), (427900, 333900, 432100, 338100))
def test_from_tuple(self):
self.assertEqual(TileCoord.from_tuple((1, 2, 3)), TileCoord(1, 2, 3))
def test_extent_border(self):
self.assertEqual(self.ftg.extent(TileCoord(1, 4, 6), 5), (427900, 335900, 430100, 338100))
def test_from_string_metatile(self):
self.assertEqual(TileCoord.from_string('1/2/3:+2/+2'), TileCoord(1, 2, 3, 2))
def get_all(self):
for key, data in self.db.items():
tile = Tile(TileCoord.from_string(key), content_type=self.content_type, data=data)
yield tile
def list(self):
return imap(lambda s: Tile(TileCoord.from_string(s)), iterkeys(self.db))
def test_extent_metatile(self):
self.assertEqual(self.ftg.extent(TileCoord(1, 4, 6, 2)), (428000, 334000, 432000, 338000))
def test_from_string(self):
self.assertEqual(TileCoord.from_string('1/2/3'), TileCoord(1, 2, 3))
