def _errors_mvt(db, params, z, min_x, min_y, max_x, max_y, limit):
params.limit = limit
results = query._gets(db, params) if z >= 6 else None
if not results or len(results) == 0:
global MVT_EMPTY
if not MVT_EMPTY:
MVT_EMPTY = mapbox_vector_tile.encode([])
return MVT_EMPTY
else:
limit_feature = []
if len(results) == limit and z < 18:
limit_feature = [{
"name": "limit",
"features": [{
"geometry": Point((min_x + max_x) / 2, (min_y + max_y) / 2)
}]
}]
issues_features = []
for res in sorted(results, key=lambda res: -res["lat"]):
issues_features.append({
"geometry": Point(res["lon"], res["lat"]),
"properties": {
"issue_id": res["id"],
"item": res["item"] or 0}
})
return mapbox_vector_tile.encode([{
"name": "issues",
"features": issues_features
}] + limit_feature, quantize_bounds=(min_x, min_y, max_x, max_y))
def test_geometry_collection_raises(self):
from mapbox_vector_tile import encode
import shapely.wkt
collection = shapely.wkt.loads(
'GEOMETRYCOLLECTION (GEOMETRYCOLLECTION (POINT (4095 3664), LINESTRING (2889 0, 2889 0)), POINT (4095 3664), LINESTRING (2889 0, 2912 158, 3757 1700, 3732 1999, 4095 3277))'
) # noqa
with self.assertRaises(ValueError):
encode({'name': 'streets', 'features': [{'geometry': collection}]})
def encode(file, features, bounds, layer_name=''):
layers = []
layers.append(get_feature_layer(layer_name, features))
if bounds:
data = mapbox_vector_tile.encode(layers, quantize_bounds=bounds)
else:
data = mapbox_vector_tile.encode(layers)
file.write(data)
def test_invalid_geometry_raise(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_raise
import shapely.wkt
geometry = 'POLYGON ((10 10, 20 10, 20 20, 15 15, 15 5, 10 10))'
shape = shapely.wkt.loads(geometry)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
with self.assertRaises(Exception):
encode(source, on_invalid_geometry=on_invalid_geometry_raise)
def test_invalid_geometry_raise(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_raise
import shapely.wkt
geometry = 'POLYGON ((10 10, 20 10, 20 20, 15 15, 15 5, 10 10))'
shape = shapely.wkt.loads(geometry)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
with self.assertRaises(Exception):
encode(source, on_invalid_geometry=on_invalid_geometry_raise)
def test_with_invalid_geometry(self):
expected_result = ('Can\'t do geometries that are not wkt, wkb, or '
'shapely geometries')
with self.assertRaises(NotImplementedError) as ex:
mapbox_vector_tile.encode([{
"name": self.layer_name,
"features": [{
"geometry": "xyz",
"properties": self.feature_properties
}]
}])
self.assertEqual(str(ex.exception), expected_result)
def test_with_invalid_geometry(self):
expected_result = ('Can\'t do geometries that are not wkt, wkb, or '
'shapely geometries')
with self.assertRaises(NotImplementedError) as ex:
mapbox_vector_tile.encode([{
"name": self.layer_name,
"features": [{
"geometry": "xyz",
"properties": self.feature_properties
}]
}])
self.assertEqual(str(ex.exception), expected_result)
def test_with_invalid_geometry(self):
geometry = "xyz"
expected_result = 'Can\'t do geometries that are not wkt or wkb'
with self.assertRaises(NotImplementedError) as ex:
mapbox_vector_tile.encode([{
"name": self.layer_name,
"features": [{
"geometry": geometry,
"properties": self.feature_properties
}]
}])
self.assertEqual(ex.exception[0], expected_result)
def get_tile(self, x, y, z, extent=4096, buffer=256, clip_geom=True):
# get tile coordinates from x, y and z
west, south, east, north = self.get_bounds(x, y, z)
features = self.get_vector_tile_queryset()
pixel = self.pixel_length(z)
final_buffer = 4 * pixel
bbox = Polygon.from_bbox((west - final_buffer, south - final_buffer, east + final_buffer, north + final_buffer))
bbox.srid = 3857
filters = {
f"{self.vector_tile_geom_name}__intersects": bbox
}
features = features.filter(**filters)
features = features.annotate(clipped=Intersection(Transform(self.vector_tile_geom_name, 3857), bbox))
if features:
tile = {
"name": self.get_vector_tile_layer_name(),
"features": [
{
"geometry": feature.clipped.simplify(pixel, preserve_topology=True).wkb.tobytes(),
"properties": {
key: getattr(feature, key) for key in self.vector_tile_fields if
self.vector_tile_fields
}
}
for feature in features
],
}
return mapbox_vector_tile.encode(tile,
quantize_bounds=(west, south, east, north),
extents=extent)
def _errors_mvt(db, results, z, min_lon, min_lat, max_lon, max_lat, limit):
if not results or len(results) == 0:
return None
else:
limit_feature = []
if len(results) == limit and z < 18:
limit_feature = [{
"name": "limit",
"features": [{
"geometry": Point((min_lon + max_lon) / 2, (min_lat + max_lat) / 2)
}]
}]
issues_features = []
for res in sorted(results, key=lambda res: -res["lat"]):
issues_features.append({
"geometry": Point(res["lon"], res["lat"]),
"properties": {
"issue_id": res["id"],
"item": res["item"] or 0,
"class": res["class"] or 0}
})
return mapbox_vector_tile.encode([{
"name": "issues",
"features": issues_features
}] + limit_feature, quantize_bounds=(min_lon, min_lat, max_lon, max_lat))
def encode(file, features, coord, layer_name=''):
layers = []
layers.append(get_feature_layer(layer_name, features))
data = mapbox_vector_tile.encode(layers)
file.write(data)
async def tile_endpoint(self, request, db_name, table, z, x, y):
start = time.time()
if not valid_zoom(z):
raise NotFound("Invalid zoom value")
if not await self.datasette.table_exists(db_name, table):
raise NotFound("Table does not exist")
geo_column = get_geo_column(self.datasette, db_name, table)
if geo_column is None:
raise NotFound("Not a spatial table")
fetch = time.time()
data = await self.get_features(db_name, table, geo_column, z, x, y)
encode = time.time()
mvt = mapbox_vector_tile.encode(data,
quantize_bounds=mercantile.xy_bounds(
x, y, z))
now = time.time()
print(
"[{}/{}/{}/{}/{}] Total: {:.3}s (init: {:.3}s + fetch: {:.3}s + encode: {:.3}s)"
.format(db_name, table, z, x, y, now - start, fetch - start,
encode - fetch, now - encode))
ttl = self.datasette.config("default_cache_ttl")
if int(ttl) == 0:
ttl_header = "no-cache"
else:
ttl_header = "max-age={}".format(ttl)
return response.raw(mvt,
headers={
"Content-Type":
"application/vnd.mapbox-vector-tile",
"Cache-Control": ttl_header
})
def test_bowtie_self_crossing(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.geometry
import shapely.wkt
bowtie = ('POLYGON ((0 0, 2 2, 2 0, 0 2, 0 0))')
shape = shapely.wkt.loads(bowtie)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
self.assertEqual(1, len(result['layername']['features']))
valid_geometries = result['layername']['features'][0]['geometry']
multipolygon = shapely.geometry.shape(valid_geometries)
self.assertEqual(multipolygon.geom_type, 'MultiPolygon')
self.assertTrue(multipolygon.is_valid)
total_area = 0
for p in multipolygon.geoms:
self.assertEqual(p.geom_type, 'Polygon')
self.assertTrue(p.is_valid)
self.assertGreater(p.area, 0)
total_area += p.area
self.assertEquals(2, total_area)
def test_make_valid_can_return_multipolygon(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
import os.path
test_dir = os.path.dirname(os.path.realpath(__file__))
file_name = 'error_nested_multipolygon.wkt'
with open(os.path.join(test_dir, file_name)) as fh:
shape = wkt.loads(fh.read())
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
quantize_bounds=(-10018754.1713946, 11271098.44281893,
-8766409.899970269, 12523442.714243261),
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(1, len(features))
geom = features[0]['geometry']
self.assertEquals(geom['type'], 'MultiPolygon')
multipolygon = shapely.geometry.shape(geom)
self.assertTrue(multipolygon.is_valid)
area = 0
for p in multipolygon.geoms:
self.assertTrue(p.is_valid)
area += p.area
self.assertEquals(4339852.5, area)
def test_make_valid_self_crossing(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.geometry
import shapely.wkt
geometry = 'POLYGON ((10 10, 20 10, 20 20, 15 15, 15 5, 10 10))'
shape = shapely.wkt.loads(geometry)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
self.assertEqual(1, len(result['layername']['features']))
valid_geometries = result['layername']['features'][0]['geometry']
geom_type = result['layername']['features'][0]['type']
self.assertEqual(3, geom_type) # 3 means POLYGON
self.assertEqual(valid_geometries['type'], 'MultiPolygon')
multipolygon = shapely.geometry.shape(valid_geometries)
self.assertTrue(multipolygon.is_valid)
total_area = 0
for p in multipolygon.geoms:
self.assertTrue(p.is_valid)
self.assertGreater(p.area, 0)
total_area += p.area
self.assertEquals(50, total_area)
self.assertEquals(50, multipolygon.area)
def _errors_mvt(db, results, z, min_lon, min_lat, max_lon, max_lat, limit):
if not results or len(results) == 0:
return None
else:
limit_feature = []
if len(results) == limit and z < 18:
limit_feature = [{
"name":
"limit",
"features": [{
"geometry":
Point((min_lon + max_lon) / 2, (min_lat + max_lat) / 2)
}]
}]
issues_features = []
for res in sorted(results, key=lambda res: -res["lat"]):
issues_features.append({
"geometry": Point(res["lon"], res["lat"]),
"properties": {
"issue_id": res["id"],
"item": res["item"] or 0,
"class": res["class"] or 0
}
})
return mapbox_vector_tile.encode([{
"name": "issues",
"features": issues_features
}] + limit_feature,
quantize_bounds=(min_lon, min_lat,
max_lon, max_lat))
def assertRoundTrip(self, input_geometry, expected_geometry, name=None,
properties=None, id=None, expected_len=1,
expected_properties=None):
if input_geometry is None:
input_geometry = self.feature_geometry
if name is None:
name = self.layer_name
if properties is None:
properties = self.feature_properties
if expected_properties is None:
expected_properties = properties
source = [{
"name": name,
"features": [{
"geometry": input_geometry,
"properties": properties
}]
}]
if id:
source[0]['features'][0]['id'] = id
encoded = encode(source)
decoded = decode(encoded)
self.assertIn(name, decoded)
layer = decoded[name]
features = layer['features']
self.assertEqual(expected_len, len(features))
self.assertEqual(features[0]['properties'], expected_properties)
self.assertEqual(features[0]['geometry'], expected_geometry)
if id:
self.assertEqual(features[0]['id'], id)
def test_make_valid_can_return_multipolygon(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
import os.path
test_dir = os.path.dirname(os.path.realpath(__file__))
file_name = 'error_nested_multipolygon.wkt'
with open(os.path.join(test_dir, file_name)) as fh:
shape = wkt.loads(fh.read())
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
quantize_bounds=(-10018754.1713946, 11271098.44281893,
-8766409.899970269, 12523442.714243261),
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(1, len(features))
geom = features[0]['geometry']
area = 0
for poly in geom:
p = shapely.geometry.Polygon(poly[0], poly[1:])
self.assertTrue(p.is_valid)
area += p.area
self.assertEquals(4339852.5, area)
def assertRoundTrip(self, input_geometry, expected_geometry, name=None,
properties=None, id=None, expected_len=1,
expected_properties=None):
if input_geometry is None:
input_geometry = self.feature_geometry
if name is None:
name = self.layer_name
if properties is None:
properties = self.feature_properties
if expected_properties is None:
expected_properties = properties
source = [{
"name": name,
"features": [{
"geometry": input_geometry,
"properties": properties
}]
}]
if id:
source[0]['features'][0]['id'] = id
encoded = encode(source)
decoded = decode(encoded)
self.assertIn(name, decoded)
layer = decoded[name]
features = layer['features']
self.assertEqual(expected_len, len(features))
self.assertEqual(features[0]['properties'], expected_properties)
self.assertEqual(features[0]['geometry'], expected_geometry)
if id:
self.assertEqual(features[0]['id'], id)
def _save_tile(self, tile):
tile_id = str(uuid.uuid4())
insert_tile = "INSERT INTO map(tile_id, zoom_level, tile_column, tile_row) VALUES(?,?,?,?)"
self.conn.execute(insert_tile,
(tile_id, tile.zoom_level, tile.column, tile.row))
self._update_progress(max_progress=len(tile.decoded_data))
all_layers = []
for index, layer_name in enumerate(tile.decoded_data):
if self._cancel_requested:
return
self._update_progress(msg="Export '{}'".format(layer_name))
if layer_name in self.layers_to_export:
converted_layer = self._convert_layer(layer_name, tile)
all_layers.append(converted_layer)
self._update_progress(progress=index + 1)
encoded_data = mapbox_vector_tile.encode(all_layers)
if self._cancel_requested:
return
out = StringIO()
with GzipFile(fileobj=out, mode="w") as f:
f.write(encoded_data)
gzipped_data = out.getvalue()
insert_sql = "INSERT INTO images(tile_id, tile_data) VALUES(?,?)"
self.conn.execute(insert_sql, (tile_id, sqlite3.Binary(gzipped_data)))
def test_bowtie_self_crossing(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.geometry
import shapely.wkt
bowtie = ('POLYGON ((0 0, 2 2, 2 0, 0 2, 0 0))')
shape = shapely.wkt.loads(bowtie)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
self.assertEqual(1, len(result['layername']['features']))
valid_geometries = result['layername']['features'][0]['geometry']
multipolygon = shapely.geometry.shape(valid_geometries)
self.assertEqual(multipolygon.geom_type, 'MultiPolygon')
self.assertTrue(multipolygon.is_valid)
total_area = 0
for p in multipolygon.geoms:
self.assertEqual(p.geom_type, 'Polygon')
self.assertTrue(p.is_valid)
self.assertGreater(p.area, 0)
total_area += p.area
self.assertEquals(2, total_area)
def encode(file, features, coord, layer_name=''):
layers = []
layers.append(get_feature_layer(layer_name, features))
data = mapbox_vector_tile.encode(layers)
file.write(data)
def test_make_valid_self_crossing(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.geometry
import shapely.wkt
geometry = 'POLYGON ((10 10, 20 10, 20 20, 15 15, 15 5, 10 10))'
shape = shapely.wkt.loads(geometry)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
self.assertEqual(1, len(result['layername']['features']))
valid_geometries = result['layername']['features'][0]['geometry']
geom_type = result['layername']['features'][0]['type']
self.assertEqual(3, geom_type) # 3 means POLYGON
self.assertEqual(valid_geometries['type'], 'MultiPolygon')
multipolygon = shapely.geometry.shape(valid_geometries)
self.assertTrue(multipolygon.is_valid)
total_area = 0
for p in multipolygon.geoms:
self.assertTrue(p.is_valid)
self.assertGreater(p.area, 0)
total_area += p.area
self.assertEquals(50, total_area)
self.assertEquals(50, multipolygon.area)
def write(self, tile, x, y, z):
self.z = str(z)
self.zdir = str(z)
if self.directory != '':
self.zdir = self.directory + '/' + self.zdir
if self.zdir != '':
if not os.path.exists(self.zdir):
os.makedirs(self.zdir)
self.x = str(x)
self.xdir = self.zdir + '/' + self.x
if not os.path.exists(self.xdir):
os.makedirs(self.xdir)
self.y = str(y)
if (self.options["format"] == 'json'):
ct = GeoJSONTile(tile)
filename = self.xdir + '/' + self.y + '.json'
with open(filename, 'w') as outfile:
json.dump(ct.getContent(),
outfile,
sort_keys=True,
indent=4,
separators=(',', ': '))
# pretty print
#json.dump(ct.getContent(), outfile, sort_keys=True, indent=4, separators=(',', ': '))
if (self.options["format"] == 'pbf'):
ct = MVTile(tile, self.options["layername"], self.config)
filename = self.xdir + '/' + self.y + '.pbf'
with open(filename, 'wb') as outfile:
# json.dump(self.content, outfile)
# pretty print
outfile.write(mapbox_vector_tile.encode(ct.getContent()))
def test_encoder(self):
expected_result = '\x1aG\n\x05water\x12\x18\x12\x06\x00\x00\x01\x01\x02\x02\x18\x03"\x0c\t\x00\x80@\x1a\x00\x01\x02\x00\x00\x02\x0f\x1a\x03foo\x1a\x03baz\x1a\x03uid"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
self.assertEqual(mapbox_vector_tile.encode([{
"name": self.layer_name,
"features": [{
"geometry": self.feature_geometry,
"properties": self.feature_properties
}]
}]), expected_result)
def render(self, data, accepted_media_type=None, renderer_context=None):
"""Returns *data* encoded as GeoJSON."""
if isinstance(data, list):
data = collections.as_feature(data)
data['name'] = renderer_context['view'].layer
for index, features in enumerate(data['features']):
data['features'][index]['name'] = renderer_context['view'].layer
return mapbox_vector_tile.encode(data)
def test_with_wkt(self):
geometry = "LINESTRING(-71.160281 42.258729,-71.160837 42.259113,-71.161144 42.25932)"
expected_result = '\x1aE\n\x05water\x12\x16\x12\x06\x00\x00\x01\x01\x02\x02\x18\x02"\n\t\x8d\x01\xaa?\x12\x00\x00\x00\x00\x1a\x03foo\x1a\x03baz\x1a\x03uid"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
self.assertEqual(mapbox_vector_tile.encode([{
"name": self.layer_name,
"features": [{
"geometry": geometry,
"properties": self.feature_properties
}]
}]), expected_result)
def write(self, tile, x, y, z):
sql = '''INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES(?, ?, ?, ?);'''
if (self.options["format"] == 'pbf'):
ct = MVTile(tile, self.options["layername"], self.config)
yfliped = 2**z - 1 - y
self.cursor.execute(sql, [
z, x, yfliped,
sqlite3.Binary(
deflate(mapbox_vector_tile.encode(ct.getContent())))
])
def test_encode_float_little_endian(self):
geometry = "LINESTRING(-71.160281 42.258729,-71.160837 42.259113,-71.161144 42.25932)"
expected_result = '\x1a\\\n\x05water\x12\x18\x12\x08\x00\x00\x01\x01\x02\x02\x03\x03\x18\x02"\n\t\x8d\x01\xaa?\x12\x00\x00\x00\x00\x1a\x08floatval\x1a\x03foo\x1a\x03baz\x1a\x03uid"\t\x19n\x86\x1b\xf0\xf9!\t@"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
self.feature_properties['floatval'] = 3.14159
self.assertEqual(mapbox_vector_tile.encode([{
"name": self.layer_name,
"features": [{
"geometry": geometry,
"properties": self.feature_properties
}]
}]), expected_result)
def test_encoder(self):
expected_result = '\x1aG\n\x05water\x12\x18\x12\x06\x00\x00\x01\x01\x02\x02\x18\x03"\x0c\t\x00\x80@\x1a\x00\x01\x02\x00\x00\x02\x0f\x1a\x03foo\x1a\x03baz\x1a\x03uid"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
self.assertEqual(
mapbox_vector_tile.encode([{
"name":
self.layer_name,
"features": [{
"geometry": self.feature_geometry,
"properties": self.feature_properties
}]
}]), expected_result)
def test_encode_multipolygon(self):
geometry = 'MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 20)))'
expected_result = '\x1a9\n\x05water\x12\x0e\x18\x03"\n\tP\xb0?\x12\'\t2\x1e\x0f\x12\x1b\x18\x03"\x17\t(\xba?"\x13\n\x00((\n\x1e\x1d\x0f\t\x1d\x00\x12\x13\n\x00\x13\x0f(\x80 x\x02'
result = mapbox_vector_tile.encode([
dict(name='water',
features=[dict(geometry=geometry, properties={})])])
self.assertEqual(expected_result, result)
decoded = mapbox_vector_tile.decode(result)
features = decoded['water']
self.assertEqual(2, len(features))
def _convert_layer(self, layer_name, tile):
layer = tile.decoded_data[layer_name]
self._get_metadata("json")["vector_layers"].append({"id": layer_name})
converted_layer = {"name": layer_name, "features": []}
debug("current layer: {}", layer_name)
for k in layer.keys():
if k != "features":
converted_layer[k] = layer[k]
for f in layer["features"]:
geo_type = geo_types[f["type"]]
geom_string = geo_type.upper()
geometry = f["geometry"]
is_polygon = geom_string == "POLYGON"
is_multi_geometry = is_multi(geo_type, geometry)
all_geometries = []
if is_multi_geometry:
VtWriter.get_subarr(geometry, all_geometries)
else:
if all(VtWriter.is_coordinate_tuple(c) for c in geometry):
all_geometries = [geometry]
else:
all_geometries = geometry
for geom in all_geometries:
new_feature = self._copy_feature(f)
new_feature["geometry"] = self._create_wkt_geometry(
geo_type, is_polygon, geom)
try:
single_feature_layer = {
"name": "dummy",
"features": [new_feature]
}
mapbox_vector_tile.encode(single_feature_layer,
y_coord_down=True)
converted_layer["features"].append(new_feature)
except:
debug("invalid geometry: {}", new_feature["geometry"])
pass
return converted_layer
def test_with_wkt(self):
geometry = "LINESTRING(-71.160281 42.258729,-71.160837 42.259113,-71.161144 42.25932)"
expected_result = '\x1aE\n\x05water\x12\x16\x12\x06\x00\x00\x01\x01\x02\x02\x18\x02"\n\t\x8d\x01\xaa?\x12\x00\x00\x00\x00\x1a\x03foo\x1a\x03baz\x1a\x03uid"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
self.assertEqual(
mapbox_vector_tile.encode([{
"name":
self.layer_name,
"features": [{
"geometry": geometry,
"properties": self.feature_properties
}]
}]), expected_result)
def test_encode_multipolygon(self):
geometry = 'MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), (30 20, 20 15, 20 25, 30 20)))'
expected_result = '\x1a9\n\x05water\x12\x0e\x18\x03"\n\tP\xb0?\x12\'\t2\x1e\x0f\x12\x1b\x18\x03"\x17\t(\xba?"\x13\n\x00((\n\x1e\x1d\x0f\t\x1d\x00\x12\x13\n\x00\x13\x0f(\x80 x\x02'
result = mapbox_vector_tile.encode([
dict(name='water',
features=[dict(geometry=geometry, properties={})])
])
self.assertEqual(expected_result, result)
decoded = mapbox_vector_tile.decode(result)
features = decoded['water']
self.assertEqual(2, len(features))
def test_too_small_linestring(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
shape = shapely.wkt.loads(
'LINESTRING(-71.160281 42.258729,-71.160837 42.259113,-71.161144 42.25932)') # noqa
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(0, len(features))
def test_encode_feature_with_id(self):
geometry = 'POINT(1 1)'
expected_result = '\x1a\x18\n\x05water\x12\n\x08*\x18\x01"\x04\t\x02\xfe?(\x80 x\x02'
result = mapbox_vector_tile.encode([
dict(name='water',
features=[dict(geometry=geometry, properties={}, id=42)])])
self.assertEqual(expected_result, result)
decoded = mapbox_vector_tile.decode(result)
features = decoded['water']
self.assertEqual(1, len(features))
feature = features[0]
self.assertEqual(42, feature['id'])
def test_too_small_geometry(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
shape = shapely.wkt.loads(
'LINESTRING (3065.656210384849 3629.831662879646, 3066.458953567231 3629.725941289478)') # noqa
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(0, len(features))
def test_invalid_geometry_ignore(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_ignore
import shapely.wkt
geometry = 'POLYGON ((10 10, 20 10, 20 20, 15 15, 15 5, 10 10))'
shape = shapely.wkt.loads(geometry)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source, on_invalid_geometry=on_invalid_geometry_ignore)
result = decode(pbf)
self.assertEqual(0, len(result['layername']['features']))
def test_invalid_geometry_ignore(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_ignore
import shapely.wkt
geometry = 'POLYGON ((10 10, 20 10, 20 20, 15 15, 15 5, 10 10))'
shape = shapely.wkt.loads(geometry)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source, on_invalid_geometry=on_invalid_geometry_ignore)
result = decode(pbf)
self.assertEqual(0, len(result['layername']['features']))
def list(self, request, aggregationlayer, x, y, z, frmt, *args, **kwargs):
# Select which agglayer to use for this tile.
lyr = get_object_or_404(AggregationLayer, pk=aggregationlayer)
# Compute tile boundary coorner coordinates.
bounds_coords = tile_bounds(int(x), int(y), int(z))
# Create a geometry with a 1% buffer around the tile. This buffered
# tile boundary will be used for clipping the geometry. The overflow
# will visually dissolve the polygons on the frontend visualization.
bounds = OGRGeometry.from_bbox(bounds_coords)
bounds.srid = WEB_MERCATOR_SRID
bounds = bounds.geos
bounds_buffer = bounds.buffer(
(bounds_coords[2] - bounds_coords[0]) / 100)
# Get the intersection of the aggregation areas and the tile boundary.
# use buffer to clip the aggregation area.
result = AggregationArea.objects.filter(
aggregationlayer=lyr,
geom__intersects=bounds,
).annotate(intersection=Intersection('geom', bounds_buffer)).only(
'id', 'name')
# Render intersection as vector tile in two different available formats.
if frmt == 'json':
result = [
'{{"geometry": {0}, "properties": {{"id": {1}, "name": "{2}"}}}}'
.format(dat.intersection.geojson, dat.id, dat.name)
for dat in result
]
result = ','.join(result)
result = '{"type": "FeatureCollection","features":[' + result + ']}'
return HttpResponse(result, content_type="application/json")
elif frmt == 'pbf':
features = [{
"geometry": bytes(dat.intersection.wkb),
"properties": {
"id": dat.id,
"name": dat.name,
"attributes": dat.attributes,
},
} for dat in result]
data = [
{
"name": lyr.name,
"features": features,
},
]
vtile = mapbox_vector_tile.encode(data,
quantize_bounds=bounds_coords)
return HttpResponse(vtile, content_type='application/x-protobuf')
def test_encode_feature_with_id(self):
geometry = 'POINT(1 1)'
expected_result = '\x1a\x18\n\x05water\x12\n\x08*\x18\x01"\x04\t\x02\xfe?(\x80 x\x02'
result = mapbox_vector_tile.encode([
dict(name='water',
features=[dict(geometry=geometry, properties={}, id=42)])
])
self.assertEqual(expected_result, result)
decoded = mapbox_vector_tile.decode(result)
features = decoded['water']
self.assertEqual(1, len(features))
feature = features[0]
self.assertEqual(42, feature['id'])
def merge(file, feature_layers, coord):
'''
Retrieve a list of mapbox vector tile responses and merge them into one.
get_tiles() retrieves data and performs basic integrity checks.
'''
layers = []
for layer in feature_layers:
layers.append(get_feature_layer(layer['name'], layer['features']))
data = mapbox_vector_tile.encode(layers)
file.write(data)
def test_quantize_makes_mutlipolygon_invalid(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
shape = shapely.wkt.loads('MULTIPOLYGON (((656510.8206577231 5674684.979891453, 656511.16 5674685.9, 656514.1758819892 5674684.979891453, 656510.8206577231 5674684.979891453)), ((657115.9120547654 5674684.979891453, 657118.85 5674690, 657118.0689111941 5674684.979891453, 657115.9120547654 5674684.979891453)))') # noqa
quantize_bounds = (645740.0149532147, 5674684.979891453, 665307.8941942193, 5694252.8591324575) # noqa
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
quantize_bounds=quantize_bounds,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(1, len(features))
def merge(file, feature_layers, bounds):
'''
Retrieve a list of protobuf vector tile responses and merge them into one.
get_tiles() retrieves data and performs basic integrity checks.
'''
layers = []
for layer in feature_layers:
layers.append(get_feature_layer(layer['name'], layer['features']))
data = mapbox_vector_tile.encode(layers)
file.write(data)
def test_encode_float_little_endian(self):
geometry = "LINESTRING(-71.160281 42.258729,-71.160837 42.259113,-71.161144 42.25932)"
expected_result = '\x1a\\\n\x05water\x12\x18\x12\x08\x00\x00\x01\x01\x02\x02\x03\x03\x18\x02"\n\t\x8d\x01\xaa?\x12\x00\x00\x00\x00\x1a\x08floatval\x1a\x03foo\x1a\x03baz\x1a\x03uid"\t\x19n\x86\x1b\xf0\xf9!\t@"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
self.feature_properties['floatval'] = 3.14159
self.assertEqual(
mapbox_vector_tile.encode([{
"name":
self.layer_name,
"features": [{
"geometry": geometry,
"properties": self.feature_properties
}]
}]), expected_result)
def test_quantize_makes_mutlipolygon_invalid(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
shape = shapely.wkt.loads('MULTIPOLYGON (((656510.8206577231 5674684.979891453, 656511.16 5674685.9, 656514.1758819892 5674684.979891453, 656510.8206577231 5674684.979891453)), ((657115.9120547654 5674684.979891453, 657118.85 5674690, 657118.0689111941 5674684.979891453, 657115.9120547654 5674684.979891453)))') # noqa
quantize_bounds = (645740.0149532147, 5674684.979891453, 665307.8941942193, 5694252.8591324575) # noqa
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
quantize_bounds=quantize_bounds,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(1, len(features))
def test_y_coord_down(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(10 10)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
pbf = encode(source, y_coord_down=True)
result = decode(pbf, y_coord_down=True)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = [[10, 10]]
self.assertEqual(exp_geom, act_geom)
def test_y_coord_down(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(10 10)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
pbf = encode(source, y_coord_down=True)
result = decode(pbf, y_coord_down=True)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = [[10, 10]]
self.assertEqual(exp_geom, act_geom)
def list(self, request, aggregationlayer, x, y, z, frmt, *args, **kwargs):
# Select which agglayer to use for this tile.
lyr = get_object_or_404(AggregationLayer, pk=aggregationlayer)
# Compute tile boundary coorner coordinates.
bounds_coords = tile_bounds(int(x), int(y), int(z))
# Create a geometry with a 1% buffer around the tile. This buffered
# tile boundary will be used for clipping the geometry. The overflow
# will visually dissolve the polygons on the frontend visualization.
bounds = OGRGeometry.from_bbox(bounds_coords)
bounds.srid = WEB_MERCATOR_SRID
bounds = bounds.geos
bounds_buffer = bounds.buffer((bounds_coords[2] - bounds_coords[0]) / 100)
# Get the intersection of the aggregation areas and the tile boundary.
# use buffer to clip the aggregation area.
result = AggregationArea.objects.filter(
aggregationlayer=lyr,
geom__intersects=bounds,
).annotate(
intersection=Intersection('geom', bounds_buffer)
).only('id', 'name')
# Render intersection as vector tile in two different available formats.
if frmt == 'json':
result = ['{{"geometry": {0}, "properties": {{"id": {1}, "name": "{2}"}}}}'.format(dat.intersection.geojson, dat.id, dat.name) for dat in result]
result = ','.join(result)
result = '{"type": "FeatureCollection","features":[' + result + ']}'
return HttpResponse(result, content_type="application/json")
elif frmt == 'pbf':
features = [
{
"geometry": bytes(dat.intersection.wkb),
"properties": {
"id": dat.id,
"name": dat.name,
"attributes": dat.attributes,
},
} for dat in result
]
data = [
{
"name": lyr.name,
"features": features,
},
]
vtile = mapbox_vector_tile.encode(data, quantize_bounds=bounds_coords)
return HttpResponse(vtile, content_type='application/x-protobuf')
def test_custom_extent(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(10 10)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
bounds = 0.0, 0.0, 10.0, 10.0
pbf = encode(source, quantize_bounds=bounds, extents=50)
result = decode(pbf)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = [[50, 50]]
self.assertEqual(exp_geom, act_geom)
def test_quantize(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(15 15)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
bounds = 10.0, 10.0, 20.0, 20.0
pbf = encode(source, quantize_bounds=bounds)
result = decode(pbf)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = {'type': 'Point', 'coordinates': [2048, 2048]}
self.assertEqual(exp_geom, act_geom)
def test_custom_extent(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(10 10)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
bounds = 0.0, 0.0, 10.0, 10.0
pbf = encode(source, quantize_bounds=bounds, extents=50)
result = decode(pbf)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = [[50, 50]]
self.assertEqual(exp_geom, act_geom)
def test_quantize(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(15 15)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
bounds = 10.0, 10.0, 20.0, 20.0
pbf = encode(source, quantize_bounds=bounds)
result = decode(pbf)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = {'type': 'Point', 'coordinates': [2048, 2048]}
self.assertEqual(exp_geom, act_geom)
def test_flipped_geometry_produces_multipolygon(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
shape = shapely.wkt.loads('POLYGON ((3449 1939, 3476 1967, 3473 1996, 3483 2027, 3542 2119, 3538 2160, 3563 2233, 3602 2255, 3639 2326, 3629 2388, 3573 2455, 3594 2493, 3558 2533, 3573 2549, 3518 2572, 3502 2592, 3505 2607, 3513 2614, 3535 2616, 3537 2610, 3535 2602, 3537 2599, 3548 2607, 3551 2636, 3528 2634, 3537 2668, 3549 2670, 3528 2711, 3550 2667, 3532 2635, 3550 2641, 3553 2613, 3549 2602, 3540 2596, 3512 2610, 3506 2589, 3576 2552, 3576 2543, 3563 2535, 3596 2506, 3597 2494, 3587 2469, 3589 2451, 3636 2385, 3644 2326, 3605 2251, 3566 2230, 3547 2122, 3482 2014, 3479 1966, 3455 1944, 3458 1910, 3449 1902, 3449 1939))') # noqa
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(1, len(features))
geom = features[0]['geometry']
for poly in geom:
p = shapely.geometry.Polygon(poly[0], poly[1:])
self.assertTrue(p.is_valid)
def test_custom_rounding_function(self):
from mapbox_vector_tile import decode
from mapbox_vector_tile import encode
props = dict(foo='bar')
shape = 'POINT(10 10)'
feature = dict(geometry=shape, properties=props)
features = [feature]
source = dict(name='layername', features=features)
bounds = 0.0, 0.0, 10.0, 10.0
# A really bad, custom "rounding" function
pbf = encode(source, quantize_bounds=bounds, round_fn=lambda x: 5)
result = decode(pbf)
act_feature = result['layername']['features'][0]
act_geom = act_feature['geometry']
exp_geom = [[5, 5]]
self.assertEqual(exp_geom, act_geom)
def test_flipped_geometry_produces_multipolygon(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.wkt
shape = shapely.wkt.loads('POLYGON ((3449 1939, 3476 1967, 3473 1996, 3483 2027, 3542 2119, 3538 2160, 3563 2233, 3602 2255, 3639 2326, 3629 2388, 3573 2455, 3594 2493, 3558 2533, 3573 2549, 3518 2572, 3502 2592, 3505 2607, 3513 2614, 3535 2616, 3537 2610, 3535 2602, 3537 2599, 3548 2607, 3551 2636, 3528 2634, 3537 2668, 3549 2670, 3528 2711, 3550 2667, 3532 2635, 3550 2641, 3553 2613, 3549 2602, 3540 2596, 3512 2610, 3506 2589, 3576 2552, 3576 2543, 3563 2535, 3596 2506, 3597 2494, 3587 2469, 3589 2451, 3636 2385, 3644 2326, 3605 2251, 3566 2230, 3547 2122, 3482 2014, 3479 1966, 3455 1944, 3458 1910, 3449 1902, 3449 1939))') # noqa
features = [dict(geometry=shape, properties={})]
pbf = encode({'name': 'foo', 'features': features},
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['foo']['features']
self.assertEqual(1, len(features))
geom = shapely.geometry.shape(features[0]['geometry'])
self.assertEqual(features[0]['geometry']['type'], 'MultiPolygon')
self.assertEqual(geom.geom_type, 'MultiPolygon')
self.assertTrue(geom.is_valid)
for poly in geom.geoms:
self.assertTrue(poly.is_valid)
def test_validate_generates_rounding_error(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.geometry
import shapely.wkt
bowtie = ('POLYGON((0 0, 1 1, 0 1, 1 0, 0 0))')
shape = shapely.wkt.loads(bowtie)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
features = result['layername']['features']
self.assertEqual(1, len(features))
shape = shapely.geometry.Polygon(features[0]['geometry'][0])
self.assertTrue(shape.is_valid)
self.assertGreater(shape.area, 0)
def run_test(layers):
print("Running perf test")
i = 0
profiler = cProfile.Profile()
for layer in layers:
layer_description = {
'features' : layer,
'name': 'bar'
}
profiler.enable()
res = encode(layer_description, on_invalid_geometry=on_invalid_geometry_ignore, round_fn=round)
profiler.disable()
if i % 100 == 0:
print("{} tiles produced".format(i))
i += 1
print ("Perf result :")
profiler.print_stats()
def test_encode_multiple_values_test(self):
geometry = 'POINT(0 0)'
properties1 = dict(foo='bar', baz='bar')
properties2 = dict(quux='morx', baz='bar')
name = 'foo'
feature1 = dict(geometry=geometry, properties=properties1)
feature2 = dict(geometry=geometry, properties=properties2)
source = [{
"name": name,
"features": [feature1, feature2]
}]
encoded = encode(source)
decoded = decode(encoded)
self.assertIn(name, decoded)
layer = decoded[name]
features = layer['features']
self.assertEqual(2, len(features))
self.assertEqual(features[0]['properties'], properties1)
self.assertEqual(features[1]['properties'], properties2)
def test_bowtie(self):
from mapbox_vector_tile import encode
from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid
import shapely.geometry
import shapely.wkt
bowtie = ('POLYGON ((0 0, 0 2, 1 1, 2 2, 2 0, 1 1, 0 0))')
shape = shapely.wkt.loads(bowtie)
self.assertFalse(shape.is_valid)
feature = dict(geometry=shape, properties={})
source = dict(name='layername', features=[feature])
pbf = encode(source,
on_invalid_geometry=on_invalid_geometry_make_valid)
result = decode(pbf)
self.assertEqual(1, len(result['layername']['features']))
valid_geometries = result['layername']['features'][0]['geometry']
self.assertEqual(2, len(valid_geometries))
shape1, shape2 = [shapely.geometry.Polygon(x[0])
for x in valid_geometries]
self.assertTrue(shape1.is_valid)
self.assertTrue(shape2.is_valid)
self.assertGreater(shape1.area, 0)
self.assertGreater(shape2.area, 0)