def test_query_min_zoom_fraction(self):
# test that fractional min zooms are included in their "floor" zoom
# tile. this is to allow over-zooming of a zoom N tile until N+1,
# where the next zoom tile kicks in.
from shapely.geometry import Point
from tilequeue.query.rawr import TilePyramid
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
def min_zoom_fn(shape, props, fid, meta):
return 11.999
shape = Point(0, 0)
tables = TestGetTable({'planet_osm_point': [(0, shape.wkb, {})]})
zoom = 10
max_zoom = zoom + 5
coord = mercator_point_to_coord(zoom, shape.x, shape.y)
tile_pyramid = TilePyramid(zoom, coord.column, coord.row, max_zoom)
fetcher = self._make(min_zoom_fn, None, tables, tile_pyramid)
# check that the fractional zoom of 11.999 means that it's included in
# the zoom 11 tile, but not the zoom 10 one.
feature_coord = mercator_point_to_coord(11, shape.x, shape.y)
for fetch, _ in fetcher.fetch_tiles(_wrap(coord)):
read_rows = fetch(11, coord_to_mercator_bounds(feature_coord))
self.assertEquals(1, len(read_rows))
feature_coord = feature_coord.zoomBy(-1).container()
for fetch, _ in fetcher.fetch_tiles(_wrap(coord)):
read_rows = fetch(10, coord_to_mercator_bounds(feature_coord))
self.assertEquals(0, len(read_rows))
def test_query_simple(self):
# just check that we can get back the mock data we put into a tile,
# and that the indexing/fetching code respects the tile boundary and
# min_zoom function.
from shapely.geometry import Point
from tilequeue.query.rawr import TilePyramid
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
feature_min_zoom = 11
def min_zoom_fn(shape, props, fid, meta):
return feature_min_zoom
shape = Point(0, 0)
# get_table(table_name) should return a generator of rows.
tables = TestGetTable({
'planet_osm_point': [(0, shape.wkb, {})],
})
zoom = 10
max_zoom = zoom + 5
coord = mercator_point_to_coord(zoom, shape.x, shape.y)
tile_pyramid = TilePyramid(zoom, coord.column, coord.row, max_zoom)
fetch = self._make(min_zoom_fn, None, tables, tile_pyramid)
# first, check that it can get the original item back when both the
# min zoom filter and geometry filter are okay.
feature_coord = mercator_point_to_coord(feature_min_zoom, shape.x,
shape.y)
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(feature_min_zoom,
coord_to_mercator_bounds(feature_coord))
self.assertEquals(1, len(read_rows))
read_row = read_rows[0]
self.assertEquals(0, read_row.get('__id__'))
# query processing code expects WKB bytes in the __geometry__ column
self.assertEquals(shape.wkb, read_row.get('__geometry__'))
self.assertEquals({'min_zoom': 11},
read_row.get('__testlayer_properties__'))
# now, check that if the min zoom or geometry filters would exclude
# the feature then it isn't returned.
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(zoom, coord_to_mercator_bounds(coord))
self.assertEquals(0, len(read_rows))
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(feature_min_zoom,
coord_to_mercator_bounds(feature_coord.left()))
self.assertEquals(0, len(read_rows))
def process_tiles(self):
unpadded_bounds = coord_to_mercator_bounds(self.coord)
all_formatted_tiles = []
all_extra_data = {}
for nominal_zoom, cut_coords in self.cut_coords_by_zoom.items():
def log_fn(data):
if self.log_fn:
self.log_fn(
dict(
coord=make_coord_dict(self.coord),
nominal_zoom=nominal_zoom,
msg=data,
))
feature_layers = self.feature_layers_by_zoom[nominal_zoom]
formatted_tiles, extra_data = process_coord(
self.coord,
nominal_zoom,
feature_layers,
self.post_process_data,
self.formats,
unpadded_bounds,
cut_coords,
self.buffer_cfg,
self.output_calc_mapping,
log_fn=log_fn,
)
all_formatted_tiles.extend(formatted_tiles)
all_extra_data.update(extra_data)
return all_formatted_tiles, all_extra_data
def _fetch(self, fetch, coord, metadata):
nominal_zoom = coord.zoom + self.metatile_zoom
start_zoom = coord.zoom + self.metatile_start_zoom
unpadded_bounds = coord_to_mercator_bounds(coord)
start = time.time()
source_rows = fetch(nominal_zoom, unpadded_bounds)
metadata['timing']['fetch'] = convert_seconds_to_millis(
time.time() - start)
# every tile job that we get from the queue is a "parent" tile
# and its four children to cut from it. at zoom 15, this may
# also include a whole bunch of other children below the max
# zoom.
cut_coords = list(
coord_children_subrange(coord, start_zoom, nominal_zoom))
return dict(
metadata=metadata,
coord=coord,
source_rows=source_rows,
unpadded_bounds=unpadded_bounds,
cut_coords=cut_coords,
nominal_zoom=nominal_zoom,
)
def _test(self, layer_name, props):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from shapely.geometry import box
def min_zoom_fn(shape, props, fid, meta):
return 0
tile = Coordinate(zoom=15, column=0, row=0)
bounds = coord_to_mercator_bounds(tile)
shape = box(*bounds)
rows = [
(1, shape, props),
]
label_placement_layers = {
'polygon': set([layer_name]),
}
fetch = self._make(rows,
min_zoom_fn,
None,
relations=[],
layer_name=layer_name,
label_placement_layers=label_placement_layers)
read_rows = fetch(16, bounds)
return read_rows
def process_tiles(self):
unpadded_bounds = coord_to_mercator_bounds(self.coord)
all_formatted_tiles = []
all_extra_data = {}
for nominal_zoom, cut_coords in self.cut_coords_by_zoom.items():
def log_fn(data):
if self.log_fn:
self.log_fn(dict(
coord=make_coord_dict(self.coord),
nominal_zoom=nominal_zoom,
msg=data,
))
feature_layers = self.feature_layers_by_zoom[nominal_zoom]
formatted_tiles, extra_data = process_coord(
self.coord, nominal_zoom, feature_layers,
self.post_process_data, self.formats, unpadded_bounds,
cut_coords, self.buffer_cfg, self.output_calc_mapping,
log_fn=log_fn,
)
all_formatted_tiles.extend(formatted_tiles)
all_extra_data.update(extra_data)
return all_formatted_tiles, all_extra_data
def _test(rels, expected_root_id):
shape = Point(0, 0)
props = {
'railway': 'station',
'name': 'Foo Station',
}
tables = TestGetTable({
'planet_osm_point': [(1, shape.wkb, props)],
'planet_osm_rels': rels,
})
zoom = 10
max_zoom = zoom + 6
coord = mercator_point_to_coord(zoom, shape.x, shape.y)
tile_pyramid = TilePyramid(zoom, coord.column, coord.row, max_zoom)
fetch = self._make(min_zoom_fn,
None,
tables,
tile_pyramid,
layer_name='pois')
feature_coord = mercator_point_to_coord(16, shape.x, shape.y)
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(16,
coord_to_mercator_bounds(feature_coord))
self.assertEquals(1, len(read_rows))
props = read_rows[0]['__pois_properties__']
self.assertEquals(expected_root_id,
props.get('mz_transit_root_relation_id'))
def reformat_selected_layers(
json_tile_data, layer_data, coord, format, buffer_cfg):
"""
Reformats the selected (subset of) layers from a JSON tile containing all
layers. We store "tiles of record" containing all layers as JSON, and this
function does most of the work of reading that, pruning the layers which
aren't needed and reformatting it to the desired output format.
"""
feature_layers = decode_json_tile_for_layers(json_tile_data, layer_data)
bounds_merc = coord_to_mercator_bounds(coord)
bounds_lnglat = (
mercator_point_to_lnglat(bounds_merc[0], bounds_merc[1]) +
mercator_point_to_lnglat(bounds_merc[2], bounds_merc[3]))
meters_per_pixel_dim = calc_meters_per_pixel_dim(coord.zoom)
scale = 4096
feature_layers = transform_feature_layers_shape(
feature_layers, format, scale, bounds_merc,
coord, meters_per_pixel_dim, buffer_cfg)
tile_data_file = StringIO()
format.format_tile(tile_data_file, feature_layers, coord,
bounds_merc, bounds_lnglat)
tile_data = tile_data_file.getvalue()
return tile_data
def fit_in_tile(z, x, y, shape):
"""
Fit shape into the tile. Shape should be a Shapely geometry or WKT string
with coordinates between 0 and 1. This unit square is then remapped into
the tile z/x/y.
"""
from ModestMaps.Core import Coordinate
from shapely.ops import transform
from shapely.wkt import loads as wkt_loads
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import reproject_mercator_to_lnglat
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
if isinstance(shape, (str, unicode)):
shape = wkt_loads(shape)
# check shape fits within unit square, so we can transform it to fit
# within the tile.
assert shape.bounds[0] >= 0
assert shape.bounds[1] >= 0
assert shape.bounds[2] <= 1
assert shape.bounds[3] <= 1
def _transform(x, y, *unused_coords):
return (
x * (bounds[2] - bounds[0]) + bounds[0],
y * (bounds[3] - bounds[1]) + bounds[1],
)
merc_shape = transform(_transform, shape)
return transform(reproject_mercator_to_lnglat, merc_shape)
def reformat_selected_layers(json_tile_data, layer_data, coord, format):
"""
Reformats the selected (subset of) layers from a JSON tile containing all
layers. We store "tiles of record" containing all layers as JSON, and this
function does most of the work of reading that, pruning the layers which
aren't needed and reformatting it to the desired output format.
"""
feature_layers = decode_json_tile_for_layers(json_tile_data, layer_data)
bounds_merc = coord_to_mercator_bounds(coord)
bounds_wgs84 = (
mercator_point_to_wgs84(bounds_merc[:2]) +
mercator_point_to_wgs84(bounds_merc[2:4]))
padded_bounds_merc = pad_bounds_for_zoom(bounds_merc, coord.zoom)
scale = 4096
feature_layers = transform_feature_layers_shape(
feature_layers, format, scale, bounds_merc,
padded_bounds_merc, coord)
tile_data_file = StringIO()
format.format_tile(tile_data_file, feature_layers, coord,
bounds_merc, bounds_wgs84)
tile_data = tile_data_file.getvalue()
return tile_data
def _make_json_tiles(
self, coord, post_process_data={}, db_features=[], cut_coords=[],
buffer_cfg={}):
from tilequeue.process import process_coord
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.format import json_format
unpadded_bounds = coord_to_mercator_bounds(coord)
feature_layers = [dict(
layer_datum=dict(
name='fake_layer',
geometry_types=['Point'],
transform_fn_names=[],
sort_fn_name=None,
is_clipped=False
),
padded_bounds=dict(point=unpadded_bounds),
features=db_features
)]
formats = [json_format]
tiles, extra = process_coord(
coord, coord.zoom, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, buffer_cfg)
return tiles
def reformat_selected_layers(
json_tile_data, layer_data, coord, format, buffer_cfg):
"""
Reformats the selected (subset of) layers from a JSON tile containing all
layers. We store "tiles of record" containing all layers as JSON, and this
function does most of the work of reading that, pruning the layers which
aren't needed and reformatting it to the desired output format.
"""
feature_layers = decode_json_tile_for_layers(json_tile_data, layer_data)
bounds_merc = coord_to_mercator_bounds(coord)
bounds_lnglat = (
mercator_point_to_lnglat(bounds_merc[0], bounds_merc[1]) +
mercator_point_to_lnglat(bounds_merc[2], bounds_merc[3]))
meters_per_pixel_dim = calc_meters_per_pixel_dim(coord.zoom)
scale = 4096
feature_layers = transform_feature_layers_shape(
feature_layers, format, scale, bounds_merc,
coord, meters_per_pixel_dim, buffer_cfg)
tile_data_file = StringIO()
format.format_tile(tile_data_file, feature_layers, coord,
bounds_merc, bounds_lnglat)
tile_data = tile_data_file.getvalue()
return tile_data
def _test(self, layer_name, props):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from shapely.geometry import box
def min_zoom_fn(shape, props, fid, meta):
return 0
tile = Coordinate(zoom=15, column=0, row=0)
bounds = coord_to_mercator_bounds(tile)
shape = box(*bounds)
rows = [
(1, shape, props),
]
label_placement_layers = {
'polygon': set([layer_name]),
}
fetch = self._make(
rows, min_zoom_fn, None, relations=[], layer_name=layer_name,
label_placement_layers=label_placement_layers)
read_rows = fetch(16, bounds)
return read_rows
def _make_json_tiles(
self, coord, post_process_data={}, db_features=[], cut_coords=[],
buffer_cfg={}):
from tilequeue.process import process_coord
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.format import json_format
unpadded_bounds = coord_to_mercator_bounds(coord)
feature_layers = [dict(
layer_datum=dict(
name='fake_layer',
geometry_types=['Point'],
transform_fn_names=[],
sort_fn_name=None,
is_clipped=False
),
padded_bounds=dict(point=unpadded_bounds),
features=db_features
)]
formats = [json_format]
def _test_output_fn(*args):
return dict(foo='bar', min_zoom=0)
output_calc_mapping = dict(fake_layer=_test_output_fn)
all_coords = [coord] + cut_coords
tiles, extra = process_coord(
coord, coord.zoom, feature_layers, post_process_data, formats,
unpadded_bounds, all_coords, buffer_cfg, output_calc_mapping)
return tiles
def process_coord(coord,
feature_layers,
post_process_data,
formats,
unpadded_bounds,
cut_coords,
layers_to_format,
buffer_cfg,
scale=4096):
feature_layers, extra_data = _preprocess_data(feature_layers)
children_formatted_tiles = []
if cut_coords:
for cut_coord in cut_coords:
unpadded_cut_bounds = coord_to_mercator_bounds(cut_coord)
meters_per_pixel_dim = calc_meters_per_pixel_dim(cut_coord.zoom)
child_feature_layers = _cut_coord(feature_layers,
unpadded_cut_bounds,
meters_per_pixel_dim, buffer_cfg)
child_formatted_tiles = _process_feature_layers(
child_feature_layers, cut_coord, post_process_data, formats,
unpadded_cut_bounds, scale, layers_to_format, buffer_cfg)
children_formatted_tiles.extend(child_formatted_tiles)
coord_formatted_tiles = _process_feature_layers(feature_layers, coord,
post_process_data, formats,
unpadded_bounds, scale,
layers_to_format,
buffer_cfg)
all_formatted_tiles = coord_formatted_tiles + children_formatted_tiles
return all_formatted_tiles, extra_data
def fit_in_tile(z, x, y, shape):
"""
Fit shape into the tile. Shape should be a Shapely geometry or WKT string
with coordinates between 0 and 1. This unit square is then remapped into
the tile z/x/y.
"""
from ModestMaps.Core import Coordinate
from shapely.ops import transform
from shapely.wkt import loads as wkt_loads
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import reproject_mercator_to_lnglat
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
if isinstance(shape, (str, unicode)):
shape = wkt_loads(shape)
# check shape fits within unit square, so we can transform it to fit
# within the tile.
assert shape.bounds[0] >= 0
assert shape.bounds[1] >= 0
assert shape.bounds[2] <= 1
assert shape.bounds[3] <= 1
def _transform(x, y, *unused_coords):
return (
x * (bounds[2] - bounds[0]) + bounds[0],
y * (bounds[3] - bounds[1]) + bounds[1],
)
merc_shape = transform(_transform, shape)
return transform(reproject_mercator_to_lnglat, merc_shape)
def bounds(self):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
coord = Coordinate(zoom=self.z, column=self.x, row=self.y)
bounds = coord_to_mercator_bounds(coord)
return bounds
def bounds(self):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
coord = Coordinate(zoom=self.z, column=self.x, row=self.y)
bounds = coord_to_mercator_bounds(coord)
return bounds
def _fetch_data(self, shape_fn, source_table):
from tilequeue.query.common import LayerInfo
from tilequeue.query.rawr import make_rawr_data_fetcher
from tilequeue.tile import coord_to_mercator_bounds
from ModestMaps.Core import Coordinate
top_zoom = 10
max_zoom = top_zoom + 6
def min_zoom_fn(shape, props, fid, meta):
return top_zoom
def props_fn(shape, props, fid, meta):
return {'kind': 'country'}
z, x, y = (max_zoom, 0, 0)
tile = Coordinate(zoom=z, column=x, row=y)
top_tile = tile.zoomTo(top_zoom).container()
bounds = coord_to_mercator_bounds(tile)
shape = shape_fn(bounds)
props = {
'name': 'Foo',
'admin_level': '2',
'boundary': 'administrative',
}
source = 'test'
tables = TestGetTable({
source_table: [
(1, shape.wkb, props),
],
}, source)
layers = {
'boundaries': LayerInfo(min_zoom_fn, props_fn),
}
storage = ConstantStorage(tables)
fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers,
[dict(type="osm")])
for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)):
read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile))
return read_rows
def _cut_child_tiles(feature_layers, cut_coord, formats, scale, buffer_cfg):
unpadded_cut_bounds = coord_to_mercator_bounds(cut_coord)
meters_per_pixel_dim = calc_meters_per_pixel_dim(cut_coord.zoom)
cut_feature_layers = _cut_coord(feature_layers, unpadded_cut_bounds,
meters_per_pixel_dim, buffer_cfg)
return _format_feature_layers(cut_feature_layers, cut_coord, formats,
unpadded_cut_bounds, scale, buffer_cfg)
def test_grass(self):
# we'll generate a patchwork of completely contiguous shapes with
# varying areas. this is to make sure that different min zooms are
# calculated for all of them.
from shapely.ops import transform
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import reproject_mercator_to_lnglat
from ModestMaps.Core import Coordinate
from shapely.geometry import box
import dsl
z, x, y = (10, 528, 332)
props = {
'source': 'openstreetmap.org',
'landuse': 'grass',
}
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
# define tile-internal coordinate system. note that we're squashing
# the features into a smaller square, since we want fairly small
# features and don't want to fill the tile with them.
ox = 0.5 * (bounds[2] + bounds[0])
w = 0.1 * (bounds[2] - bounds[0])
oy = 0.5 * (bounds[3] + bounds[1])
h = 0.1 * (bounds[3] - bounds[1])
way_id = 1
ways = []
splits = 8
swidth = 2 ** splits
for sx in xrange(splits):
for sy in xrange(splits):
minx = ox + w * (0.5 * float(1 << sx) / swidth)
maxx = ox + w * (float(1 << sx) / swidth)
miny = oy + h * (0.5 * float(1 << sy) / swidth)
maxy = oy + h * (float(1 << sy) / swidth)
merc_shape = box(minx, miny, maxx, maxy)
latlng_shape = transform(
reproject_mercator_to_lnglat, merc_shape)
ways.append(dsl.way(way_id, latlng_shape, props))
way_id += 1
self.generate_fixtures(*ways)
with self.features_in_tile_layer(z, x, y, 'landuse') as features:
# have to use assertTrue here rather than the more natural
# assertEqual so that when this is run as --download-only the test
# case class can skip this test.
self.assertTrue(len(features) == 1)
def test_grass(self):
# we'll generate a patchwork of completely contiguous shapes with
# varying areas. this is to make sure that different min zooms are
# calculated for all of them.
from shapely.ops import transform
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import reproject_mercator_to_lnglat
from ModestMaps.Core import Coordinate
from shapely.geometry import box
import dsl
z, x, y = (10, 528, 332)
props = {
'source': 'openstreetmap.org',
'landuse': 'grass',
}
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
# define tile-internal coordinate system. note that we're squashing
# the features into a smaller square, since we want fairly small
# features and don't want to fill the tile with them.
ox = 0.5 * (bounds[2] + bounds[0])
w = 0.1 * (bounds[2] - bounds[0])
oy = 0.5 * (bounds[3] + bounds[1])
h = 0.1 * (bounds[3] - bounds[1])
way_id = 1
ways = []
splits = 8
swidth = 2**splits
for sx in xrange(splits):
for sy in xrange(splits):
minx = ox + w * (0.5 * float(1 << sx) / swidth)
maxx = ox + w * (float(1 << sx) / swidth)
miny = oy + h * (0.5 * float(1 << sy) / swidth)
maxy = oy + h * (float(1 << sy) / swidth)
merc_shape = box(minx, miny, maxx, maxy)
latlng_shape = transform(reproject_mercator_to_lnglat,
merc_shape)
ways.append(dsl.way(way_id, latlng_shape, props))
way_id += 1
self.generate_fixtures(*ways)
with self.features_in_tile_layer(z, x, y, 'landuse') as features:
# have to use assertTrue here rather than the more natural
# assertEqual so that when this is run as --download-only the test
# case class can skip this test.
self.assertTrue(len(features) == 1)
def _cut_child_tiles(
feature_layers, cut_coord, nominal_zoom, formats, scale, buffer_cfg):
unpadded_cut_bounds = coord_to_mercator_bounds(cut_coord)
meters_per_pixel_dim = calc_meters_per_pixel_dim(nominal_zoom)
cut_feature_layers = _cut_coord(
feature_layers, unpadded_cut_bounds, meters_per_pixel_dim, buffer_cfg)
return _format_feature_layers(
cut_feature_layers, cut_coord, nominal_zoom, formats,
unpadded_cut_bounds, scale, buffer_cfg)
def test_compare_z10_bounds(self):
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('10/100/100')
merc_bounds = coord_to_mercator_bounds(coord)
tile_meters_wide = merc_bounds[2] - merc_bounds[0]
exp_meters_per_pixel_dim = tile_meters_wide / 256
act_meters_per_pixel_dim = self._call_fut(10)
self.assertAlmostEquals(
exp_meters_per_pixel_dim, act_meters_per_pixel_dim, places=0)
def test_compare_z10_bounds(self):
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('10/100/100')
merc_bounds = coord_to_mercator_bounds(coord)
tile_meters_wide = merc_bounds[2] - merc_bounds[0]
exp_meters_per_pixel_dim = tile_meters_wide / 256
act_meters_per_pixel_dim = self._call_fut(10)
self.assertAlmostEquals(exp_meters_per_pixel_dim,
act_meters_per_pixel_dim,
places=0)
def _test(self, input_layer_names, expected_layer_names):
from shapely.geometry import Point
from tilequeue.query.common import LayerInfo
from tilequeue.query.fixture import make_fixture_data_fetcher
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
def min_zoom_fn(shape, props, fid, meta):
return 0
def props_fn(shape, props, fid, meta):
return {}
shape = Point(0, 0)
props = {'name': 'Foo', 'name:en': 'Bar'}
rows = [
(1, shape, props),
]
layers = {}
for name in input_layer_names:
layers[name] = LayerInfo(min_zoom_fn, props_fn)
fetch = make_fixture_data_fetcher(layers, rows)
feature_coord = mercator_point_to_coord(16, shape.x, shape.y)
read_rows = fetch(16, coord_to_mercator_bounds(feature_coord))
self.assertEqual(1, len(read_rows))
all_layer_names = set(expected_layer_names) | set(input_layer_names)
for layer_name in all_layer_names:
properties_name = '__%s_properties__' % layer_name
self.assertTrue(properties_name in read_rows[0])
for key in props.keys():
actual_name = read_rows[0][properties_name].get(key)
if layer_name in expected_layer_names:
expected_name = props.get(key)
self.assertEquals(
expected_name,
actual_name,
msg=('expected=%r, actual=%r for key=%r' %
(expected_name, actual_name, key)))
else:
# check the name doesn't appear anywhere else
self.assertEquals(
None,
actual_name,
msg=('got actual=%r for key=%r, expected no value' %
(actual_name, key)))
def fetch(self):
unpadded_bounds = coord_to_mercator_bounds(self.coord)
cut_coords_by_zoom = calculate_cut_coords_by_zoom(
self.coord, self.metatile_zoom, self.cfg_tile_sizes, self.max_zoom)
feature_layers_by_zoom = {}
for nominal_zoom, _ in cut_coords_by_zoom.items():
source_rows = self.fetch_fn(nominal_zoom, unpadded_bounds)
feature_layers = convert_source_data_to_feature_layers(
source_rows, self.layer_data, unpadded_bounds, self.coord.zoom)
feature_layers_by_zoom[nominal_zoom] = feature_layers
self.cut_coords_by_zoom = cut_coords_by_zoom
self.feature_layers_by_zoom = feature_layers_by_zoom
def test_query_past_max_zoom(self):
# check that features with a min_zoom beyond the maximum zoom are still
# included at the maximum zoom. since this is the last zoom level we
# generate, it must include everything.
from shapely.geometry import Point
from tilequeue.query.rawr import TilePyramid
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
def min_zoom_fn(shape, props, fid, meta):
return 20
shape = Point(0, 0)
tables = TestGetTable({'planet_osm_point': [(0, shape.wkb, {})]})
zoom = 10
max_zoom = zoom + 6
coord = mercator_point_to_coord(zoom, shape.x, shape.y)
tile_pyramid = TilePyramid(zoom, coord.column, coord.row, max_zoom)
fetch = self._make(min_zoom_fn, None, tables, tile_pyramid)
# the min_zoom of 20 should mean that the feature is included at zoom
# 16, even though 16<20, because 16 is the "max zoom" at which all the
# data is included.
feature_coord = mercator_point_to_coord(16, shape.x, shape.y)
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(16, coord_to_mercator_bounds(feature_coord))
self.assertEquals(1, len(read_rows))
# but it should not exist at zoom 15
feature_coord = feature_coord.zoomBy(-1).container()
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(10, coord_to_mercator_bounds(feature_coord))
self.assertEquals(0, len(read_rows))
def fetch(self):
unpadded_bounds = coord_to_mercator_bounds(self.coord)
cut_coords_by_zoom = calculate_cut_coords_by_zoom(
self.coord, self.metatile_zoom, self.cfg_tile_sizes, self.max_zoom)
feature_layers_by_zoom = {}
for nominal_zoom, _ in cut_coords_by_zoom.items():
source_rows = self.fetch_fn(nominal_zoom, unpadded_bounds)
feature_layers = convert_source_data_to_feature_layers(
source_rows, self.layer_data, unpadded_bounds, self.coord.zoom)
feature_layers_by_zoom[nominal_zoom] = feature_layers
self.cut_coords_by_zoom = cut_coords_by_zoom
self.feature_layers_by_zoom = feature_layers_by_zoom
def test_query_source(self):
# check that the source is added to the result properties, and that
# it overrides any existing source.
from shapely.geometry import Point
from tilequeue.query.rawr import TilePyramid
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
feature_min_zoom = 11
def min_zoom_fn(shape, props, fid, meta):
return feature_min_zoom
shape = Point(0, 0)
# get_table(table_name) should return a generator of rows.
tables = TestGetTable(
{
'planet_osm_point': [(0, shape.wkb, {
'source': 'originalrowsource'
})],
},
source='testingquerysource')
zoom = 10
max_zoom = zoom + 5
coord = mercator_point_to_coord(zoom, shape.x, shape.y)
tile_pyramid = TilePyramid(zoom, coord.column, coord.row, max_zoom)
fetch = self._make(min_zoom_fn, None, tables, tile_pyramid)
# first, check that it can get the original item back when both the
# min zoom filter and geometry filter are okay.
feature_coord = mercator_point_to_coord(feature_min_zoom, shape.x,
shape.y)
for fetcher, _ in fetch.fetch_tiles(_wrap(coord)):
read_rows = fetcher(feature_min_zoom,
coord_to_mercator_bounds(feature_coord))
self.assertEquals(1, len(read_rows))
read_row = read_rows[0]
self.assertEquals(0, read_row.get('__id__'))
# query processing code expects WKB bytes in the __geometry__ column
self.assertEquals(shape.wkb, read_row.get('__geometry__'))
self.assertEquals({'min_zoom': 11},
read_row.get('__testlayer_properties__'))
self.assertEquals({'source': 'testingquerysource'},
read_row.get('__properties__'))
def _test(self, input_layer_names, expected_layer_names):
from shapely.geometry import Point
from tilequeue.query.common import LayerInfo
from tilequeue.query.fixture import make_fixture_data_fetcher
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
def min_zoom_fn(shape, props, fid, meta):
return 0
def props_fn(shape, props, fid, meta):
return {}
shape = Point(0, 0)
props = {'name': 'Foo', 'name:en': 'Bar'}
rows = [
(1, shape, props),
]
layers = {}
for name in input_layer_names:
layers[name] = LayerInfo(min_zoom_fn, props_fn)
fetch = make_fixture_data_fetcher(layers, rows)
feature_coord = mercator_point_to_coord(16, shape.x, shape.y)
read_rows = fetch(16, coord_to_mercator_bounds(feature_coord))
self.assertEqual(1, len(read_rows))
all_layer_names = set(expected_layer_names) | set(input_layer_names)
for layer_name in all_layer_names:
properties_name = '__%s_properties__' % layer_name
self.assertTrue(properties_name in read_rows[0])
for key in props.keys():
actual_name = read_rows[0][properties_name].get(key)
if layer_name in expected_layer_names:
expected_name = props.get(key)
self.assertEquals(
expected_name, actual_name,
msg=('expected=%r, actual=%r for key=%r'
% (expected_name, actual_name, key)))
else:
# check the name doesn't appear anywhere else
self.assertEquals(
None, actual_name,
msg=('got actual=%r for key=%r, expected no value'
% (actual_name, key)))
def test_normal_layer(self):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from shapely import wkb
tile = Coordinate(zoom=15, column=10, row=10)
bounds = coord_to_mercator_bounds(tile)
read_row = self._test('testlayer', bounds, 1.0)
clipped_shape = wkb.loads(read_row['__geometry__'])
# for normal layers, clipped shape is inside the bounds of the tile.
x_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
y_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
self.assertAlmostEqual(1.0, x_factor)
self.assertAlmostEqual(1.0, y_factor)
def box_area(z, x, y, area, include_boundary=False):
"""
Returns a Shapely Polygon which is in the z/x/y tile and has the given
area in Mercator square meters.
If include_boundary is truthy, set up the shape such that the boundary
is part of the tile. Try to keep the centroid within the tile. If that
isn't possible, throw an exception.
"""
from ModestMaps.Core import Coordinate
from math import sqrt
from shapely.geometry import box
from shapely.ops import transform
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import half_earth_circum
from tilequeue.tile import reproject_mercator_to_lnglat
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
if include_boundary:
# make the shape 90% of the tile's height, and whatever width is
# necessary for that without wrapping around the world.
size_y = 0.9 * (bounds[3] - bounds[1])
size_x = area / size_y
else:
# otherwise, a square shape is easiest to reason about.
size_y = size_x = sqrt(area)
# make a shape with the given size
centre_x = 0.5 * (bounds[0] + bounds[2])
centre_y = 0.5 * (bounds[1] + bounds[3])
def _check(coord):
assert -half_earth_circum <= coord <= half_earth_circum
return coord
mercator_shape = box(
_check(centre_x - 0.5 * size_x),
_check(centre_y - 0.5 * size_y),
_check(centre_x + 0.5 * size_x),
_check(centre_y + 0.5 * size_y),
)
return transform(reproject_mercator_to_lnglat, mercator_shape)
def box_area(z, x, y, area, include_boundary=False):
"""
Returns a Shapely Polygon which is in the z/x/y tile and has the given
area in Mercator square meters.
If include_boundary is truthy, set up the shape such that the boundary
is part of the tile. Try to keep the centroid within the tile. If that
isn't possible, throw an exception.
"""
from ModestMaps.Core import Coordinate
from math import sqrt
from shapely.geometry import box
from shapely.ops import transform
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import half_earth_circum
from tilequeue.tile import reproject_mercator_to_lnglat
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
if include_boundary:
# make the shape 90% of the tile's height, and whatever width is
# necessary for that without wrapping around the world.
size_y = 0.9 * (bounds[3] - bounds[1])
size_x = area / size_y
else:
# otherwise, a square shape is easiest to reason about.
size_y = size_x = sqrt(area)
# make a shape with the given size
centre_x = 0.5 * (bounds[0] + bounds[2])
centre_y = 0.5 * (bounds[1] + bounds[3])
def _check(coord):
assert -half_earth_circum <= coord <= half_earth_circum
return coord
mercator_shape = box(
_check(centre_x - 0.5 * size_x),
_check(centre_y - 0.5 * size_y),
_check(centre_x + 0.5 * size_x),
_check(centre_y + 0.5 * size_y),
)
return transform(reproject_mercator_to_lnglat, mercator_shape)
def __init__(self, z, x, y, landuse_kind=None):
from tilequeue.tile import coord_to_mercator_bounds
from ModestMaps.Core import Coordinate
import dsl
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
self.origin_x = bounds[0]
self.origin_y = bounds[1]
self.buildings = []
self.way_id = 1
if landuse_kind is not None:
self.buildings.append(dsl.way(self.way_id, dsl.tile_box(z, x, y), {
'landuse': landuse_kind,
'source': 'openstreetmap.org',
}))
self.way_id += 1
def test_normal_layer(self):
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from shapely import wkb
tile = Coordinate(zoom=15, column=10, row=10)
bounds = coord_to_mercator_bounds(tile)
read_row = self._test('testlayer', bounds, 1.0)
clipped_shape = wkb.loads(read_row['__geometry__'])
# for normal layers, clipped shape is inside the bounds of the tile.
x_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
y_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
self.assertAlmostEqual(1.0, x_factor)
self.assertAlmostEqual(1.0, y_factor)
def __init__(self, z, x, y, landuse_kind=None):
from tilequeue.tile import coord_to_mercator_bounds
from ModestMaps.Core import Coordinate
import dsl
bounds = coord_to_mercator_bounds(Coordinate(zoom=z, column=x, row=y))
self.origin_x = bounds[0]
self.origin_y = bounds[1]
self.buildings = []
self.way_id = 1
if landuse_kind is not None:
self.buildings.append(
dsl.way(self.way_id, dsl.tile_box(z, x, y), {
'landuse': landuse_kind,
'source': 'openstreetmap.org',
}))
self.way_id += 1
def test_process_coord_empty(self):
from tilequeue.process import process_coord
from tilequeue.tile import coord_to_mercator_bounds
coord = Coordinate(0, 0, 0)
feature_layers = []
post_process_data = {}
formats = []
unpadded_bounds = coord_to_mercator_bounds(coord)
cut_coords = []
buffer_cfg = {}
tiles, extra = process_coord(
coord, coord.zoom, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, buffer_cfg)
self.assertEqual([], tiles)
self.assertEqual({'size': {}}, extra)
def test_water_layer(self):
# water layer should be expanded by 10% on each side.
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from shapely import wkb
tile = Coordinate(zoom=15, column=10, row=10)
bounds = coord_to_mercator_bounds(tile)
read_row = self._test('water', bounds, 1.0)
clipped_shape = wkb.loads(read_row['__geometry__'])
# for water layer, the geometry should be 10% larger than the tile
# bounds.
x_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
y_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
self.assertAlmostEqual(1.1, x_factor)
self.assertAlmostEqual(1.1, y_factor)
def _test(self, layer_name, tile, factor):
from shapely.geometry import box
from tilequeue.query.rawr import TilePyramid
from tilequeue.tile import coord_to_mercator_bounds
top_zoom = 10
max_zoom = top_zoom + 6
def min_zoom_fn(shape, props, fid, meta):
return top_zoom
top_tile = tile.zoomTo(top_zoom).container()
tile_pyramid = TilePyramid(top_zoom, top_tile.column, top_tile.row,
max_zoom)
bounds = coord_to_mercator_bounds(tile)
boxwidth = bounds[2] - bounds[0]
boxheight = bounds[3] - bounds[1]
# make shape overlap the edges of the bounds. that way we can check to
# see if the shape gets clipped.
shape = box(bounds[0] - factor * boxwidth,
bounds[1] - factor * boxheight,
bounds[2] + factor * boxwidth,
bounds[3] + factor * boxheight)
props = {'name': 'Foo'}
tables = TestGetTable({
'planet_osm_polygon': [
(1, shape.wkb, props),
],
})
fetch = self._make(min_zoom_fn,
None,
tables,
tile_pyramid,
layer_name=layer_name)
for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)):
read_rows = fetcher(tile.zoom, bounds)
self.assertEqual(1, len(read_rows))
return read_rows[0]
def _make_tiles(self, shape, coord, metatile_zoom):
from tilequeue.format import mvt_format
from tilequeue.process import process_coord
from tilequeue.tile import coord_children_range
from tilequeue.tile import coord_to_mercator_bounds
db_features = [dict(
__id__=1,
__geometry__=shape.wkb,
__properties__={},
)]
nominal_zoom = coord.zoom + metatile_zoom
unpadded_bounds = coord_to_mercator_bounds(coord)
feature_layers = [dict(
layer_datum=dict(
name='fake_layer',
geometry_types=[shape.geom_type],
transform_fn_names=[],
sort_fn_name=None,
is_clipped=False
),
padded_bounds={shape.geom_type.lower(): unpadded_bounds},
features=db_features
)]
formats = [mvt_format]
post_process_data = {}
buffer_cfg = {}
cut_coords = [coord]
if nominal_zoom > coord.zoom:
cut_coords.extend(coord_children_range(coord, nominal_zoom))
def _output_fn(shape, props, fid, meta):
return dict(fake='data', min_zoom=0)
output_calc_mapping = dict(fake_layer=_output_fn)
tiles, extra = process_coord(
coord, nominal_zoom, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, buffer_cfg, output_calc_mapping)
self.assertEqual(len(cut_coords), len(tiles))
return tiles, cut_coords
def test_high_line(self):
from ModestMaps.Core import Coordinate
from shapely.geometry import box
from shapely.geometry import shape
from tilequeue.tile import coord_to_mercator_bounds
# this is mid way along the High Line in NYC, which is a huge long
# "building". we should be clipping it to the bounds of the tile.
#
# NOTE: we _don't_ clip the fixture, that has to happen in the
# query.
#
# NOTE: https://github.com/tilezen/vector-datasource/issues/1142
# we want to clip all buildings to the bounding box of the tile, so
# that there are no overlaps.
self.generate_fixtures(dsl.way(-7141751, wkt_loads('POLYGON ((-74.00832281293381 40.7394398124915, -74.00830439747038 40.73966530823048, -74.0082918210565 40.73982512183188, -74.00828544301798 40.73989665667138, -74.00824780360749 40.74035717312061, -74.0081553669648 40.74069231492279, -74.00811530210309 40.74088125683539, -74.0080881729816 40.74100921431009, -74.0080344537276 40.7412720027987, -74.00796914620641 40.74157065885181, -74.00796609193439 40.7415827057976, -74.00790518615818 40.7418352147232, -74.00789764030981 40.7418808840249, -74.0078936877226 40.74190436524828, -74.00789207075501 40.74191437028878, -74.007885602885 40.74195357369829, -74.007860450057 40.74210528250789, -74.00779999343838 40.7424276881022, -74.00749717135611 40.74284265453919, -74.007488817024 40.74285388452819, -74.00746671846798 40.74288471594268, -74.00752034789051 40.74290778844879, -74.0074379723789 40.74302110904868, -74.0073881158806 40.74299286399228, -74.00735317141609 40.74304166327489, -74.00734670354599 40.74303737547328, -74.0071343418129 40.74333268865769, -74.00714152833518 40.7433354110594, -74.00720117647001 40.74335909594929, -74.0071084703327 40.7434796981892, -74.0072566923546 40.7437145724654, -74.00722453266739 40.74375826674808, -74.00704909169241 40.7435598046351, -74.007006511548 40.74347636325429, -74.0069896232206 40.74356552165899, -74.0069869282748 40.7438065890523, -74.00709068369009 40.74385001103649, -74.00699887586801 40.7439780989187, -74.00706741732419 40.74432778710959, -74.00708035306428 40.74433329989039, -74.00711341106678 40.744347388106, -74.007046127252 40.74443368686849, -74.007101373642 40.74445587407167, -74.00708762941809 40.74447459026498, -74.0070408271918 40.74453808916498, -74.0070927498152 40.7445568053351, -74.00703382033259 40.74463806748181, -74.00705088832299 40.7446452817044, -74.00700031317248 40.74471701553578, -74.0070119014397 40.74472218799129, -74.0069620449414 40.7447906549294, -74.00697138742039 40.74479501067649, -74.0068542471073 40.74495338272707, -74.0068584691891 40.7449572620549, -74.00670602508541 40.74516769823619, -74.0065572640744 40.74538031160611, -74.0064573714148 40.7455893852941, -74.00637858916438 40.74570011513548, -74.00529737688839 40.74719049227489, -74.00533169253218 40.74720430766089, -74.0051680194875 40.74743569793549, -74.00471814319322 40.7480852878009, -74.0046617289933 40.74806351013981, -74.0045451276695 40.74821847165479, -74.0042063729758 40.7486820609962, -74.00387318783689 40.74914911786458, -74.0037918903037 40.74930155885421, -74.0029797234554 40.75042022003169, -74.0024423512524 40.75115756792688, -74.00222927086701 40.75145250479549, -74.00216468199808 40.75154546339898, -74.00211725095109 40.75161909523129, -74.00202679060199 40.75180405912329, -74.0020020869317 40.75189048431209, -74.0019877138871 40.75201991773139, -74.00199759535521 40.75216098762209, -74.0020531112398 40.75232410568999, -74.00212012556001 40.7524433987087, -74.0023955490261 40.75275990185269, -74.00240471184199 40.75277038161278, -74.00259560383988 40.75289178338889, -74.00570575101661 40.7542046641078, -74.00567574728599 40.75424855564479, -74.00574958880239 40.7543362705831, -74.00584202544509 40.75444487635099, -74.00588595306249 40.754521090819, -74.0059412892841 40.7546320099334, -74.00597362863429 40.7547336062359, -74.00598063549349 40.7548837888313, -74.00595278771969 40.75507779402898, -74.00586277652819 40.75525798495069, -74.0051998198485 40.75615648254439, -74.0049826970444 40.75629046679348, -74.0047818337468 40.75638049262529, -74.004583126406 40.75643601870109, -74.0043705850098 40.7564614000904, -74.00415813344509 40.75645445933609, -74.00391792393809 40.75641975555381, -74.0036937942747 40.75633660249569, -74.0035551842264 40.75626270370049, -74.00342133524899 40.75614716011579, -74.00323655179511 40.75604100682211, -74.0032781437928 40.7559694213001, -74.00344442195181 40.75607326111, -74.00358303200019 40.75617961849289, -74.0037192165972 40.75625807651728, -74.00387399632069 40.75632966172837, -74.00404952712719 40.75637130632149, -74.0042944078737 40.7564013149091, -74.0045462056478 40.75638049262529, -74.0047724912679 40.75632278900709, -74.00491568272419 40.75625120378858, -74.0051167256847 40.75611490586368, -74.00575650583011 40.75523716230868, -74.0058257659385 40.7551008623049, -74.00586511214789 40.7549114844519, -74.0058535238808 40.75474517445868, -74.00580267923569 40.75459036425079, -74.00572883771929 40.75445637657539, -74.0056178957818 40.7543247703379, -74.00550937929539 40.7542461739342, -74.0054216138922 40.75418846846359, -74.00530842616639 40.7541330766085, -74.00510280179779 40.75403842039558, -74.0024419020948 40.75292267820008, -74.002192889098 40.75281686000951, -74.00141404974669 40.75248606649619, -74.00118758446349 40.75237527990329, -74.00101214348858 40.75224128776399, -74.0011183243551 40.75214901064349, -74.00120168801348 40.75223618394428, -74.00128118891618 40.75228171000207, -74.00163764042088 40.75243958786818, -74.00213333079469 40.75264966012008, -74.0020663164745 40.75254806063321, -74.00206191472959 40.752541187521, -74.00196157291228 40.7523855555718, -74.0019134232131 40.75228232245999, -74.0018975230326 40.75222339036788, -74.0018773109387 40.7521055940779, -74.00187533464511 40.7519693557054, -74.00189949932619 40.75184686342378, -74.0018782990855 40.7518331170536, -74.00193462345381 40.75175057072168, -74.0019308505296 40.7517460793273, -74.0020133158727 40.75158295984141, -74.00235772995261 40.75111578396379, -74.00276080402058 40.75055911563168, -74.00280293500738 40.75050031811769, -74.00320879385281 40.74992418139269, -74.0032147227337 40.7498771565471, -74.003217148185 40.74985850992759, -74.00400919277099 40.7487908801634, -74.00460522496201 40.74797197582978, -74.00628336774419 40.74566179868388, -74.0063335835686 40.74558707133109, -74.0063781400067 40.7455128202991, -74.0064454238215 40.74536928621749, -74.00646042568668 40.74533940876668, -74.00645476630049 40.745336754505, -74.0066254462044 40.74511311556489, -74.00678471750429 40.74488757023619, -74.0068654760483 40.7447548561216, -74.0069019476489 40.7446670604849, -74.00692377671029 40.74458089814269, -74.0069330293577 40.74452508993101, -74.00693788026018 40.74449548433628, -74.0069397667223 40.74444797923952, -74.00694201251051 40.74438060089219, -74.0069160511988 40.74437638123579, -74.0069313225587 40.74427116197648, -74.00690634939379 40.74400049045201, -74.00676773934539 40.74393699103869, -74.0068508335092 40.7438227872252, -74.006828016301 40.7437319957343, -74.0067652240626 40.74370980828958, -74.00683502316021 40.74362167097459, -74.00684301816629 40.74344927541001, -74.0068445453022 40.7434198054563, -74.0069314123902 40.74324441472271, -74.0069853113072 40.7432670106833, -74.0070602308019 40.74316417177648, -74.0071325451823 40.7430648718473, -74.0071720710548 40.74301035551058, -74.00724510408739 40.74291010250508, -74.00734311028489 40.74277568292929, -74.0075530465668 40.74249166538279, -74.00761943206631 40.74235336548188, -74.0076729716572 40.7421333918186, -74.0077141144973 40.7417787916529, -74.00771788742139 40.7417611636875, -74.00776837274039 40.74152716731758, -74.00777636774639 40.74149027776248, -74.00778535089928 40.7414489641588, -74.00784490920259 40.74118386225317, -74.0079128218381 40.7408822097112, -74.0079423764109 40.74075071273261, -74.0079467781558 40.74073077036279, -74.0079894481318 40.74054148770819, -74.00804208940748 40.74043360792479, -74.00809320354709 40.74026106797047, -74.00813228026199 40.7398473105725, -74.0081335379034 40.73983077111318, -74.00813488537628 40.7397962628454, -74.00815689410079 40.73947901735949, -74.00815994837281 40.73943511607349, -74.00818644867358 40.739436205098, -74.00832281293381 40.7394398124915))'), {u'website': u'http://www.thehighline.org/', u'building': u'yes', u'layer': u'1', u'way_area': u'48210.6', u'name': u'The High Line', u'building:colour': u'#3D4647', u'building:part': u'yes', u'wheelchair': u'yes', u'name:he': u'\u05e8\u05db\u05d1\u05ea \u05de\u05e2\u05d2\u05dc\u05d9\u05ea \u05e0\u05d7\u05de\u05d3\u05d4', u'wikipedia': u'en:High Line (New York City)', u'leisure': u'park', u'height': u'7.5', u'min_height': u'5;5.5', u'roof:shape': u'flat', u'wikidata': u'Q843869', u'source': u'openstreetmap.org', u'old_railway_operator': u'NYC', u'building:material': u'metal', u'roof:material': u'concrete;grass'}),dsl.way(-7141751, wkt_loads('POLYGON ((-74.0076264389255 40.74321957276918, -74.0075904164826 40.74326918860679, -74.00753804470159 40.74324237291861, -74.00753391245129 40.74323760870891, -74.00745557935851 40.7431462719373, -74.0073948532453 40.74307290296429, -74.00735317141609 40.74304166327489, -74.0073881158806 40.74299286399228, -74.0074379723789 40.74302110904868, -74.00751720378699 40.74311666572878, -74.00757927737311 40.74318949017198, -74.0076264389255 40.74321957276918))'), {u'website': u'http://www.thehighline.org/', u'building': u'yes', u'layer': u'1', u'way_area': u'355.099', u'name': u'The High Line', u'building:colour': u'#3D4647', u'building:part': u'yes', u'wheelchair': u'yes', u'name:he': u'\u05e8\u05db\u05d1\u05ea \u05de\u05e2\u05d2\u05dc\u05d9\u05ea \u05e0\u05d7\u05de\u05d3\u05d4', u'wikipedia': u'en:High Line (New York City)', u'leisure': u'park', u'height': u'7.5', u'min_height': u'5;5.5', u'roof:shape': u'flat', u'wikidata': u'Q843869', u'source': u'openstreetmap.org', u'old_railway_operator': u'NYC', u'building:material': u'metal', u'roof:material': u'concrete;grass'})) # noqa
coord = Coordinate(zoom=16, column=19295, row=24631)
with self.features_in_tile_layer(
coord.zoom, coord.column, coord.row, 'buildings') as buildings:
# tile bounds as a box
tile_bounds = coord_to_mercator_bounds(coord)
bbox = box(*tile_bounds)
# need to check that we at least saw the high line
saw_the_high_line = False
for building in buildings:
building_bounds = shape(building['geometry']).bounds
building_box = box(*building_bounds)
if building['properties']['id'] == -7141751:
saw_the_high_line = True
self.assertTrue(
building_box.within(bbox),
'feature %r extends outside of the bounds of the '
'tile (%r not within %r).' %
(building['properties']['id'], building_bounds,
tile_bounds))
self.assertTrue(
saw_the_high_line,
"Expected to see the High Line in this tile, but didn't.")
def test_water_layer(self):
# water layer should be expanded by 10% on each side.
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from shapely import wkb
tile = Coordinate(zoom=15, column=10, row=10)
bounds = coord_to_mercator_bounds(tile)
read_row = self._test('water', bounds, 1.0)
clipped_shape = wkb.loads(read_row['__geometry__'])
# for water layer, the geometry should be 10% larger than the tile
# bounds.
x_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
y_factor = ((clipped_shape.bounds[2] - clipped_shape.bounds[0]) /
(bounds[2] - bounds[0]))
self.assertAlmostEqual(1.1, x_factor)
self.assertAlmostEqual(1.1, y_factor)
def _check_metatile(self, metatile_size):
from mock import patch
from shapely.geometry import box
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import metatile_zoom_from_size
name = 'tilequeue.format.mvt.mvt_encode'
with patch(name, return_value='') as encode:
coord = Coordinate(0, 0, 0)
bounds = coord_to_mercator_bounds(coord)
pixel_fraction = 1.0 / 4096.0
box_width = pixel_fraction * (bounds[2] - bounds[0])
box_height = pixel_fraction * (bounds[3] - bounds[1])
shape = box(bounds[0], bounds[1],
bounds[0] + box_width,
bounds[1] + box_height)
metatile_zoom = metatile_zoom_from_size(metatile_size)
tiles, tile_coords = self._make_tiles(shape, coord, metatile_zoom)
num_tiles = 0
for z in range(0, metatile_zoom + 1):
num_tiles += 4**z
# resolution should be 4096 at 256px, which is metatile_zoom
# levels down from the extent of the world.
resolution = (bounds[2] - bounds[0]) / (4096 * 2**metatile_zoom)
self.assertEqual(num_tiles, len(tiles))
self.assertEqual(num_tiles, encode.call_count)
for (posargs, kwargs), coord in zip(encode.call_args_list,
tile_coords):
self.assertIn('quantize_bounds', kwargs)
quantize_bounds = kwargs['quantize_bounds']
extent = int(round((quantize_bounds[2] - quantize_bounds[0]) /
resolution))
self.assertIn('extents', kwargs)
actual_extent = kwargs['extents']
self.assertEquals(extent, actual_extent,
"Expected %r, not %r, for coord %r" %
(extent, actual_extent, coord))
def test_cut_coord_exclusive(self):
# test that cut coords are the only ones in the response, and that
# the coordinate itself can be omitted.
from tilequeue.process import process_coord
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.format import json_format
coord = Coordinate(0, 0, 0)
db_features = []
cut_coords = [
Coordinate(zoom=1, column=0, row=0),
Coordinate(zoom=1, column=1, row=0),
Coordinate(zoom=1, column=0, row=1),
]
buffer_cfg = {}
post_process_data = {}
unpadded_bounds = coord_to_mercator_bounds(coord)
feature_layers = [dict(
layer_datum=dict(
name='fake_layer',
geometry_types=['Point'],
transform_fn_names=[],
sort_fn_name=None,
is_clipped=False
),
padded_bounds=dict(point=unpadded_bounds),
features=db_features
)]
formats = [json_format]
def _test_output_fn(*args):
return dict(foo='bar', min_zoom=0)
output_calc_mapping = dict(fake_layer=_test_output_fn)
tiles, extra = process_coord(
coord, coord.zoom, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, buffer_cfg, output_calc_mapping)
self.assertEqual(len(cut_coords), len(tiles))
self.assertNotIn(coord, [t['coord'] for t in tiles])
def test_process_coord_empty(self):
from tilequeue.process import process_coord
from tilequeue.tile import coord_to_mercator_bounds
coord = Coordinate(0, 0, 0)
feature_layers = []
post_process_data = {}
formats = []
unpadded_bounds = coord_to_mercator_bounds(coord)
cut_coords = [coord]
buffer_cfg = {}
def _test_output_fn(*args):
return dict(foo='bar')
output_calc_mapping = dict(fake_layer=_test_output_fn)
tiles, extra = process_coord(
coord, coord.zoom, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, buffer_cfg, output_calc_mapping)
self.assertEqual([], tiles)
self.assertEqual({'size': {}}, extra)
def process_coord(coord, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, layers_to_format,
buffer_cfg, scale=4096):
feature_layers, extra_data = _preprocess_data(feature_layers)
children_formatted_tiles = []
if cut_coords:
for cut_coord in cut_coords:
unpadded_cut_bounds = coord_to_mercator_bounds(cut_coord)
meters_per_pixel_dim = calc_meters_per_pixel_dim(cut_coord.zoom)
child_feature_layers = _cut_coord(
feature_layers, unpadded_cut_bounds, meters_per_pixel_dim,
buffer_cfg)
child_formatted_tiles = _process_feature_layers(
child_feature_layers, cut_coord, post_process_data, formats,
unpadded_cut_bounds, scale, layers_to_format, buffer_cfg)
children_formatted_tiles.extend(child_formatted_tiles)
coord_formatted_tiles = _process_feature_layers(
feature_layers, coord, post_process_data, formats, unpadded_bounds,
scale, layers_to_format, buffer_cfg)
all_formatted_tiles = coord_formatted_tiles + children_formatted_tiles
return all_formatted_tiles, extra_data
def handle_request(self, request):
if (self.health_checker and
self.health_checker.is_health_check(request)):
return self.health_checker(request)
request_data = parse_request_path(request.path)
if request_data is None:
return self.generate_404()
layer_spec = request_data.layer_spec
layer_data = parse_layer_spec(request_data.layer_spec,
self.layer_config)
if layer_data is None:
return self.generate_404()
coord = request_data.coord
format = request_data.format
if self.store and layer_spec != 'all' and coord.zoom <= 20:
# we have a dynamic layer request
# in this case, we should try to fetch the data from the
# cache, and if present, prune the layers that aren't
# necessary from there.
tile_data = self.store.read_tile(coord, json_format)
if tile_data is not None:
# we were able to fetch the cached data
# we'll need to decode it into the expected
# feature_layers shape, prune the layers that aren't
# needed, and then format the data
feature_layers = decode_json_tile_for_layers(
tile_data, layer_data)
bounds_merc = coord_to_mercator_bounds(coord)
bounds_wgs84 = (
mercator_point_to_wgs84(bounds_merc[:2]) +
mercator_point_to_wgs84(bounds_merc[2:4]))
tile_data_file = StringIO()
format.format_tile(tile_data_file, feature_layers, coord,
bounds_merc, bounds_wgs84)
tile_data = tile_data_file.getvalue()
response = self.create_response(request, tile_data, format)
return response
feature_data = self.data_fetcher(coord, layer_data)
formatted_tiles = process_coord(
coord,
feature_data['feature_layers'],
self.post_process_data,
[format],
feature_data['unpadded_bounds'],
feature_data['padded_bounds'],
[])
assert len(formatted_tiles) == 1, \
'unexpected number of tiles: %d' % len(formatted_tiles)
formatted_tile = formatted_tiles[0]
tile_data = formatted_tile['tile']
# we only want to store requests for the all layer
if self.store and layer_spec == 'all' and coord.zoom <= 20:
self.io_pool.apply_async(
async_store, (self.store, tile_data, coord, format))
# update the tiles of interest set with the new coordinate
if self.redis_cache_index:
self.io_pool.apply_async(async_update_tiles_of_interest,
(self.redis_cache_index, coord))
response = self.create_response(request, tile_data, format)
return response
def __call__(self, coord, layer_data=None):
if layer_data is None:
layer_data = self.layer_data
zoom = coord.zoom
unpadded_bounds = coord_to_mercator_bounds(coord)
sql_conns, conn_info = self.sql_conn_pool.get_conns()
try:
# the padded bounds are used here in order to only have to
# issue a single set of queries to the database for all
# formats
empty_results, async_results = enqueue_queries(
sql_conns, self.io_pool, layer_data, zoom, unpadded_bounds)
feature_layers = []
async_exception = None
for async_result in async_results:
try:
rows, layer_datum, padded_bounds = async_result.get()
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
async_exception = exc_value
# iterate through all async results to give others
# a chance to close any connections that yielded
# exceptions
continue
# don't continue processing if an error occurred on
# any results
if async_exception is not None:
continue
# read the bytes out of each row, otherwise the pickle
# will fail because the geometry is a read buffer
# only keep values that are not None
read_rows = []
for row in rows:
read_row = {}
for k, v in row.items():
if isinstance(v, buffer):
v = bytes(v)
if v is not None:
read_row[k] = v
read_rows.append(read_row)
feature_layer = dict(
name=layer_datum['name'], features=read_rows,
layer_datum=layer_datum,
padded_bounds=padded_bounds,
)
feature_layers.append(feature_layer)
# bail if an error occurred
if async_exception is not None:
raise async_exception
feature_layers.extend(empty_results)
return dict(
feature_layers=feature_layers,
unpadded_bounds=unpadded_bounds,
padded_bounds=padded_bounds,
)
finally:
self.sql_conn_pool.put_conns(sql_conns)
