def _fetch_data(self, shape_fn, source_table):
from tilequeue.query.common import LayerInfo
from tilequeue.query.rawr import make_rawr_data_fetcher
from tilequeue.tile import coord_to_mercator_bounds
from ModestMaps.Core import Coordinate
top_zoom = 10
max_zoom = top_zoom + 6
def min_zoom_fn(shape, props, fid, meta):
return top_zoom
def props_fn(shape, props, fid, meta):
return {'kind': 'country'}
z, x, y = (max_zoom, 0, 0)
tile = Coordinate(zoom=z, column=x, row=y)
top_tile = tile.zoomTo(top_zoom).container()
bounds = coord_to_mercator_bounds(tile)
shape = shape_fn(bounds)
props = {
'name': 'Foo',
'admin_level': '2',
'boundary': 'administrative',
}
source = 'test'
tables = TestGetTable({
source_table: [
(1, shape.wkb, props),
],
}, source)
layers = {
'boundaries': LayerInfo(min_zoom_fn, props_fn),
}
storage = ConstantStorage(tables)
fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers,
[dict(type="osm")])
for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)):
read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile))
return read_rows
def _make(self,
min_zoom_fn,
props_fn,
tables,
tile_pyramid,
layer_name='testlayer',
label_placement_layers={},
min_z=10,
max_z=16):
from tilequeue.query.common import LayerInfo
from tilequeue.query.rawr import make_rawr_data_fetcher
layers = {layer_name: LayerInfo(min_zoom_fn, props_fn)}
storage = ConstantStorage(tables)
indexes_cfg = [dict(type="osm")]
return make_rawr_data_fetcher(
min_z,
max_z,
storage,
layers,
indexes_cfg,
label_placement_layers=label_placement_layers)
def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool):
rawr_yaml = cfg.yml.get('rawr')
assert rawr_yaml is not None, 'Missing rawr configuration in yaml'
group_by_zoom = rawr_yaml.get('group-zoom')
assert group_by_zoom is not None, 'Missing group-zoom rawr config'
rawr_source_yaml = rawr_yaml.get('source')
assert rawr_source_yaml, 'Missing rawr source config'
table_sources = rawr_source_yaml.get('table-sources')
assert table_sources, 'Missing definitions of source per table'
# map text for table source onto Source objects
for tbl, data in table_sources.items():
source_name = data['name']
source_value = data['value']
table_sources[tbl] = Source(source_name, source_value)
label_placement_layers = rawr_yaml.get('label-placement-layers', {})
for geom_type, layers in label_placement_layers.items():
assert geom_type in ('point', 'polygon', 'linestring'), \
'Geom type %r not understood, expecting point, polygon or ' \
'linestring.' % (geom_type,)
label_placement_layers[geom_type] = set(layers)
indexes_cfg = rawr_yaml.get('indexes')
assert indexes_cfg, 'Missing definitions of table indexes.'
# source types are:
# s3 - to fetch RAWR tiles from S3
# store - to fetch RAWR tiles from any tilequeue tile source
# generate - to generate RAWR tiles directly, rather than trying to load
# them from S3. this can be useful for standalone use and
# testing. provide a postgresql subkey for database connection
# settings.
source_type = rawr_source_yaml.get('type')
if source_type == 's3':
rawr_source_s3_yaml = rawr_source_yaml.get('s3')
bucket = rawr_source_s3_yaml.get('bucket')
assert bucket, 'Missing rawr source s3 bucket'
region = rawr_source_s3_yaml.get('region')
assert region, 'Missing rawr source s3 region'
prefix = rawr_source_s3_yaml.get('prefix')
assert prefix, 'Missing rawr source s3 prefix'
suffix = rawr_source_s3_yaml.get('suffix')
assert suffix, 'Missing rawr source s3 suffix'
allow_missing_tiles = rawr_source_s3_yaml.get('allow-missing-tiles',
False)
import boto3
from tilequeue.rawr import RawrS3Source
s3_client = boto3.client('s3', region_name=region)
storage = RawrS3Source(s3_client, bucket, prefix, suffix,
table_sources, allow_missing_tiles)
elif source_type == 'generate':
from raw_tiles.source.conn import ConnectionContextManager
from raw_tiles.source.osm import OsmSource
postgresql_cfg = rawr_source_yaml.get('postgresql')
assert postgresql_cfg, 'Missing rawr postgresql config'
conn_ctx = ConnectionContextManager(postgresql_cfg)
rawr_osm_source = OsmSource(conn_ctx)
storage = _NullRawrStorage(rawr_osm_source, table_sources)
elif source_type == 'store':
from tilequeue.store import make_store
from tilequeue.rawr import RawrStoreSource
store_cfg = rawr_source_yaml.get('store')
store = make_store(store_cfg,
credentials=cfg.subtree('aws credentials'))
storage = RawrStoreSource(store, table_sources)
else:
assert False, 'Source type %r not understood. ' \
'Options are s3, generate and store.' % (source_type,)
# TODO: this needs to be configurable, everywhere! this is a long term
# refactor - it's hard-coded in a bunch of places :-(
max_z = 16
layers = _make_layer_info(layer_data, cfg.process_yaml_cfg)
return make_rawr_data_fetcher(group_by_zoom, max_z, storage, layers,
indexes_cfg, label_placement_layers)
def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool):
rawr_yaml = cfg.yml.get('rawr')
assert rawr_yaml is not None, 'Missing rawr configuration in yaml'
group_by_zoom = rawr_yaml.get('group-zoom')
assert group_by_zoom is not None, 'Missing group-zoom rawr config'
rawr_source_yaml = rawr_yaml.get('source')
assert rawr_source_yaml, 'Missing rawr source config'
table_sources = rawr_source_yaml.get('table-sources')
assert table_sources, 'Missing definitions of source per table'
# map text for table source onto Source objects
for tbl, data in table_sources.items():
source_name = data['name']
source_value = data['value']
table_sources[tbl] = Source(source_name, source_value)
label_placement_layers = rawr_yaml.get('label-placement-layers', {})
for geom_type, layers in label_placement_layers.items():
assert geom_type in ('point', 'polygon', 'linestring'), \
'Geom type %r not understood, expecting point, polygon or ' \
'linestring.' % (geom_type,)
label_placement_layers[geom_type] = set(layers)
indexes_cfg = rawr_yaml.get('indexes')
assert indexes_cfg, 'Missing definitions of table indexes.'
# source types are:
# s3 - to fetch RAWR tiles from S3
# store - to fetch RAWR tiles from any tilequeue tile source
# generate - to generate RAWR tiles directly, rather than trying to load
# them from S3. this can be useful for standalone use and
# testing. provide a postgresql subkey for database connection
# settings.
source_type = rawr_source_yaml.get('type')
if source_type == 's3':
rawr_source_s3_yaml = rawr_source_yaml.get('s3')
bucket = rawr_source_s3_yaml.get('bucket')
assert bucket, 'Missing rawr source s3 bucket'
region = rawr_source_s3_yaml.get('region')
assert region, 'Missing rawr source s3 region'
prefix = rawr_source_s3_yaml.get('prefix')
assert prefix, 'Missing rawr source s3 prefix'
extension = rawr_source_s3_yaml.get('extension')
assert extension, 'Missing rawr source s3 extension'
allow_missing_tiles = rawr_source_s3_yaml.get(
'allow-missing-tiles', False)
import boto3
from tilequeue.rawr import RawrS3Source
s3_client = boto3.client('s3', region_name=region)
tile_key_gen = make_s3_tile_key_generator(rawr_source_s3_yaml)
storage = RawrS3Source(
s3_client, bucket, prefix, extension, table_sources, tile_key_gen,
allow_missing_tiles)
elif source_type == 'generate':
from raw_tiles.source.conn import ConnectionContextManager
from raw_tiles.source.osm import OsmSource
postgresql_cfg = rawr_source_yaml.get('postgresql')
assert postgresql_cfg, 'Missing rawr postgresql config'
conn_ctx = ConnectionContextManager(postgresql_cfg)
rawr_osm_source = OsmSource(conn_ctx)
storage = _NullRawrStorage(rawr_osm_source, table_sources)
elif source_type == 'store':
from tilequeue.store import make_store
from tilequeue.rawr import RawrStoreSource
store_cfg = rawr_source_yaml.get('store')
store = make_store(store_cfg,
credentials=cfg.subtree('aws credentials'))
storage = RawrStoreSource(store, table_sources)
else:
assert False, 'Source type %r not understood. ' \
'Options are s3, generate and store.' % (source_type,)
# TODO: this needs to be configurable, everywhere! this is a long term
# refactor - it's hard-coded in a bunch of places :-(
max_z = 16
layers = _make_layer_info(layer_data, cfg.process_yaml_cfg)
return make_rawr_data_fetcher(
group_by_zoom, max_z, storage, layers, indexes_cfg,
label_placement_layers)
def _test(self, input_layer_names, expected_layer_names):
from shapely.geometry import Point
from tilequeue.query.common import LayerInfo
from tilequeue.query.rawr import make_rawr_data_fetcher
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import mercator_point_to_coord
top_zoom = 10
max_zoom = top_zoom + 6
def min_zoom_fn(shape, props, fid, meta):
return top_zoom
def props_fn(shape, props, fid, meta):
return {}
shape = Point(0, 0)
props = {'name': 'Foo', 'name:en': 'Bar'}
source = 'test'
tables = TestGetTable({
'planet_osm_point': [
(1, shape.wkb, props),
],
}, source)
tile = mercator_point_to_coord(max_zoom, shape.x, shape.y)
top_tile = tile.zoomTo(top_zoom).container()
layers = {}
for name in input_layer_names:
layers[name] = LayerInfo(min_zoom_fn, props_fn)
storage = ConstantStorage(tables)
fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers,
[dict(type="osm")])
for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)):
read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile))
# the RAWR query goes over features multiple times because of the
# indexing, so we can't rely on all the properties for one feature to
# be all together in the same place. this loops over all the features,
# checking that there's only really one of them and gathering together
# all the __%s_properties__ from all the rows for further testing.
all_props = {}
for row in read_rows:
self.assertEquals(1, row['__id__'])
self.assertEquals(shape.wkb, row['__geometry__'])
for key, val in row.items():
if key.endswith('_properties__'):
self.assertFalse(key in all_props)
all_props[key] = val
all_layer_names = set(expected_layer_names) | set(input_layer_names)
for layer_name in all_layer_names:
properties_name = '__%s_properties__' % layer_name
self.assertTrue(properties_name in all_props)
for key in props.keys():
actual_name = all_props[properties_name].get(key)
if layer_name in expected_layer_names:
expected_name = props.get(key)
self.assertEquals(
expected_name,
actual_name,
msg=('expected=%r, actual=%r for key=%r' %
(expected_name, actual_name, key)))
else:
# check the name doesn't appear anywhere else
self.assertEquals(
None,
actual_name,
msg=('got actual=%r for key=%r, expected no value' %
(actual_name, key)))
def test_buffered_land(self):
# the buffered land is a polygon in the boundaries table (sort of),
# but we shouldn't turn it into linestrings of the exterior ring. it
# should stay as a polygon otherwise the border features won't be
# correctly assigned maritime boundary properties, and they'll all
# appear to be maritime boundaries. :-(
from tilequeue.query.common import LayerInfo
from tilequeue.query.rawr import make_rawr_data_fetcher
from tilequeue.tile import coord_to_mercator_bounds
from ModestMaps.Core import Coordinate
from shapely.geometry import Polygon
import shapely.wkb
top_zoom = 10
max_zoom = top_zoom + 6
def min_zoom_fn(shape, props, fid, meta):
return 8
def props_fn(shape, props, fid, meta):
props['kind'] = 'maritime'
return props
z, x, y = (max_zoom, 0, 0)
tile = Coordinate(zoom=z, column=x, row=y)
top_tile = tile.zoomTo(top_zoom).container()
bounds = coord_to_mercator_bounds(tile)
shape = Polygon([
[bounds[0], bounds[1]],
[bounds[0], bounds[3]],
[bounds[2], bounds[1]],
[bounds[0], bounds[1]],
])
props = {
'kind': 'maritime',
'maritime_boundary': True,
'source': 'tilezen.org',
'min_zoom': 0,
}
source = 'tilezen.org'
tables = TestGetTable({
'buffered_land': [
(1, shape.wkb, props),
],
}, source)
layers = {
'boundaries': LayerInfo(min_zoom_fn, props_fn),
}
storage = ConstantStorage(tables)
index_cfg = [{
'type': 'simple',
'table': 'buffered_land',
'layer': 'boundaries',
}]
fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers,
index_cfg)
for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)):
read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile))
self.assertEqual(len(read_rows), 1)
props = read_rows[0]['__boundaries_properties__']
self.assertEqual(props.get('kind'), 'maritime')
# check no special boundaries geometry - should just be the polygon
self.assertIsNone(read_rows[0].get('__boundaries_geometry__'))
shape = shapely.wkb.loads(read_rows[0]['__geometry__'])
self.assertEqual(shape.geom_type, 'Polygon')