def _fetch_data(self, shape_fn, source_table): from tilequeue.query.common import LayerInfo from tilequeue.query.rawr import make_rawr_data_fetcher from tilequeue.tile import coord_to_mercator_bounds from ModestMaps.Core import Coordinate top_zoom = 10 max_zoom = top_zoom + 6 def min_zoom_fn(shape, props, fid, meta): return top_zoom def props_fn(shape, props, fid, meta): return {'kind': 'country'} z, x, y = (max_zoom, 0, 0) tile = Coordinate(zoom=z, column=x, row=y) top_tile = tile.zoomTo(top_zoom).container() bounds = coord_to_mercator_bounds(tile) shape = shape_fn(bounds) props = { 'name': 'Foo', 'admin_level': '2', 'boundary': 'administrative', } source = 'test' tables = TestGetTable({ source_table: [ (1, shape.wkb, props), ], }, source) layers = { 'boundaries': LayerInfo(min_zoom_fn, props_fn), } storage = ConstantStorage(tables) fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers, [dict(type="osm")]) for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)): read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile)) return read_rows
def _make(self, min_zoom_fn, props_fn, tables, tile_pyramid, layer_name='testlayer', label_placement_layers={}, min_z=10, max_z=16): from tilequeue.query.common import LayerInfo from tilequeue.query.rawr import make_rawr_data_fetcher layers = {layer_name: LayerInfo(min_zoom_fn, props_fn)} storage = ConstantStorage(tables) indexes_cfg = [dict(type="osm")] return make_rawr_data_fetcher( min_z, max_z, storage, layers, indexes_cfg, label_placement_layers=label_placement_layers)
def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool): rawr_yaml = cfg.yml.get('rawr') assert rawr_yaml is not None, 'Missing rawr configuration in yaml' group_by_zoom = rawr_yaml.get('group-zoom') assert group_by_zoom is not None, 'Missing group-zoom rawr config' rawr_source_yaml = rawr_yaml.get('source') assert rawr_source_yaml, 'Missing rawr source config' table_sources = rawr_source_yaml.get('table-sources') assert table_sources, 'Missing definitions of source per table' # map text for table source onto Source objects for tbl, data in table_sources.items(): source_name = data['name'] source_value = data['value'] table_sources[tbl] = Source(source_name, source_value) label_placement_layers = rawr_yaml.get('label-placement-layers', {}) for geom_type, layers in label_placement_layers.items(): assert geom_type in ('point', 'polygon', 'linestring'), \ 'Geom type %r not understood, expecting point, polygon or ' \ 'linestring.' % (geom_type,) label_placement_layers[geom_type] = set(layers) indexes_cfg = rawr_yaml.get('indexes') assert indexes_cfg, 'Missing definitions of table indexes.' # source types are: # s3 - to fetch RAWR tiles from S3 # store - to fetch RAWR tiles from any tilequeue tile source # generate - to generate RAWR tiles directly, rather than trying to load # them from S3. this can be useful for standalone use and # testing. provide a postgresql subkey for database connection # settings. source_type = rawr_source_yaml.get('type') if source_type == 's3': rawr_source_s3_yaml = rawr_source_yaml.get('s3') bucket = rawr_source_s3_yaml.get('bucket') assert bucket, 'Missing rawr source s3 bucket' region = rawr_source_s3_yaml.get('region') assert region, 'Missing rawr source s3 region' prefix = rawr_source_s3_yaml.get('prefix') assert prefix, 'Missing rawr source s3 prefix' suffix = rawr_source_s3_yaml.get('suffix') assert suffix, 'Missing rawr source s3 suffix' allow_missing_tiles = rawr_source_s3_yaml.get('allow-missing-tiles', False) import boto3 from tilequeue.rawr import RawrS3Source s3_client = boto3.client('s3', region_name=region) storage = RawrS3Source(s3_client, bucket, prefix, suffix, table_sources, allow_missing_tiles) elif source_type == 'generate': from raw_tiles.source.conn import ConnectionContextManager from raw_tiles.source.osm import OsmSource postgresql_cfg = rawr_source_yaml.get('postgresql') assert postgresql_cfg, 'Missing rawr postgresql config' conn_ctx = ConnectionContextManager(postgresql_cfg) rawr_osm_source = OsmSource(conn_ctx) storage = _NullRawrStorage(rawr_osm_source, table_sources) elif source_type == 'store': from tilequeue.store import make_store from tilequeue.rawr import RawrStoreSource store_cfg = rawr_source_yaml.get('store') store = make_store(store_cfg, credentials=cfg.subtree('aws credentials')) storage = RawrStoreSource(store, table_sources) else: assert False, 'Source type %r not understood. ' \ 'Options are s3, generate and store.' % (source_type,) # TODO: this needs to be configurable, everywhere! this is a long term # refactor - it's hard-coded in a bunch of places :-( max_z = 16 layers = _make_layer_info(layer_data, cfg.process_yaml_cfg) return make_rawr_data_fetcher(group_by_zoom, max_z, storage, layers, indexes_cfg, label_placement_layers)
def _make_rawr_fetcher(cfg, layer_data, query_cfg, io_pool): rawr_yaml = cfg.yml.get('rawr') assert rawr_yaml is not None, 'Missing rawr configuration in yaml' group_by_zoom = rawr_yaml.get('group-zoom') assert group_by_zoom is not None, 'Missing group-zoom rawr config' rawr_source_yaml = rawr_yaml.get('source') assert rawr_source_yaml, 'Missing rawr source config' table_sources = rawr_source_yaml.get('table-sources') assert table_sources, 'Missing definitions of source per table' # map text for table source onto Source objects for tbl, data in table_sources.items(): source_name = data['name'] source_value = data['value'] table_sources[tbl] = Source(source_name, source_value) label_placement_layers = rawr_yaml.get('label-placement-layers', {}) for geom_type, layers in label_placement_layers.items(): assert geom_type in ('point', 'polygon', 'linestring'), \ 'Geom type %r not understood, expecting point, polygon or ' \ 'linestring.' % (geom_type,) label_placement_layers[geom_type] = set(layers) indexes_cfg = rawr_yaml.get('indexes') assert indexes_cfg, 'Missing definitions of table indexes.' # source types are: # s3 - to fetch RAWR tiles from S3 # store - to fetch RAWR tiles from any tilequeue tile source # generate - to generate RAWR tiles directly, rather than trying to load # them from S3. this can be useful for standalone use and # testing. provide a postgresql subkey for database connection # settings. source_type = rawr_source_yaml.get('type') if source_type == 's3': rawr_source_s3_yaml = rawr_source_yaml.get('s3') bucket = rawr_source_s3_yaml.get('bucket') assert bucket, 'Missing rawr source s3 bucket' region = rawr_source_s3_yaml.get('region') assert region, 'Missing rawr source s3 region' prefix = rawr_source_s3_yaml.get('prefix') assert prefix, 'Missing rawr source s3 prefix' extension = rawr_source_s3_yaml.get('extension') assert extension, 'Missing rawr source s3 extension' allow_missing_tiles = rawr_source_s3_yaml.get( 'allow-missing-tiles', False) import boto3 from tilequeue.rawr import RawrS3Source s3_client = boto3.client('s3', region_name=region) tile_key_gen = make_s3_tile_key_generator(rawr_source_s3_yaml) storage = RawrS3Source( s3_client, bucket, prefix, extension, table_sources, tile_key_gen, allow_missing_tiles) elif source_type == 'generate': from raw_tiles.source.conn import ConnectionContextManager from raw_tiles.source.osm import OsmSource postgresql_cfg = rawr_source_yaml.get('postgresql') assert postgresql_cfg, 'Missing rawr postgresql config' conn_ctx = ConnectionContextManager(postgresql_cfg) rawr_osm_source = OsmSource(conn_ctx) storage = _NullRawrStorage(rawr_osm_source, table_sources) elif source_type == 'store': from tilequeue.store import make_store from tilequeue.rawr import RawrStoreSource store_cfg = rawr_source_yaml.get('store') store = make_store(store_cfg, credentials=cfg.subtree('aws credentials')) storage = RawrStoreSource(store, table_sources) else: assert False, 'Source type %r not understood. ' \ 'Options are s3, generate and store.' % (source_type,) # TODO: this needs to be configurable, everywhere! this is a long term # refactor - it's hard-coded in a bunch of places :-( max_z = 16 layers = _make_layer_info(layer_data, cfg.process_yaml_cfg) return make_rawr_data_fetcher( group_by_zoom, max_z, storage, layers, indexes_cfg, label_placement_layers)
def _test(self, input_layer_names, expected_layer_names): from shapely.geometry import Point from tilequeue.query.common import LayerInfo from tilequeue.query.rawr import make_rawr_data_fetcher from tilequeue.tile import coord_to_mercator_bounds from tilequeue.tile import mercator_point_to_coord top_zoom = 10 max_zoom = top_zoom + 6 def min_zoom_fn(shape, props, fid, meta): return top_zoom def props_fn(shape, props, fid, meta): return {} shape = Point(0, 0) props = {'name': 'Foo', 'name:en': 'Bar'} source = 'test' tables = TestGetTable({ 'planet_osm_point': [ (1, shape.wkb, props), ], }, source) tile = mercator_point_to_coord(max_zoom, shape.x, shape.y) top_tile = tile.zoomTo(top_zoom).container() layers = {} for name in input_layer_names: layers[name] = LayerInfo(min_zoom_fn, props_fn) storage = ConstantStorage(tables) fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers, [dict(type="osm")]) for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)): read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile)) # the RAWR query goes over features multiple times because of the # indexing, so we can't rely on all the properties for one feature to # be all together in the same place. this loops over all the features, # checking that there's only really one of them and gathering together # all the __%s_properties__ from all the rows for further testing. all_props = {} for row in read_rows: self.assertEquals(1, row['__id__']) self.assertEquals(shape.wkb, row['__geometry__']) for key, val in row.items(): if key.endswith('_properties__'): self.assertFalse(key in all_props) all_props[key] = val all_layer_names = set(expected_layer_names) | set(input_layer_names) for layer_name in all_layer_names: properties_name = '__%s_properties__' % layer_name self.assertTrue(properties_name in all_props) for key in props.keys(): actual_name = all_props[properties_name].get(key) if layer_name in expected_layer_names: expected_name = props.get(key) self.assertEquals( expected_name, actual_name, msg=('expected=%r, actual=%r for key=%r' % (expected_name, actual_name, key))) else: # check the name doesn't appear anywhere else self.assertEquals( None, actual_name, msg=('got actual=%r for key=%r, expected no value' % (actual_name, key)))
def test_buffered_land(self): # the buffered land is a polygon in the boundaries table (sort of), # but we shouldn't turn it into linestrings of the exterior ring. it # should stay as a polygon otherwise the border features won't be # correctly assigned maritime boundary properties, and they'll all # appear to be maritime boundaries. :-( from tilequeue.query.common import LayerInfo from tilequeue.query.rawr import make_rawr_data_fetcher from tilequeue.tile import coord_to_mercator_bounds from ModestMaps.Core import Coordinate from shapely.geometry import Polygon import shapely.wkb top_zoom = 10 max_zoom = top_zoom + 6 def min_zoom_fn(shape, props, fid, meta): return 8 def props_fn(shape, props, fid, meta): props['kind'] = 'maritime' return props z, x, y = (max_zoom, 0, 0) tile = Coordinate(zoom=z, column=x, row=y) top_tile = tile.zoomTo(top_zoom).container() bounds = coord_to_mercator_bounds(tile) shape = Polygon([ [bounds[0], bounds[1]], [bounds[0], bounds[3]], [bounds[2], bounds[1]], [bounds[0], bounds[1]], ]) props = { 'kind': 'maritime', 'maritime_boundary': True, 'source': 'tilezen.org', 'min_zoom': 0, } source = 'tilezen.org' tables = TestGetTable({ 'buffered_land': [ (1, shape.wkb, props), ], }, source) layers = { 'boundaries': LayerInfo(min_zoom_fn, props_fn), } storage = ConstantStorage(tables) index_cfg = [{ 'type': 'simple', 'table': 'buffered_land', 'layer': 'boundaries', }] fetch = make_rawr_data_fetcher(top_zoom, max_zoom, storage, layers, index_cfg) for fetcher, _ in fetch.fetch_tiles(_wrap(top_tile)): read_rows = fetcher(tile.zoom, coord_to_mercator_bounds(tile)) self.assertEqual(len(read_rows), 1) props = read_rows[0]['__boundaries_properties__'] self.assertEqual(props.get('kind'), 'maritime') # check no special boundaries geometry - should just be the polygon self.assertIsNone(read_rows[0].get('__boundaries_geometry__')) shape = shapely.wkb.loads(read_rows[0]['__geometry__']) self.assertEqual(shape.geom_type, 'Polygon')