def _calc(metatile_size, tile_sizes): coord = Coordinate(zoom=0, row=0, column=0) metatile_zoom = metatile_zoom_from_size(metatile_size) max_zoom = 16 - metatile_zoom return calculate_sizes_by_zoom( coord, metatile_zoom, tile_sizes, max_zoom)
def calculate_sizes_by_zoom(coord, metatile_zoom, cfg_tile_sizes, max_zoom): """ Returns a map of nominal zoom to the list of tile sizes to generate at that zoom. This is because we want to generate different metatile contents at different zoom levels. At the most detailed zoom level, we want to generate the smallest tiles possible, as this allows "overzooming" by simply extracting the smaller tiles. At the minimum zoom, we want to get as close as we can to zero nominal zoom by using any "unused" space in the metatile for larger tile sizes that we're not generating. For example, with 1x1 metatiles, the tile size is always 256px, and the function will return {coord.zoom: [256]} Note that max_zoom should be the maximum *coordinate* zoom, not nominal zoom. """ from tilequeue.tile import metatile_zoom_from_size tile_size_by_zoom = {} nominal_zoom = coord.zoom + metatile_zoom # check that the tile sizes are correct and within range. for tile_size in cfg_tile_sizes: assert tile_size >= 256 assert tile_size <= 256 * (1 << metatile_zoom) assert _is_power_of_2(tile_size) if coord.zoom >= max_zoom: # all the tile_sizes down to 256 at the nominal zoom. tile_sizes = [] tile_sizes.extend(cfg_tile_sizes) lowest_tile_size = min(tile_sizes) while lowest_tile_size > 256: lowest_tile_size //= 2 tile_sizes.append(lowest_tile_size) tile_size_by_zoom[nominal_zoom] = tile_sizes elif coord.zoom <= 0: # the tile_sizes, plus max(tile_sizes) size at nominal zooms decreasing # down to 0 (or as close as we can get) tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes max_tile_size = max(cfg_tile_sizes) max_tile_zoom = metatile_zoom_from_size(max_tile_size // 256) assert max_tile_zoom <= metatile_zoom for delta in range(0, metatile_zoom - max_tile_zoom): z = nominal_zoom - (delta + 1) tile_size_by_zoom[z] = [max_tile_size] else: # the tile_sizes at nominal zoom only. tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes return tile_size_by_zoom
def calculate_sizes_by_zoom(coord, metatile_zoom, cfg_tile_sizes, max_zoom): """ Returns a map of nominal zoom to the list of tile sizes to generate at that zoom. This is because we want to generate different metatile contents at different zoom levels. At the most detailed zoom level, we want to generate the smallest tiles possible, as this allows "overzooming" by simply extracting the smaller tiles. At the minimum zoom, we want to get as close as we can to zero nominal zoom by using any "unused" space in the metatile for larger tile sizes that we're not generating. For example, with 1x1 metatiles, the tile size is always 256px, and the function will return {coord.zoom: [256]} Note that max_zoom should be the maximum *coordinate* zoom, not nominal zoom. """ from tilequeue.tile import metatile_zoom_from_size tile_size_by_zoom = {} nominal_zoom = coord.zoom + metatile_zoom # check that the tile sizes are correct and within range. for tile_size in cfg_tile_sizes: assert tile_size >= 256 assert tile_size <= 256 * (1 << metatile_zoom) assert _is_power_of_2(tile_size) if coord.zoom >= max_zoom: # all the tile_sizes down to 256 at the nominal zoom. tile_sizes = [] tile_sizes.extend(cfg_tile_sizes) lowest_tile_size = min(tile_sizes) while lowest_tile_size > 256: lowest_tile_size //= 2 tile_sizes.append(lowest_tile_size) tile_size_by_zoom[nominal_zoom] = tile_sizes elif coord.zoom <= 0: # the tile_sizes, plus max(tile_sizes) size at nominal zooms decreasing # down to 0 (or as close as we can get) tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes max_tile_size = max(cfg_tile_sizes) max_tile_zoom = metatile_zoom_from_size(max_tile_size // 256) assert max_tile_zoom <= metatile_zoom for delta in range(0, metatile_zoom - max_tile_zoom): z = nominal_zoom - (delta + 1) tile_size_by_zoom[z] = [max_tile_size] else: # the tile_sizes at nominal zoom only. tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes return tile_size_by_zoom
def test_mid_zoom(self): from tilequeue.process import calculate_sizes_by_zoom from tilequeue.tile import metatile_zoom_from_size tile_sizes = [512] metatile_size = 8 metatile_zoom = metatile_zoom_from_size(metatile_size) max_zoom = 16 - metatile_zoom for zoom in range(1, max_zoom - metatile_zoom): coord = Coordinate(zoom=zoom, row=0, column=0) sizes_by_zoom = calculate_sizes_by_zoom( coord, metatile_zoom, tile_sizes, max_zoom) nominal_zoom = zoom + metatile_zoom self.assertEqual({nominal_zoom: tile_sizes}, sizes_by_zoom)
def _check_metatile(self, metatile_size): from mock import patch from shapely.geometry import box from ModestMaps.Core import Coordinate from tilequeue.tile import coord_to_mercator_bounds from tilequeue.tile import metatile_zoom_from_size name = 'tilequeue.format.mvt.mvt_encode' with patch(name, return_value='') as encode: coord = Coordinate(0, 0, 0) bounds = coord_to_mercator_bounds(coord) pixel_fraction = 1.0 / 4096.0 box_width = pixel_fraction * (bounds[2] - bounds[0]) box_height = pixel_fraction * (bounds[3] - bounds[1]) shape = box(bounds[0], bounds[1], bounds[0] + box_width, bounds[1] + box_height) metatile_zoom = metatile_zoom_from_size(metatile_size) tiles, tile_coords = self._make_tiles(shape, coord, metatile_zoom) num_tiles = 0 for z in range(0, metatile_zoom + 1): num_tiles += 4**z # resolution should be 4096 at 256px, which is metatile_zoom # levels down from the extent of the world. resolution = (bounds[2] - bounds[0]) / (4096 * 2**metatile_zoom) self.assertEqual(num_tiles, len(tiles)) self.assertEqual(num_tiles, encode.call_count) for (posargs, kwargs), coord in zip(encode.call_args_list, tile_coords): self.assertIn('quantize_bounds', kwargs) quantize_bounds = kwargs['quantize_bounds'] extent = int(round((quantize_bounds[2] - quantize_bounds[0]) / resolution)) self.assertIn('extents', kwargs) actual_extent = kwargs['extents'] self.assertEquals(extent, actual_extent, "Expected %r, not %r, for coord %r" % (extent, actual_extent, coord))
def metatile_children_with_size(coord, metatile_zoom, nominal_zoom, tile_size): """ Return a list of all the coords which are children of the input metatile at `coord` with zoom `metatile_zoom` (i.e: 0 for a single tile metatile, 1 for 2x2, 2 for 4x4, etc...) with size `tile_size` corrected for the `nominal_zoom`. For example, in a single tile metatile, the `tile_size` must be 256 and the returned list contains only `coord`. For an 8x8 metatile (`metatile_zoom = 3`), requesting the 512px children would give a list of the 4x4 512px children at `coord.zoom + 2` with nominal zoom `nominal_zoom`. Correcting for nominal zoom means that some tiles may have coordinate zooms lower than they would otherwise be. For example, the 0/0/0 tile with metatile zoom 3 (8x8 256px tiles) would have 4x4 512px tiles at coordinate zoom 2 and nominal zoom 3. At nominal zoom 2, there would be 2x2 512px tiles at coordinate zoom 1. """ from tilequeue.tile import coord_children_subrange from tilequeue.tile import metatile_zoom_from_size assert tile_size >= 256 assert tile_size <= 256 * (1 << metatile_zoom) assert _is_power_of_2(tile_size) # delta is how many zoom levels _lower_ we want the child tiles, based on # their tile size. 256px tiles are defined as being at nominal zoom, so # delta = 0 for them. delta = metatile_zoom_from_size(tile_size // 256) zoom = nominal_zoom - delta return list(coord_children_subrange(coord, zoom, zoom))
def metatile_children_with_size(coord, metatile_zoom, nominal_zoom, tile_size): """ Return a list of all the coords which are children of the input metatile at `coord` with zoom `metatile_zoom` (i.e: 0 for a single tile metatile, 1 for 2x2, 2 for 4x4, etc...) with size `tile_size` corrected for the `nominal_zoom`. For example, in a single tile metatile, the `tile_size` must be 256 and the returned list contains only `coord`. For an 8x8 metatile (`metatile_zoom = 3`), requesting the 512px children would give a list of the 4x4 512px children at `coord.zoom + 2` with nominal zoom `nominal_zoom`. Correcting for nominal zoom means that some tiles may have coordinate zooms lower than they would otherwise be. For example, the 0/0/0 tile with metatile zoom 3 (8x8 256px tiles) would have 4x4 512px tiles at coordinate zoom 2 and nominal zoom 3. At nominal zoom 2, there would be 2x2 512px tiles at coordinate zoom 1. """ from tilequeue.tile import coord_children_subrange from tilequeue.tile import metatile_zoom_from_size assert tile_size >= 256 assert tile_size <= 256 * (1 << metatile_zoom) assert _is_power_of_2(tile_size) # delta is how many zoom levels _lower_ we want the child tiles, based on # their tile size. 256px tiles are defined as being at nominal zoom, so # delta = 0 for them. delta = metatile_zoom_from_size(tile_size // 256) zoom = nominal_zoom - delta return list(coord_children_subrange(coord, zoom, zoom))
'group-zoom'] == high_zoom_tilequeue_config['rawr'][ 'group-zoom'] == missing_tile_tilequeue_config['rawr'][ 'group-zoom'] queue_zoom = low_zoom_tilequeue_config['batch']['queue-zoom'] group_by_zoom = low_zoom_tilequeue_config['rawr']['group-zoom'] region = args.region or os.environ.get('AWS_DEFAULT_REGION') if region is None: print('[make_meta_tiles] ERROR: Need environment variable ' 'AWS_DEFAULT_REGION to be set.') sys.exit(1) # check that metatile_size is within a sensible range assert args.metatile_size > 0 assert args.metatile_size < 100 metatile_max_zoom = 16 - metatile_zoom_from_size(args.metatile_size) def _extract_s3_buckets_args(json_list): buckets = json.loads(json_list) assert type(buckets) == list assert len(buckets) == 1 buckets_str = [b.encode('utf-8') for b in buckets] return buckets_str[0] generator = None tile_verifier = None if args.use_tile_coords_generator: bboxes = args.tile_coords_generator_bbox.split(',') assert len( bboxes ) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(
def __init__(self, yml): self.yml = yml self.aws_access_key_id = \ self._cfg('aws credentials aws_access_key_id') or \ os.environ.get('AWS_ACCESS_KEY_ID') self.aws_secret_access_key = \ self._cfg('aws credentials aws_secret_access_key') or \ os.environ.get('AWS_SECRET_ACCESS_KEY') self.queue_cfg = self.yml['queue'] self.store_type = self._cfg('store type') self.s3_bucket = self._cfg('store name') self.s3_reduced_redundancy = self._cfg('store reduced-redundancy') self.s3_path = self._cfg('store path') self.s3_date_prefix = self._cfg('store date-prefix') self.s3_delete_retry_interval = \ self._cfg('store delete-retry-interval') seed_cfg = self.yml['tiles']['seed'] self.seed_all_zoom_start = seed_cfg['all']['zoom-start'] self.seed_all_zoom_until = seed_cfg['all']['zoom-until'] self.seed_n_threads = seed_cfg['n-threads'] seed_metro_cfg = seed_cfg['metro-extract'] self.seed_metro_extract_url = seed_metro_cfg['url'] self.seed_metro_extract_zoom_start = seed_metro_cfg['zoom-start'] self.seed_metro_extract_zoom_until = seed_metro_cfg['zoom-until'] self.seed_metro_extract_cities = seed_metro_cfg['cities'] seed_top_tiles_cfg = seed_cfg['top-tiles'] self.seed_top_tiles_url = seed_top_tiles_cfg['url'] self.seed_top_tiles_zoom_start = seed_top_tiles_cfg['zoom-start'] self.seed_top_tiles_zoom_until = seed_top_tiles_cfg['zoom-until'] toi_store_cfg = self.yml['toi-store'] self.toi_store_type = toi_store_cfg['type'] if self.toi_store_type == 's3': self.toi_store_s3_bucket = toi_store_cfg['s3']['bucket'] self.toi_store_s3_key = toi_store_cfg['s3']['key'] elif self.toi_store_type == 'file': self.toi_store_file_name = toi_store_cfg['file']['name'] self.seed_should_add_to_tiles_of_interest = \ seed_cfg['should-add-to-tiles-of-interest'] seed_custom = seed_cfg['custom'] self.seed_custom_zoom_start = seed_custom['zoom-start'] self.seed_custom_zoom_until = seed_custom['zoom-until'] self.seed_custom_bboxes = seed_custom['bboxes'] if self.seed_custom_bboxes: for bbox in self.seed_custom_bboxes: assert len(bbox) == 4, ( 'Seed config: custom bbox {} does not have exactly ' 'four elements!').format(bbox) min_x, min_y, max_x, max_y = bbox assert min_x < max_x, \ 'Invalid bbox. {} not less than {}'.format(min_x, max_x) assert min_y < max_y, \ 'Invalid bbox. {} not less than {}'.format(min_y, max_y) self.seed_unique = seed_cfg['unique'] intersect_cfg = self.yml['tiles']['intersect'] self.intersect_expired_tiles_location = ( intersect_cfg['expired-location']) self.intersect_zoom_until = intersect_cfg['parent-zoom-until'] self.logconfig = self._cfg('logging config') self.redis_type = self._cfg('redis type') self.redis_host = self._cfg('redis host') self.redis_port = self._cfg('redis port') self.redis_db = self._cfg('redis db') self.redis_cache_set_key = self._cfg('redis cache-set-key') self.statsd_host = None if self.yml.get('statsd'): self.statsd_host = self._cfg('statsd host') self.statsd_port = self._cfg('statsd port') self.statsd_prefix = self._cfg('statsd prefix') process_cfg = self.yml['process'] self.n_simultaneous_query_sets = \ process_cfg['n-simultaneous-query-sets'] self.n_simultaneous_s3_storage = \ process_cfg['n-simultaneous-s3-storage'] self.log_queue_sizes = process_cfg['log-queue-sizes'] self.log_queue_sizes_interval_seconds = \ process_cfg['log-queue-sizes-interval-seconds'] self.query_cfg = process_cfg['query-config'] self.template_path = process_cfg['template-path'] self.reload_templates = process_cfg['reload-templates'] self.output_formats = process_cfg['formats'] self.buffer_cfg = process_cfg['buffer'] self.process_yaml_cfg = process_cfg['yaml'] self.postgresql_conn_info = self.yml['postgresql'] dbnames = self.postgresql_conn_info.get('dbnames') assert dbnames is not None, 'Missing postgresql dbnames' assert isinstance(dbnames, (tuple, list)), \ "Expecting postgresql 'dbnames' to be a list" assert len(dbnames) > 0, 'No postgresql dbnames configured' self.wof = self.yml.get('wof') self.metatile_size = self._cfg('metatile size') self.metatile_zoom = metatile_zoom_from_size(self.metatile_size) self.metatile_start_zoom = self._cfg('metatile start-zoom') self.max_zoom_with_changes = self._cfg('tiles max-zoom-with-changes') assert self.max_zoom_with_changes > self.metatile_zoom self.max_zoom = self.max_zoom_with_changes - self.metatile_zoom self.sql_queue_buffer_size = self._cfg('queue_buffer_size sql') self.proc_queue_buffer_size = self._cfg('queue_buffer_size proc') self.s3_queue_buffer_size = self._cfg('queue_buffer_size s3') self.tile_traffic_log_path = self._cfg( 'toi-prune tile-traffic-log-path') self.group_by_zoom = self.subtree('rawr group-zoom') self.tile_sizes = self._cfg('metatile tile-sizes') if self.tile_sizes is None: self.tile_sizes = [256 * (1 << z) for z in reversed(xrange(0, self.metatile_zoom + 1))]
def __init__(self, yml): self.yml = yml self.aws_access_key_id = \ self._cfg('aws credentials aws_access_key_id') or \ os.environ.get('AWS_ACCESS_KEY_ID') self.aws_secret_access_key = \ self._cfg('aws credentials aws_secret_access_key') or \ os.environ.get('AWS_SECRET_ACCESS_KEY') self.queue_cfg = self.yml['queue'] self.store_type = self._cfg('store type') self.s3_bucket = self._cfg('store name') self.s3_reduced_redundancy = self._cfg('store reduced-redundancy') self.s3_path = self._cfg('store path') self.s3_date_prefix = self._cfg('store date-prefix') self.s3_delete_retry_interval = \ self._cfg('store delete-retry-interval') seed_cfg = self.yml['tiles']['seed'] self.seed_all_zoom_start = seed_cfg['all']['zoom-start'] self.seed_all_zoom_until = seed_cfg['all']['zoom-until'] self.seed_n_threads = seed_cfg['n-threads'] seed_metro_cfg = seed_cfg['metro-extract'] self.seed_metro_extract_url = seed_metro_cfg['url'] self.seed_metro_extract_zoom_start = seed_metro_cfg['zoom-start'] self.seed_metro_extract_zoom_until = seed_metro_cfg['zoom-until'] self.seed_metro_extract_cities = seed_metro_cfg['cities'] seed_top_tiles_cfg = seed_cfg['top-tiles'] self.seed_top_tiles_url = seed_top_tiles_cfg['url'] self.seed_top_tiles_zoom_start = seed_top_tiles_cfg['zoom-start'] self.seed_top_tiles_zoom_until = seed_top_tiles_cfg['zoom-until'] toi_store_cfg = self.yml['toi-store'] self.toi_store_type = toi_store_cfg['type'] if self.toi_store_type == 's3': self.toi_store_s3_bucket = toi_store_cfg['s3']['bucket'] self.toi_store_s3_key = toi_store_cfg['s3']['key'] elif self.toi_store_type == 'file': self.toi_store_file_name = toi_store_cfg['file']['name'] self.seed_should_add_to_tiles_of_interest = \ seed_cfg['should-add-to-tiles-of-interest'] seed_custom = seed_cfg['custom'] self.seed_custom_zoom_start = seed_custom['zoom-start'] self.seed_custom_zoom_until = seed_custom['zoom-until'] self.seed_custom_bboxes = seed_custom['bboxes'] if self.seed_custom_bboxes: for bbox in self.seed_custom_bboxes: assert len(bbox) == 4, ( 'Seed config: custom bbox {} does not have exactly ' 'four elements!').format(bbox) min_x, min_y, max_x, max_y = bbox assert min_x < max_x, \ 'Invalid bbox. {} not less than {}'.format(min_x, max_x) assert min_y < max_y, \ 'Invalid bbox. {} not less than {}'.format(min_y, max_y) self.seed_unique = seed_cfg['unique'] intersect_cfg = self.yml['tiles']['intersect'] self.intersect_expired_tiles_location = ( intersect_cfg['expired-location']) self.intersect_zoom_until = intersect_cfg['parent-zoom-until'] self.logconfig = self._cfg('logging config') self.redis_type = self._cfg('redis type') self.redis_host = self._cfg('redis host') self.redis_port = self._cfg('redis port') self.redis_db = self._cfg('redis db') self.redis_cache_set_key = self._cfg('redis cache-set-key') self.statsd_host = None if self.yml.get('statsd'): self.statsd_host = self._cfg('statsd host') self.statsd_port = self._cfg('statsd port') self.statsd_prefix = self._cfg('statsd prefix') process_cfg = self.yml['process'] self.n_simultaneous_query_sets = \ process_cfg['n-simultaneous-query-sets'] self.n_simultaneous_s3_storage = \ process_cfg['n-simultaneous-s3-storage'] self.log_queue_sizes = process_cfg['log-queue-sizes'] self.log_queue_sizes_interval_seconds = \ process_cfg['log-queue-sizes-interval-seconds'] self.query_cfg = process_cfg['query-config'] self.template_path = process_cfg['template-path'] self.reload_templates = process_cfg['reload-templates'] self.output_formats = process_cfg['formats'] self.buffer_cfg = process_cfg['buffer'] self.process_yaml_cfg = process_cfg['yaml'] self.postgresql_conn_info = self.yml['postgresql'] dbnames = self.postgresql_conn_info.get('dbnames') assert dbnames is not None, 'Missing postgresql dbnames' assert isinstance(dbnames, (tuple, list)), \ "Expecting postgresql 'dbnames' to be a list" assert len(dbnames) > 0, 'No postgresql dbnames configured' self.wof = self.yml.get('wof') self.metatile_size = self._cfg('metatile size') self.metatile_zoom = metatile_zoom_from_size(self.metatile_size) self.metatile_start_zoom = self._cfg('metatile start-zoom') self.max_zoom_with_changes = self._cfg('tiles max-zoom-with-changes') assert self.max_zoom_with_changes > self.metatile_zoom self.max_zoom = self.max_zoom_with_changes - self.metatile_zoom self.sql_queue_buffer_size = self._cfg('queue_buffer_size sql') self.proc_queue_buffer_size = self._cfg('queue_buffer_size proc') self.s3_queue_buffer_size = self._cfg('queue_buffer_size s3') self.tile_traffic_log_path = self._cfg( 'toi-prune tile-traffic-log-path') self.group_by_zoom = self.subtree('rawr group-zoom')
def _calc(metatile_size, tile_sizes, max_zoom): metatile_zoom = metatile_zoom_from_size(metatile_size) coord = Coordinate(zoom=max_zoom - metatile_zoom, row=0, column=0) return calculate_sizes_by_zoom( coord, metatile_zoom, tile_sizes, max_zoom - metatile_zoom)