def _calc(metatile_size, tile_sizes):
coord = Coordinate(zoom=0, row=0, column=0)
metatile_zoom = metatile_zoom_from_size(metatile_size)
max_zoom = 16 - metatile_zoom
return calculate_sizes_by_zoom(
coord, metatile_zoom, tile_sizes, max_zoom)
def calculate_sizes_by_zoom(coord, metatile_zoom, cfg_tile_sizes, max_zoom):
"""
Returns a map of nominal zoom to the list of tile sizes to generate at that
zoom.
This is because we want to generate different metatile contents at
different zoom levels. At the most detailed zoom level, we want to generate
the smallest tiles possible, as this allows "overzooming" by simply
extracting the smaller tiles. At the minimum zoom, we want to get as close
as we can to zero nominal zoom by using any "unused" space in the metatile
for larger tile sizes that we're not generating.
For example, with 1x1 metatiles, the tile size is always 256px, and the
function will return {coord.zoom: [256]}
Note that max_zoom should be the maximum *coordinate* zoom, not nominal
zoom.
"""
from tilequeue.tile import metatile_zoom_from_size
tile_size_by_zoom = {}
nominal_zoom = coord.zoom + metatile_zoom
# check that the tile sizes are correct and within range.
for tile_size in cfg_tile_sizes:
assert tile_size >= 256
assert tile_size <= 256 * (1 << metatile_zoom)
assert _is_power_of_2(tile_size)
if coord.zoom >= max_zoom:
# all the tile_sizes down to 256 at the nominal zoom.
tile_sizes = []
tile_sizes.extend(cfg_tile_sizes)
lowest_tile_size = min(tile_sizes)
while lowest_tile_size > 256:
lowest_tile_size //= 2
tile_sizes.append(lowest_tile_size)
tile_size_by_zoom[nominal_zoom] = tile_sizes
elif coord.zoom <= 0:
# the tile_sizes, plus max(tile_sizes) size at nominal zooms decreasing
# down to 0 (or as close as we can get)
tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes
max_tile_size = max(cfg_tile_sizes)
max_tile_zoom = metatile_zoom_from_size(max_tile_size // 256)
assert max_tile_zoom <= metatile_zoom
for delta in range(0, metatile_zoom - max_tile_zoom):
z = nominal_zoom - (delta + 1)
tile_size_by_zoom[z] = [max_tile_size]
else:
# the tile_sizes at nominal zoom only.
tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes
return tile_size_by_zoom
def calculate_sizes_by_zoom(coord, metatile_zoom, cfg_tile_sizes, max_zoom):
"""
Returns a map of nominal zoom to the list of tile sizes to generate at that
zoom.
This is because we want to generate different metatile contents at
different zoom levels. At the most detailed zoom level, we want to generate
the smallest tiles possible, as this allows "overzooming" by simply
extracting the smaller tiles. At the minimum zoom, we want to get as close
as we can to zero nominal zoom by using any "unused" space in the metatile
for larger tile sizes that we're not generating.
For example, with 1x1 metatiles, the tile size is always 256px, and the
function will return {coord.zoom: [256]}
Note that max_zoom should be the maximum *coordinate* zoom, not nominal
zoom.
"""
from tilequeue.tile import metatile_zoom_from_size
tile_size_by_zoom = {}
nominal_zoom = coord.zoom + metatile_zoom
# check that the tile sizes are correct and within range.
for tile_size in cfg_tile_sizes:
assert tile_size >= 256
assert tile_size <= 256 * (1 << metatile_zoom)
assert _is_power_of_2(tile_size)
if coord.zoom >= max_zoom:
# all the tile_sizes down to 256 at the nominal zoom.
tile_sizes = []
tile_sizes.extend(cfg_tile_sizes)
lowest_tile_size = min(tile_sizes)
while lowest_tile_size > 256:
lowest_tile_size //= 2
tile_sizes.append(lowest_tile_size)
tile_size_by_zoom[nominal_zoom] = tile_sizes
elif coord.zoom <= 0:
# the tile_sizes, plus max(tile_sizes) size at nominal zooms decreasing
# down to 0 (or as close as we can get)
tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes
max_tile_size = max(cfg_tile_sizes)
max_tile_zoom = metatile_zoom_from_size(max_tile_size // 256)
assert max_tile_zoom <= metatile_zoom
for delta in range(0, metatile_zoom - max_tile_zoom):
z = nominal_zoom - (delta + 1)
tile_size_by_zoom[z] = [max_tile_size]
else:
# the tile_sizes at nominal zoom only.
tile_size_by_zoom[nominal_zoom] = cfg_tile_sizes
return tile_size_by_zoom
def test_mid_zoom(self):
from tilequeue.process import calculate_sizes_by_zoom
from tilequeue.tile import metatile_zoom_from_size
tile_sizes = [512]
metatile_size = 8
metatile_zoom = metatile_zoom_from_size(metatile_size)
max_zoom = 16 - metatile_zoom
for zoom in range(1, max_zoom - metatile_zoom):
coord = Coordinate(zoom=zoom, row=0, column=0)
sizes_by_zoom = calculate_sizes_by_zoom(
coord, metatile_zoom, tile_sizes, max_zoom)
nominal_zoom = zoom + metatile_zoom
self.assertEqual({nominal_zoom: tile_sizes}, sizes_by_zoom)
def _check_metatile(self, metatile_size):
from mock import patch
from shapely.geometry import box
from ModestMaps.Core import Coordinate
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import metatile_zoom_from_size
name = 'tilequeue.format.mvt.mvt_encode'
with patch(name, return_value='') as encode:
coord = Coordinate(0, 0, 0)
bounds = coord_to_mercator_bounds(coord)
pixel_fraction = 1.0 / 4096.0
box_width = pixel_fraction * (bounds[2] - bounds[0])
box_height = pixel_fraction * (bounds[3] - bounds[1])
shape = box(bounds[0], bounds[1],
bounds[0] + box_width,
bounds[1] + box_height)
metatile_zoom = metatile_zoom_from_size(metatile_size)
tiles, tile_coords = self._make_tiles(shape, coord, metatile_zoom)
num_tiles = 0
for z in range(0, metatile_zoom + 1):
num_tiles += 4**z
# resolution should be 4096 at 256px, which is metatile_zoom
# levels down from the extent of the world.
resolution = (bounds[2] - bounds[0]) / (4096 * 2**metatile_zoom)
self.assertEqual(num_tiles, len(tiles))
self.assertEqual(num_tiles, encode.call_count)
for (posargs, kwargs), coord in zip(encode.call_args_list,
tile_coords):
self.assertIn('quantize_bounds', kwargs)
quantize_bounds = kwargs['quantize_bounds']
extent = int(round((quantize_bounds[2] - quantize_bounds[0]) /
resolution))
self.assertIn('extents', kwargs)
actual_extent = kwargs['extents']
self.assertEquals(extent, actual_extent,
"Expected %r, not %r, for coord %r" %
(extent, actual_extent, coord))
def metatile_children_with_size(coord, metatile_zoom, nominal_zoom, tile_size):
"""
Return a list of all the coords which are children of the input metatile
at `coord` with zoom `metatile_zoom` (i.e: 0 for a single tile metatile,
1 for 2x2, 2 for 4x4, etc...) with size `tile_size` corrected for the
`nominal_zoom`.
For example, in a single tile metatile, the `tile_size` must be 256 and the
returned list contains only `coord`.
For an 8x8 metatile (`metatile_zoom = 3`), requesting the 512px children
would give a list of the 4x4 512px children at `coord.zoom + 2` with
nominal zoom `nominal_zoom`.
Correcting for nominal zoom means that some tiles may have coordinate zooms
lower than they would otherwise be. For example, the 0/0/0 tile with
metatile zoom 3 (8x8 256px tiles) would have 4x4 512px tiles at coordinate
zoom 2 and nominal zoom 3. At nominal zoom 2, there would be 2x2 512px
tiles at coordinate zoom 1.
"""
from tilequeue.tile import coord_children_subrange
from tilequeue.tile import metatile_zoom_from_size
assert tile_size >= 256
assert tile_size <= 256 * (1 << metatile_zoom)
assert _is_power_of_2(tile_size)
# delta is how many zoom levels _lower_ we want the child tiles, based on
# their tile size. 256px tiles are defined as being at nominal zoom, so
# delta = 0 for them.
delta = metatile_zoom_from_size(tile_size // 256)
zoom = nominal_zoom - delta
return list(coord_children_subrange(coord, zoom, zoom))
def metatile_children_with_size(coord, metatile_zoom, nominal_zoom, tile_size):
"""
Return a list of all the coords which are children of the input metatile
at `coord` with zoom `metatile_zoom` (i.e: 0 for a single tile metatile,
1 for 2x2, 2 for 4x4, etc...) with size `tile_size` corrected for the
`nominal_zoom`.
For example, in a single tile metatile, the `tile_size` must be 256 and the
returned list contains only `coord`.
For an 8x8 metatile (`metatile_zoom = 3`), requesting the 512px children
would give a list of the 4x4 512px children at `coord.zoom + 2` with
nominal zoom `nominal_zoom`.
Correcting for nominal zoom means that some tiles may have coordinate zooms
lower than they would otherwise be. For example, the 0/0/0 tile with
metatile zoom 3 (8x8 256px tiles) would have 4x4 512px tiles at coordinate
zoom 2 and nominal zoom 3. At nominal zoom 2, there would be 2x2 512px
tiles at coordinate zoom 1.
"""
from tilequeue.tile import coord_children_subrange
from tilequeue.tile import metatile_zoom_from_size
assert tile_size >= 256
assert tile_size <= 256 * (1 << metatile_zoom)
assert _is_power_of_2(tile_size)
# delta is how many zoom levels _lower_ we want the child tiles, based on
# their tile size. 256px tiles are defined as being at nominal zoom, so
# delta = 0 for them.
delta = metatile_zoom_from_size(tile_size // 256)
zoom = nominal_zoom - delta
return list(coord_children_subrange(coord, zoom, zoom))
'group-zoom'] == high_zoom_tilequeue_config['rawr'][
'group-zoom'] == missing_tile_tilequeue_config['rawr'][
'group-zoom']
queue_zoom = low_zoom_tilequeue_config['batch']['queue-zoom']
group_by_zoom = low_zoom_tilequeue_config['rawr']['group-zoom']
region = args.region or os.environ.get('AWS_DEFAULT_REGION')
if region is None:
print('[make_meta_tiles] ERROR: Need environment variable '
'AWS_DEFAULT_REGION to be set.')
sys.exit(1)
# check that metatile_size is within a sensible range
assert args.metatile_size > 0
assert args.metatile_size < 100
metatile_max_zoom = 16 - metatile_zoom_from_size(args.metatile_size)
def _extract_s3_buckets_args(json_list):
buckets = json.loads(json_list)
assert type(buckets) == list
assert len(buckets) == 1
buckets_str = [b.encode('utf-8') for b in buckets]
return buckets_str[0]
generator = None
tile_verifier = None
if args.use_tile_coords_generator:
bboxes = args.tile_coords_generator_bbox.split(',')
assert len(
bboxes
) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(
def __init__(self, yml):
self.yml = yml
self.aws_access_key_id = \
self._cfg('aws credentials aws_access_key_id') or \
os.environ.get('AWS_ACCESS_KEY_ID')
self.aws_secret_access_key = \
self._cfg('aws credentials aws_secret_access_key') or \
os.environ.get('AWS_SECRET_ACCESS_KEY')
self.queue_cfg = self.yml['queue']
self.store_type = self._cfg('store type')
self.s3_bucket = self._cfg('store name')
self.s3_reduced_redundancy = self._cfg('store reduced-redundancy')
self.s3_path = self._cfg('store path')
self.s3_date_prefix = self._cfg('store date-prefix')
self.s3_delete_retry_interval = \
self._cfg('store delete-retry-interval')
seed_cfg = self.yml['tiles']['seed']
self.seed_all_zoom_start = seed_cfg['all']['zoom-start']
self.seed_all_zoom_until = seed_cfg['all']['zoom-until']
self.seed_n_threads = seed_cfg['n-threads']
seed_metro_cfg = seed_cfg['metro-extract']
self.seed_metro_extract_url = seed_metro_cfg['url']
self.seed_metro_extract_zoom_start = seed_metro_cfg['zoom-start']
self.seed_metro_extract_zoom_until = seed_metro_cfg['zoom-until']
self.seed_metro_extract_cities = seed_metro_cfg['cities']
seed_top_tiles_cfg = seed_cfg['top-tiles']
self.seed_top_tiles_url = seed_top_tiles_cfg['url']
self.seed_top_tiles_zoom_start = seed_top_tiles_cfg['zoom-start']
self.seed_top_tiles_zoom_until = seed_top_tiles_cfg['zoom-until']
toi_store_cfg = self.yml['toi-store']
self.toi_store_type = toi_store_cfg['type']
if self.toi_store_type == 's3':
self.toi_store_s3_bucket = toi_store_cfg['s3']['bucket']
self.toi_store_s3_key = toi_store_cfg['s3']['key']
elif self.toi_store_type == 'file':
self.toi_store_file_name = toi_store_cfg['file']['name']
self.seed_should_add_to_tiles_of_interest = \
seed_cfg['should-add-to-tiles-of-interest']
seed_custom = seed_cfg['custom']
self.seed_custom_zoom_start = seed_custom['zoom-start']
self.seed_custom_zoom_until = seed_custom['zoom-until']
self.seed_custom_bboxes = seed_custom['bboxes']
if self.seed_custom_bboxes:
for bbox in self.seed_custom_bboxes:
assert len(bbox) == 4, (
'Seed config: custom bbox {} does not have exactly '
'four elements!').format(bbox)
min_x, min_y, max_x, max_y = bbox
assert min_x < max_x, \
'Invalid bbox. {} not less than {}'.format(min_x, max_x)
assert min_y < max_y, \
'Invalid bbox. {} not less than {}'.format(min_y, max_y)
self.seed_unique = seed_cfg['unique']
intersect_cfg = self.yml['tiles']['intersect']
self.intersect_expired_tiles_location = (
intersect_cfg['expired-location'])
self.intersect_zoom_until = intersect_cfg['parent-zoom-until']
self.logconfig = self._cfg('logging config')
self.redis_type = self._cfg('redis type')
self.redis_host = self._cfg('redis host')
self.redis_port = self._cfg('redis port')
self.redis_db = self._cfg('redis db')
self.redis_cache_set_key = self._cfg('redis cache-set-key')
self.statsd_host = None
if self.yml.get('statsd'):
self.statsd_host = self._cfg('statsd host')
self.statsd_port = self._cfg('statsd port')
self.statsd_prefix = self._cfg('statsd prefix')
process_cfg = self.yml['process']
self.n_simultaneous_query_sets = \
process_cfg['n-simultaneous-query-sets']
self.n_simultaneous_s3_storage = \
process_cfg['n-simultaneous-s3-storage']
self.log_queue_sizes = process_cfg['log-queue-sizes']
self.log_queue_sizes_interval_seconds = \
process_cfg['log-queue-sizes-interval-seconds']
self.query_cfg = process_cfg['query-config']
self.template_path = process_cfg['template-path']
self.reload_templates = process_cfg['reload-templates']
self.output_formats = process_cfg['formats']
self.buffer_cfg = process_cfg['buffer']
self.process_yaml_cfg = process_cfg['yaml']
self.postgresql_conn_info = self.yml['postgresql']
dbnames = self.postgresql_conn_info.get('dbnames')
assert dbnames is not None, 'Missing postgresql dbnames'
assert isinstance(dbnames, (tuple, list)), \
"Expecting postgresql 'dbnames' to be a list"
assert len(dbnames) > 0, 'No postgresql dbnames configured'
self.wof = self.yml.get('wof')
self.metatile_size = self._cfg('metatile size')
self.metatile_zoom = metatile_zoom_from_size(self.metatile_size)
self.metatile_start_zoom = self._cfg('metatile start-zoom')
self.max_zoom_with_changes = self._cfg('tiles max-zoom-with-changes')
assert self.max_zoom_with_changes > self.metatile_zoom
self.max_zoom = self.max_zoom_with_changes - self.metatile_zoom
self.sql_queue_buffer_size = self._cfg('queue_buffer_size sql')
self.proc_queue_buffer_size = self._cfg('queue_buffer_size proc')
self.s3_queue_buffer_size = self._cfg('queue_buffer_size s3')
self.tile_traffic_log_path = self._cfg(
'toi-prune tile-traffic-log-path')
self.group_by_zoom = self.subtree('rawr group-zoom')
self.tile_sizes = self._cfg('metatile tile-sizes')
if self.tile_sizes is None:
self.tile_sizes = [256 * (1 << z) for z in
reversed(xrange(0, self.metatile_zoom + 1))]
def __init__(self, yml):
self.yml = yml
self.aws_access_key_id = \
self._cfg('aws credentials aws_access_key_id') or \
os.environ.get('AWS_ACCESS_KEY_ID')
self.aws_secret_access_key = \
self._cfg('aws credentials aws_secret_access_key') or \
os.environ.get('AWS_SECRET_ACCESS_KEY')
self.queue_cfg = self.yml['queue']
self.store_type = self._cfg('store type')
self.s3_bucket = self._cfg('store name')
self.s3_reduced_redundancy = self._cfg('store reduced-redundancy')
self.s3_path = self._cfg('store path')
self.s3_date_prefix = self._cfg('store date-prefix')
self.s3_delete_retry_interval = \
self._cfg('store delete-retry-interval')
seed_cfg = self.yml['tiles']['seed']
self.seed_all_zoom_start = seed_cfg['all']['zoom-start']
self.seed_all_zoom_until = seed_cfg['all']['zoom-until']
self.seed_n_threads = seed_cfg['n-threads']
seed_metro_cfg = seed_cfg['metro-extract']
self.seed_metro_extract_url = seed_metro_cfg['url']
self.seed_metro_extract_zoom_start = seed_metro_cfg['zoom-start']
self.seed_metro_extract_zoom_until = seed_metro_cfg['zoom-until']
self.seed_metro_extract_cities = seed_metro_cfg['cities']
seed_top_tiles_cfg = seed_cfg['top-tiles']
self.seed_top_tiles_url = seed_top_tiles_cfg['url']
self.seed_top_tiles_zoom_start = seed_top_tiles_cfg['zoom-start']
self.seed_top_tiles_zoom_until = seed_top_tiles_cfg['zoom-until']
toi_store_cfg = self.yml['toi-store']
self.toi_store_type = toi_store_cfg['type']
if self.toi_store_type == 's3':
self.toi_store_s3_bucket = toi_store_cfg['s3']['bucket']
self.toi_store_s3_key = toi_store_cfg['s3']['key']
elif self.toi_store_type == 'file':
self.toi_store_file_name = toi_store_cfg['file']['name']
self.seed_should_add_to_tiles_of_interest = \
seed_cfg['should-add-to-tiles-of-interest']
seed_custom = seed_cfg['custom']
self.seed_custom_zoom_start = seed_custom['zoom-start']
self.seed_custom_zoom_until = seed_custom['zoom-until']
self.seed_custom_bboxes = seed_custom['bboxes']
if self.seed_custom_bboxes:
for bbox in self.seed_custom_bboxes:
assert len(bbox) == 4, (
'Seed config: custom bbox {} does not have exactly '
'four elements!').format(bbox)
min_x, min_y, max_x, max_y = bbox
assert min_x < max_x, \
'Invalid bbox. {} not less than {}'.format(min_x, max_x)
assert min_y < max_y, \
'Invalid bbox. {} not less than {}'.format(min_y, max_y)
self.seed_unique = seed_cfg['unique']
intersect_cfg = self.yml['tiles']['intersect']
self.intersect_expired_tiles_location = (
intersect_cfg['expired-location'])
self.intersect_zoom_until = intersect_cfg['parent-zoom-until']
self.logconfig = self._cfg('logging config')
self.redis_type = self._cfg('redis type')
self.redis_host = self._cfg('redis host')
self.redis_port = self._cfg('redis port')
self.redis_db = self._cfg('redis db')
self.redis_cache_set_key = self._cfg('redis cache-set-key')
self.statsd_host = None
if self.yml.get('statsd'):
self.statsd_host = self._cfg('statsd host')
self.statsd_port = self._cfg('statsd port')
self.statsd_prefix = self._cfg('statsd prefix')
process_cfg = self.yml['process']
self.n_simultaneous_query_sets = \
process_cfg['n-simultaneous-query-sets']
self.n_simultaneous_s3_storage = \
process_cfg['n-simultaneous-s3-storage']
self.log_queue_sizes = process_cfg['log-queue-sizes']
self.log_queue_sizes_interval_seconds = \
process_cfg['log-queue-sizes-interval-seconds']
self.query_cfg = process_cfg['query-config']
self.template_path = process_cfg['template-path']
self.reload_templates = process_cfg['reload-templates']
self.output_formats = process_cfg['formats']
self.buffer_cfg = process_cfg['buffer']
self.process_yaml_cfg = process_cfg['yaml']
self.postgresql_conn_info = self.yml['postgresql']
dbnames = self.postgresql_conn_info.get('dbnames')
assert dbnames is not None, 'Missing postgresql dbnames'
assert isinstance(dbnames, (tuple, list)), \
"Expecting postgresql 'dbnames' to be a list"
assert len(dbnames) > 0, 'No postgresql dbnames configured'
self.wof = self.yml.get('wof')
self.metatile_size = self._cfg('metatile size')
self.metatile_zoom = metatile_zoom_from_size(self.metatile_size)
self.metatile_start_zoom = self._cfg('metatile start-zoom')
self.max_zoom_with_changes = self._cfg('tiles max-zoom-with-changes')
assert self.max_zoom_with_changes > self.metatile_zoom
self.max_zoom = self.max_zoom_with_changes - self.metatile_zoom
self.sql_queue_buffer_size = self._cfg('queue_buffer_size sql')
self.proc_queue_buffer_size = self._cfg('queue_buffer_size proc')
self.s3_queue_buffer_size = self._cfg('queue_buffer_size s3')
self.tile_traffic_log_path = self._cfg(
'toi-prune tile-traffic-log-path')
self.group_by_zoom = self.subtree('rawr group-zoom')
def _calc(metatile_size, tile_sizes, max_zoom):
metatile_zoom = metatile_zoom_from_size(metatile_size)
coord = Coordinate(zoom=max_zoom - metatile_zoom, row=0, column=0)
return calculate_sizes_by_zoom(
coord, metatile_zoom, tile_sizes, max_zoom - metatile_zoom)