def test_tiles_children_range(self):
from tilequeue.tile import coord_children
from tilequeue.tile import coord_children_range
from ModestMaps.Core import Coordinate
coord = Coordinate(3, 4, 2)
actual = list(coord_children_range(coord, 4))
self.assertEqual(20, len(actual))
children = list(coord_children(coord))
grandchildren_list = map(coord_children, children)
from itertools import chain
grandchildren = list(chain(*grandchildren_list))
exp = children + grandchildren
actual.sort()
exp.sort()
for actual_child, exp_child in zip(actual, exp):
self.assertEqual(exp_child, actual_child)
def test_tiles_children_range(self):
from tilequeue.tile import coord_children
from tilequeue.tile import coord_children_range
from ModestMaps.Core import Coordinate
coord = Coordinate(3, 4, 2)
actual = list(coord_children_range(coord, 4))
self.assertEqual(20, len(actual))
children = list(coord_children(coord))
grandchildren_list = map(coord_children, children)
from itertools import chain
grandchildren = list(chain(*grandchildren_list))
exp = children + grandchildren
actual.sort()
exp.sort()
for actual_child, exp_child in zip(actual, exp):
self.assertEqual(exp_child, actual_child)
def _make_tiles(self, shape, coord, metatile_zoom):
from tilequeue.format import mvt_format
from tilequeue.process import process_coord
from tilequeue.tile import coord_children_range
from tilequeue.tile import coord_to_mercator_bounds
db_features = [dict(
__id__=1,
__geometry__=shape.wkb,
__properties__={},
)]
nominal_zoom = coord.zoom + metatile_zoom
unpadded_bounds = coord_to_mercator_bounds(coord)
feature_layers = [dict(
layer_datum=dict(
name='fake_layer',
geometry_types=[shape.geom_type],
transform_fn_names=[],
sort_fn_name=None,
is_clipped=False
),
padded_bounds={shape.geom_type.lower(): unpadded_bounds},
features=db_features
)]
formats = [mvt_format]
post_process_data = {}
buffer_cfg = {}
cut_coords = [coord]
if nominal_zoom > coord.zoom:
cut_coords.extend(coord_children_range(coord, nominal_zoom))
def _output_fn(shape, props, fid, meta):
return dict(fake='data', min_zoom=0)
output_calc_mapping = dict(fake_layer=_output_fn)
tiles, extra = process_coord(
coord, nominal_zoom, feature_layers, post_process_data, formats,
unpadded_bounds, cut_coords, buffer_cfg, output_calc_mapping)
self.assertEqual(len(cut_coords), len(tiles))
return tiles, cut_coords
def __call__(self, stop):
saw_sentinel = False
while not stop.is_set():
try:
data = self.input_queue.get(timeout=timeout_seconds)
except Queue.Empty:
continue
if data is None:
saw_sentinel = True
break
coord = data['coord']
start = time.time()
try:
fetch_data = self.fetcher(coord)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
stacktrace = format_stacktrace_one_line(
(exc_type, exc_value, exc_traceback))
if isinstance(exc_value, TransactionRollbackError):
log_level = logging.WARNING
else:
log_level = logging.ERROR
self.logger.log(log_level, 'Error fetching: %s - %s' % (
serialize_coord(coord), stacktrace))
continue
metadata = data['metadata']
metadata['timing']['fetch_seconds'] = time.time() - start
# if we are at zoom level 16, it will serve as a metatile
# to derive the tiles underneath it
cut_coords = None
if coord.zoom == 16:
cut_coords = []
async_jobs = []
children_until = 20
# ask redis if there are any tiles underneath in the
# tiles of interest set
rci = self.redis_cache_index
async_fn = rci.is_coord_int_in_tiles_of_interest
for child in coord_children_range(coord, children_until):
zoomed_coord_int = coord_marshall_int(child)
async_result = self.io_pool.apply_async(
async_fn, (zoomed_coord_int,))
async_jobs.append((child, async_result))
async_exc_info = None
for async_job in async_jobs:
zoomed_coord, async_result = async_job
try:
is_coord_in_tiles_of_interest = async_result.get()
except:
async_exc_info = sys.exc_info()
stacktrace = format_stacktrace_one_line(async_exc_info)
self.logger.error(stacktrace)
else:
if is_coord_in_tiles_of_interest:
cut_coords.append(zoomed_coord)
if async_exc_info:
continue
data = dict(
metadata=metadata,
coord=coord,
feature_layers=fetch_data['feature_layers'],
unpadded_bounds=fetch_data['unpadded_bounds'],
cut_coords=cut_coords,
)
while not _non_blocking_put(self.output_queue, data):
if stop.is_set():
break
if not saw_sentinel:
_force_empty_queue(self.input_queue)
self.logger.debug('data fetch stopped')
def __call__(self, stop):
saw_sentinel = False
while not stop.is_set():
try:
data = self.input_queue.get(timeout=timeout_seconds)
except Queue.Empty:
continue
if data is None:
saw_sentinel = True
break
coord = data['coord']
start = time.time()
try:
fetch_data = self.fetcher(coord)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
stacktrace = format_stacktrace_one_line(
(exc_type, exc_value, exc_traceback))
if isinstance(exc_value, TransactionRollbackError):
log_level = logging.WARNING
else:
log_level = logging.ERROR
self.logger.log(log_level, 'Error fetching: %s - %s' % (
serialize_coord(coord), stacktrace))
continue
metadata = data['metadata']
metadata['timing']['fetch_seconds'] = time.time() - start
# if we are at zoom level 16, it will serve as a metatile
# to derive the tiles underneath it
cut_coords = None
if coord.zoom == 16:
cut_coords = []
async_jobs = []
children_until = 20
# ask redis if there are any tiles underneath in the
# tiles of interest set
rci = self.redis_cache_index
async_fn = rci.is_coord_int_in_tiles_of_interest
for child in coord_children_range(coord, children_until):
zoomed_coord_int = coord_marshall_int(child)
async_result = self.io_pool.apply_async(
async_fn, (zoomed_coord_int,))
async_jobs.append((child, async_result))
async_exc_info = None
for async_job in async_jobs:
zoomed_coord, async_result = async_job
try:
is_coord_in_tiles_of_interest = async_result.get()
except:
async_exc_info = sys.exc_info()
stacktrace = format_stacktrace_one_line(async_exc_info)
self.logger.error(stacktrace)
else:
if is_coord_in_tiles_of_interest:
cut_coords.append(zoomed_coord)
if async_exc_info:
continue
data = dict(
metadata=metadata,
coord=coord,
feature_layers=fetch_data['feature_layers'],
unpadded_bounds=fetch_data['unpadded_bounds'],
cut_coords=cut_coords,
)
while not _non_blocking_put(self.output_queue, data):
if stop.is_set():
break
if not saw_sentinel:
_force_empty_queue(self.input_queue)
self.logger.debug('data fetch stopped')