def __call__(self, coords):
# this will produce the intersected list of coordinates with the toi,
# all the way to low zoom level tiles
intersected_coords, intersect_metrics, timing = \
self.toi_intersector(coords)
low_zoom_coord_ints = set()
grouped_by_zoom = defaultdict(list)
for coord in intersected_coords:
if self.group_by_zoom <= coord.zoom:
parent = coord.zoomTo(self.group_by_zoom).container()
parent_coord_int = coord_marshall_int(parent)
grouped_by_zoom[parent_coord_int].append(coord)
else:
coord_int = coord_marshall_int(coord)
low_zoom_coord_ints.add(coord_int)
n_coords = 0
payloads = []
for _, coords in grouped_by_zoom.iteritems():
payload = self.msg_marshaller.marshall(coords)
payloads.append(payload)
n_coords += len(coords)
# add all low zooms into a single payload
low_zoom_coords = []
for coord_int in low_zoom_coord_ints:
coord = coord_unmarshall_int(coord_int)
low_zoom_coords.append(coord)
if low_zoom_coords:
low_zoom_payload = self.msg_marshaller.marshall(low_zoom_coords)
payloads.append(low_zoom_payload)
n_payloads = len(payloads)
rawr_queue_batch_size = 10
n_msgs_sent = 0
for payloads_chunk in grouper(payloads, rawr_queue_batch_size):
self.rawr_queue.send(payloads_chunk, self.logger)
n_msgs_sent += 1
if self.logger:
self.logger.info(
'Rawr tiles enqueued: '
'coords(%d) payloads(%d) enqueue_calls(%d) '
'toi(%d) hits(%d) misses(%d)' %
(n_coords, n_payloads, n_msgs_sent,
intersect_metrics['n_toi'], intersect_metrics['hits'],
intersect_metrics['misses']))
self.stats_handler(n_coords, n_payloads, n_msgs_sent,
intersect_metrics, timing)
def __call__(self, coords):
# this will produce the intersected list of coordinates with the toi,
# all the way to low zoom level tiles
intersected_coords, intersect_metrics, timing = \
self.toi_intersector(coords)
low_zoom_coord_ints = set()
grouped_by_zoom = defaultdict(list)
for coord in intersected_coords:
if self.group_by_zoom <= coord.zoom:
parent = coord.zoomTo(self.group_by_zoom).container()
parent_coord_int = coord_marshall_int(parent)
grouped_by_zoom[parent_coord_int].append(coord)
else:
coord_int = coord_marshall_int(coord)
low_zoom_coord_ints.add(coord_int)
n_coords = 0
payloads = []
for _, coords in grouped_by_zoom.iteritems():
payload = self.msg_marshaller.marshall(coords)
payloads.append(payload)
n_coords += len(coords)
# add all low zooms into a single payload
low_zoom_coords = []
for coord_int in low_zoom_coord_ints:
coord = coord_unmarshall_int(coord_int)
low_zoom_coords.append(coord)
if low_zoom_coords:
low_zoom_payload = self.msg_marshaller.marshall(low_zoom_coords)
payloads.append(low_zoom_payload)
n_payloads = len(payloads)
rawr_queue_batch_size = 10
n_msgs_sent = 0
for payloads_chunk in grouper(payloads, rawr_queue_batch_size):
self.rawr_queue.send(payloads_chunk, self.logger)
n_msgs_sent += 1
if self.logger:
self.logger.info(
'Rawr tiles enqueued: '
'coords(%d) payloads(%d) enqueue_calls(%d) '
'toi(%d) hits(%d) misses(%d)' %
(n_coords, n_payloads, n_msgs_sent,
intersect_metrics['n_toi'], intersect_metrics['hits'],
intersect_metrics['misses']))
self.stats_handler(n_coords, n_payloads, n_msgs_sent,
intersect_metrics, timing)
def read(self, max_to_read=10):
with self.redis_client.pipeline() as pipe:
pipe.lrange(self.queue_key, 0, max_to_read - 1)
pipe.ltrim(self.queue_key, max_to_read, -1)
coord_ints, _ = pipe.execute()
if not coord_ints:
time.sleep(self.sleep_time_seconds_when_empty)
return []
coord_msgs = []
for coord_int in coord_ints:
coord = coord_unmarshall_int(coord_int)
coord_msg = CoordMessage(coord, None)
coord_msgs.append(coord_msg)
return coord_msgs
def test_roundtrip_serialization(self):
from tilequeue.tile import coord_marshall_int
from tilequeue.tile import coord_unmarshall_int
from tilequeue.tile import seed_tiles
from ModestMaps.Core import Coordinate
from itertools import chain
seed_coords = seed_tiles(0, 5)
example_coords = [
Coordinate(zoom=20, column=1002463, row=312816),
Coordinate(zoom=30, column=12345678, row=12345678),
]
coords = chain(seed_coords, example_coords)
for coord in coords:
self.assertEquals(coord,
coord_unmarshall_int(coord_marshall_int(coord)))
def tilequeue_dump_tiles_of_interest(cfg, peripherals):
logger = make_logger(cfg, 'dump_tiles_of_interest')
logger.info('Dumping tiles of interest')
logger.info('Fetching tiles of interest ...')
coords = peripherals.redis_cache_index.fetch_tiles_of_interest()
n_toi = len(coords)
logger.info('Fetching tiles of interest ... done')
toi_filename = "toi.txt"
logger.info('Writing %d tiles of interest to %s ...', n_toi, toi_filename)
with open(toi_filename, "w") as f:
for coord in coords:
c = coord_unmarshall_int(coord)
f.write("{}/{}/{}\n".format(c.zoom, c.column, c.row))
logger.info('Writing %d tiles of interest to %s ... done', n_toi,
toi_filename)
def test_tilequeue_explode_and_intersect(self):
from tilequeue.command import explode_and_intersect
from tilequeue.tile import coord_marshall_int
from tilequeue.tile import coord_unmarshall_int
from ModestMaps.Core import Coordinate
sample_coord = Coordinate(zoom=14, column=250, row=250)
sample_coord_int = coord_marshall_int(sample_coord)
tiles_of_interest = [sample_coord_int]
for i in (10, 11, 12, 13):
coord = sample_coord.zoomTo(i)
coord_int = coord_marshall_int(coord)
tiles_of_interest.append(coord_int)
exploded = explode_and_intersect([sample_coord_int], tiles_of_interest,
until=11)
coord_ints = list(exploded)
for coord_int in coord_ints:
coord = coord_unmarshall_int(coord_int)
self.failUnless(coord.zoom > 10)
self.assertEqual(4, len(coord_ints))
def tilequeue_enqueue_tiles_of_interest(cfg, peripherals):
logger = make_logger(cfg, 'enqueue_tiles_of_interest')
logger.info('Enqueueing tiles of interest')
sqs_queue = peripherals.queue
logger.info('Fetching tiles of interest ...')
tiles_of_interest = peripherals.redis_cache_index.fetch_tiles_of_interest()
n_toi = len(tiles_of_interest)
logger.info('Fetching tiles of interest ... done')
coords = []
for coord_int in tiles_of_interest:
coord = coord_unmarshall_int(coord_int)
if coord.zoom <= 16:
coords.append(coord)
enqueuer = ThreadedEnqueuer(sqs_queue, cfg.seed_n_threads, logger)
n_queued, n_in_flight = enqueuer(coords)
logger.info('%d enqueued - %d in flight' % (n_queued, n_in_flight))
logger.info('%d tiles of interest processed' % n_toi)
def tilequeue_enqueue_tiles_of_interest(cfg, peripherals):
logger = make_logger(cfg, 'enqueue_tiles_of_interest')
logger.info('Enqueueing tiles of interest')
sqs_queue = peripherals.queue
logger.info('Fetching tiles of interest ...')
tiles_of_interest = peripherals.redis_cache_index.fetch_tiles_of_interest()
n_toi = len(tiles_of_interest)
logger.info('Fetching tiles of interest ... done')
coords = []
for coord_int in tiles_of_interest:
coord = coord_unmarshall_int(coord_int)
if coord.zoom <= 16:
coords.append(coord)
enqueuer = ThreadedEnqueuer(sqs_queue, cfg.seed_n_threads, logger)
n_queued, n_in_flight = enqueuer(coords)
logger.info('%d enqueued - %d in flight' % (n_queued, n_in_flight))
logger.info('%d tiles of interest processed' % n_toi)
def test_tilequeue_explode_and_intersect(self):
from tilequeue.command import explode_and_intersect
from tilequeue.tile import coord_marshall_int
from tilequeue.tile import coord_unmarshall_int
from ModestMaps.Core import Coordinate
sample_coord = Coordinate(zoom=14, column=250, row=250)
sample_coord_int = coord_marshall_int(sample_coord)
tiles_of_interest = [sample_coord_int]
for i in (10, 11, 12, 13):
coord = sample_coord.zoomTo(i)
coord_int = coord_marshall_int(coord)
tiles_of_interest.append(coord_int)
exploded, metrics = explode_and_intersect(
[sample_coord_int], tiles_of_interest, until=11)
coord_ints = list(exploded)
for coord_int in coord_ints:
coord = coord_unmarshall_int(coord_int)
self.failUnless(coord.zoom > 10)
self.assertEqual(4, len(coord_ints))
self.assertEqual(4, metrics['hits'])
self.assertEqual(0, metrics['misses'])
self.assertEqual(4, metrics['total'])
def save_set_to_fp(the_set, fp):
for coord_int in sorted(the_set):
coord = coord_unmarshall_int(coord_int)
fp.write(serialize_coord(coord))
fp.write('\n')
def find_intersection(self, diff_set_key):
intersection = self.redis_client.sinter(self.cache_set_key,
diff_set_key)
for coord_int in intersection:
coord = coord_unmarshall_int(coord_int)
yield coord
