def __call__(self, coords): # this will produce the intersected list of coordinates with the toi, # all the way to low zoom level tiles intersected_coords, intersect_metrics, timing = \ self.toi_intersector(coords) low_zoom_coord_ints = set() grouped_by_zoom = defaultdict(list) for coord in intersected_coords: if self.group_by_zoom <= coord.zoom: parent = coord.zoomTo(self.group_by_zoom).container() parent_coord_int = coord_marshall_int(parent) grouped_by_zoom[parent_coord_int].append(coord) else: coord_int = coord_marshall_int(coord) low_zoom_coord_ints.add(coord_int) n_coords = 0 payloads = [] for _, coords in grouped_by_zoom.iteritems(): payload = self.msg_marshaller.marshall(coords) payloads.append(payload) n_coords += len(coords) # add all low zooms into a single payload low_zoom_coords = [] for coord_int in low_zoom_coord_ints: coord = coord_unmarshall_int(coord_int) low_zoom_coords.append(coord) if low_zoom_coords: low_zoom_payload = self.msg_marshaller.marshall(low_zoom_coords) payloads.append(low_zoom_payload) n_payloads = len(payloads) rawr_queue_batch_size = 10 n_msgs_sent = 0 for payloads_chunk in grouper(payloads, rawr_queue_batch_size): self.rawr_queue.send(payloads_chunk, self.logger) n_msgs_sent += 1 if self.logger: self.logger.info( 'Rawr tiles enqueued: ' 'coords(%d) payloads(%d) enqueue_calls(%d) ' 'toi(%d) hits(%d) misses(%d)' % (n_coords, n_payloads, n_msgs_sent, intersect_metrics['n_toi'], intersect_metrics['hits'], intersect_metrics['misses'])) self.stats_handler(n_coords, n_payloads, n_msgs_sent, intersect_metrics, timing)
def __call__(self, coords): # this will produce the intersected list of coordinates with the toi, # all the way to low zoom level tiles intersected_coords, intersect_metrics, timing = \ self.toi_intersector(coords) low_zoom_coord_ints = set() grouped_by_zoom = defaultdict(list) for coord in intersected_coords: if self.group_by_zoom <= coord.zoom: parent = coord.zoomTo(self.group_by_zoom).container() parent_coord_int = coord_marshall_int(parent) grouped_by_zoom[parent_coord_int].append(coord) else: coord_int = coord_marshall_int(coord) low_zoom_coord_ints.add(coord_int) n_coords = 0 payloads = [] for _, coords in grouped_by_zoom.iteritems(): payload = self.msg_marshaller.marshall(coords) payloads.append(payload) n_coords += len(coords) # add all low zooms into a single payload low_zoom_coords = [] for coord_int in low_zoom_coord_ints: coord = coord_unmarshall_int(coord_int) low_zoom_coords.append(coord) if low_zoom_coords: low_zoom_payload = self.msg_marshaller.marshall(low_zoom_coords) payloads.append(low_zoom_payload) n_payloads = len(payloads) rawr_queue_batch_size = 10 n_msgs_sent = 0 for payloads_chunk in grouper(payloads, rawr_queue_batch_size): self.rawr_queue.send(payloads_chunk, self.logger) n_msgs_sent += 1 if self.logger: self.logger.info( 'Rawr tiles enqueued: ' 'coords(%d) payloads(%d) enqueue_calls(%d) ' 'toi(%d) hits(%d) misses(%d)' % (n_coords, n_payloads, n_msgs_sent, intersect_metrics['n_toi'], intersect_metrics['hits'], intersect_metrics['misses'])) self.stats_handler(n_coords, n_payloads, n_msgs_sent, intersect_metrics, timing)
def read(self, max_to_read=10): with self.redis_client.pipeline() as pipe: pipe.lrange(self.queue_key, 0, max_to_read - 1) pipe.ltrim(self.queue_key, max_to_read, -1) coord_ints, _ = pipe.execute() if not coord_ints: time.sleep(self.sleep_time_seconds_when_empty) return [] coord_msgs = [] for coord_int in coord_ints: coord = coord_unmarshall_int(coord_int) coord_msg = CoordMessage(coord, None) coord_msgs.append(coord_msg) return coord_msgs
def test_roundtrip_serialization(self): from tilequeue.tile import coord_marshall_int from tilequeue.tile import coord_unmarshall_int from tilequeue.tile import seed_tiles from ModestMaps.Core import Coordinate from itertools import chain seed_coords = seed_tiles(0, 5) example_coords = [ Coordinate(zoom=20, column=1002463, row=312816), Coordinate(zoom=30, column=12345678, row=12345678), ] coords = chain(seed_coords, example_coords) for coord in coords: self.assertEquals(coord, coord_unmarshall_int(coord_marshall_int(coord)))
def tilequeue_dump_tiles_of_interest(cfg, peripherals): logger = make_logger(cfg, 'dump_tiles_of_interest') logger.info('Dumping tiles of interest') logger.info('Fetching tiles of interest ...') coords = peripherals.redis_cache_index.fetch_tiles_of_interest() n_toi = len(coords) logger.info('Fetching tiles of interest ... done') toi_filename = "toi.txt" logger.info('Writing %d tiles of interest to %s ...', n_toi, toi_filename) with open(toi_filename, "w") as f: for coord in coords: c = coord_unmarshall_int(coord) f.write("{}/{}/{}\n".format(c.zoom, c.column, c.row)) logger.info('Writing %d tiles of interest to %s ... done', n_toi, toi_filename)
def test_tilequeue_explode_and_intersect(self): from tilequeue.command import explode_and_intersect from tilequeue.tile import coord_marshall_int from tilequeue.tile import coord_unmarshall_int from ModestMaps.Core import Coordinate sample_coord = Coordinate(zoom=14, column=250, row=250) sample_coord_int = coord_marshall_int(sample_coord) tiles_of_interest = [sample_coord_int] for i in (10, 11, 12, 13): coord = sample_coord.zoomTo(i) coord_int = coord_marshall_int(coord) tiles_of_interest.append(coord_int) exploded = explode_and_intersect([sample_coord_int], tiles_of_interest, until=11) coord_ints = list(exploded) for coord_int in coord_ints: coord = coord_unmarshall_int(coord_int) self.failUnless(coord.zoom > 10) self.assertEqual(4, len(coord_ints))
def tilequeue_enqueue_tiles_of_interest(cfg, peripherals): logger = make_logger(cfg, 'enqueue_tiles_of_interest') logger.info('Enqueueing tiles of interest') sqs_queue = peripherals.queue logger.info('Fetching tiles of interest ...') tiles_of_interest = peripherals.redis_cache_index.fetch_tiles_of_interest() n_toi = len(tiles_of_interest) logger.info('Fetching tiles of interest ... done') coords = [] for coord_int in tiles_of_interest: coord = coord_unmarshall_int(coord_int) if coord.zoom <= 16: coords.append(coord) enqueuer = ThreadedEnqueuer(sqs_queue, cfg.seed_n_threads, logger) n_queued, n_in_flight = enqueuer(coords) logger.info('%d enqueued - %d in flight' % (n_queued, n_in_flight)) logger.info('%d tiles of interest processed' % n_toi)
def tilequeue_enqueue_tiles_of_interest(cfg, peripherals): logger = make_logger(cfg, 'enqueue_tiles_of_interest') logger.info('Enqueueing tiles of interest') sqs_queue = peripherals.queue logger.info('Fetching tiles of interest ...') tiles_of_interest = peripherals.redis_cache_index.fetch_tiles_of_interest() n_toi = len(tiles_of_interest) logger.info('Fetching tiles of interest ... done') coords = [] for coord_int in tiles_of_interest: coord = coord_unmarshall_int(coord_int) if coord.zoom <= 16: coords.append(coord) enqueuer = ThreadedEnqueuer(sqs_queue, cfg.seed_n_threads, logger) n_queued, n_in_flight = enqueuer(coords) logger.info('%d enqueued - %d in flight' % (n_queued, n_in_flight)) logger.info('%d tiles of interest processed' % n_toi)
def test_tilequeue_explode_and_intersect(self): from tilequeue.command import explode_and_intersect from tilequeue.tile import coord_marshall_int from tilequeue.tile import coord_unmarshall_int from ModestMaps.Core import Coordinate sample_coord = Coordinate(zoom=14, column=250, row=250) sample_coord_int = coord_marshall_int(sample_coord) tiles_of_interest = [sample_coord_int] for i in (10, 11, 12, 13): coord = sample_coord.zoomTo(i) coord_int = coord_marshall_int(coord) tiles_of_interest.append(coord_int) exploded, metrics = explode_and_intersect( [sample_coord_int], tiles_of_interest, until=11) coord_ints = list(exploded) for coord_int in coord_ints: coord = coord_unmarshall_int(coord_int) self.failUnless(coord.zoom > 10) self.assertEqual(4, len(coord_ints)) self.assertEqual(4, metrics['hits']) self.assertEqual(0, metrics['misses']) self.assertEqual(4, metrics['total'])
def save_set_to_fp(the_set, fp): for coord_int in sorted(the_set): coord = coord_unmarshall_int(coord_int) fp.write(serialize_coord(coord)) fp.write('\n')
def find_intersection(self, diff_set_key): intersection = self.redis_client.sinter(self.cache_set_key, diff_set_key) for coord_int in intersection: coord = coord_unmarshall_int(coord_int) yield coord