def test_read_multiple(self): from mock import MagicMock from tilequeue.tile import deserialize_coord from tilequeue.tile import serialize_coord coord1 = deserialize_coord('1/1/1') coord2 = deserialize_coord('2/2/2') sqs_queue1 = MagicMock() sqs_queue2 = MagicMock() sqs_queue1.name = 'q1' sqs_queue1.get_messages = self._make_get_messages( [serialize_coord(coord1)]) sqs_queue2.name = 'q2' sqs_queue2.get_messages = self._make_get_messages( [serialize_coord(coord2)]) msq = self._make_one( [sqs_queue1, sqs_queue2], _test_q1_q2_get_queue_name_for_zoom, ) coord_msgs = msq.read() self.assertEqual(2, len(coord_msgs)) self.assertEqual(coord1, coord_msgs[0].coord) self.assertEqual(coord2, coord_msgs[1].coord)
def test_write_coords(self): from tilequeue.tile import deserialize_coord coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')] queue_writer = self.make_queue_writer() n_enqueued, n_inflight = queue_writer.enqueue_batch(coords) self.assertEquals(2, n_enqueued) self.assertEquals(0, n_inflight)
def test_job_done_multiple(self): from mock import MagicMock from tilequeue.tile import CoordMessage from tilequeue.tile import deserialize_coord coord1 = deserialize_coord('1/1/1') sqs_queue1 = MagicMock() sqs_queue1.name = 'q1' sqs_queue1.delete_message = self._mock_delete_message coord2 = deserialize_coord('2/2/2') sqs_queue2 = MagicMock() sqs_queue2.name = 'q2' sqs_queue2.delete_message = self._mock_delete_message msq = self._make_one( [sqs_queue1, sqs_queue2], _test_q1_q2_get_queue_name_for_zoom, ) cm1 = CoordMessage(coord1, 'msg_handle1', metadata=dict(queue_name=sqs_queue1.name)) msq.job_done(cm1) self.assertIsNotNone(self.mock_handle) self.assertEqual(self.mock_handle, 'msg_handle1') self.mock_handle = None cm2 = CoordMessage(coord2, 'msg_handle2', metadata=dict(queue_name=sqs_queue2.name)) msq.job_done(cm2) self.assertIsNotNone(self.mock_handle) self.assertEqual(self.mock_handle, 'msg_handle2') self.mock_handle = None
def __init__(self, filename, msg_marshaller): self.queue = [] with open(filename, 'r') as fh: for line in fh: coord = deserialize_coord(line) payload = msg_marshaller.marshall([coord]) self.queue.append(payload)
def missing_tiles(missing_bucket, rawr_bucket, date_prefix, region, key_format_type, config, group_by_zoom, tile_coords_generator, tile_verifier): from make_meta_tiles import MissingTileFinder if bool(tile_coords_generator): return { c for c in tile_coords_generator.generate_tiles_coordinates( [group_by_zoom]) } else: present = set() finder = MissingTileFinder(missing_bucket, rawr_bucket, date_prefix, date_prefix, region, key_format_type, config, group_by_zoom, tile_coords_generator, tile_verifier) with finder.present_tiles() as present_file: with open(present_file) as fh: for line in fh: coord = deserialize_coord(line) if coord.zoom == group_by_zoom: present.add(coord) missing = set(all_tiles_at(group_by_zoom)) - set(present) return missing
def _call_fut(self, shape, bounds): from tilequeue.process import Context from tilequeue.tile import deserialize_coord from vectordatasource.transform import rank_features props = dict(foo='bar') feature = shape, props, 1 feature_layer = dict( features=[feature], layer_datum=dict(name='layer-name'), ) params = dict( source_layer='layer-name', rank_key='rank', items_matching=dict(foo='bar'), ) ctx = Context( feature_layers=[feature_layer], tile_coord=deserialize_coord('0/0/0'), unpadded_bounds=bounds, params=params, resources=None, ) rank_features(ctx) rank = props.get('rank') return rank
def test_track_and_done_not_including_parent(self): from tilequeue.tile import deserialize_coord from tilequeue.queue.message import QueueHandle queue_id = 1 queue_handle = QueueHandle(queue_id, 'handle') coords = map(deserialize_coord, ('2/2/3', '2/2/2')) parent_tile = deserialize_coord('1/1/1') self._assert_track_done(coords, queue_handle, parent_tile)
def test_group_coord_out_of_range(self): from tilequeue.tile import deserialize_coord specs = ( (0, 10, 'tile_queue_1', object(), None), (10, 16, 'tile_queue_2', object(), 10), ) qm = self.make_queue_mapper(specs) coords = [deserialize_coord('20/0/0')] coord_groups = list(qm.group(coords)) self.assertEquals(0, len(coord_groups)) coords = map(deserialize_coord, ['20/0/0', '1/1/1', '16/0/0']) coord_groups = list(qm.group(coords)) self.assertEquals(1, len(coord_groups)) self.assertEquals([deserialize_coord('1/1/1')], coord_groups[0].coords) self.assertEquals(0, coord_groups[0].queue_id)
def load_set_from_fp(fp): toi_set = set() for coord_str in fp: coord = deserialize_coord(coord_str) coord_int = coord_marshall_int(coord) toi_set.add(coord_int) return toi_set
def unmarshall(self, payload): coord_strs = payload.split(',') coords = [] for coord_str in coord_strs: coord_str = coord_str.strip() if coord_str: coord = deserialize_coord(coord_str) assert coord coords.append(coord) return coords
def test_enqueue(self): from mock import MagicMock from tilequeue.tile import deserialize_coord sqs_queue = MagicMock() sqs_queue.name = 'q1' msq = self._make_one([sqs_queue]) sqs_queue.write = self._mock_write msq.enqueue(deserialize_coord('1/1/1')) self.assertIsNotNone(self.mock_written) self.assertEqual(self.mock_written.get_body(), '1/1/1')
def test_group_coords(self): from tilequeue.tile import deserialize_coord specs = ( (0, 10, 'tile_queue_1', object(), None), (10, 16, 'tile_queue_2', object(), 10), ) qm = self.make_queue_mapper(specs) coord_strs = ( '1/1/1', '9/0/0', '10/0/0', '14/65536/65536', '15/0/0', ) coords = map(deserialize_coord, coord_strs) coord_groups = list(qm.group(coords)) assert len(coord_groups) == 4 cg1, cg2, cg3, cg4 = coord_groups lo_zoom_queue_id = 0 hi_zoom_queue_id = 1 # low zooms are grouped separately self.assertEquals(1, len(cg1.coords)) self.assertEquals([deserialize_coord('1/1/1')], cg1.coords) self.assertEquals(lo_zoom_queue_id, cg1.queue_id) self.assertEquals(1, len(cg2.coords)) self.assertEquals([deserialize_coord('9/0/0')], cg2.coords) self.assertEquals(lo_zoom_queue_id, cg2.queue_id) # common z10 parents are grouped together self.assertEquals(2, len(cg3.coords)) self.assertEquals(map(deserialize_coord, ['10/0/0', '15/0/0']), cg3.coords) self.assertEquals(hi_zoom_queue_id, cg3.queue_id) # different z10 parent grouped separately, even though it # should be sent to the same queue self.assertEquals(1, len(cg4.coords)) self.assertEquals([deserialize_coord('14/65536/65536')], cg4.coords) self.assertEquals(hi_zoom_queue_id, cg4.queue_id)
def test_track_and_done_asserts_on_duplicates(self): from tilequeue.tile import deserialize_coord from tilequeue.queue.message import QueueHandle queue_id = 1 queue_handle = QueueHandle(queue_id, 'handle') coords = map(deserialize_coord, ('2/2/2', '2/2/2')) parent_tile = deserialize_coord('1/1/1') with self.assertRaises(AssertionError): self.tracker.track(queue_handle, coords, parent_tile)
def test_example_coord(self): from tilequeue.tile import deserialize_coord from tilequeue.format import json_format from tilequeue.store import KeyFormatType from tilequeue.store import S3TileKeyGenerator coord = deserialize_coord('8/72/105') prefix = '20160121' tile_key_gen = S3TileKeyGenerator( key_format_type=KeyFormatType.hash_prefix) tile_key = tile_key_gen(prefix, coord, json_format.extension) self.assertEqual(tile_key, 'b57e9/20160121/8/72/105.json')
def read(self, max_to_read=1, timeout_seconds=20): with self.lock: coords = [] for _ in range(max_to_read): coord = self.fp.readline() if coord: coords.append(CoordMessage(deserialize_coord(coord), None)) else: break return coords
def test_s3_path(self): from tilequeue.tile import deserialize_coord from tilequeue.store import KeyFormatType from tilequeue.store import S3TileKeyGenerator coord = deserialize_coord('10/1/2') prefix = '19851026' extension = 'zip' tile_key_gen = S3TileKeyGenerator( key_format_type=KeyFormatType.hash_prefix) key = tile_key_gen(prefix, coord, extension) self.assertEqual('c35b6/19851026/10/1/2.zip', key)
def test_no_path(self): from tilequeue.store import s3_tile_key from tilequeue.tile import deserialize_coord from tilequeue.format import json_format coord = deserialize_coord('8/72/105') date_str = '20160121' path = '' layer = 'all' tile_key = s3_tile_key(date_str, path, layer, coord, json_format.extension) self.assertEqual(tile_key, '20160121/cfc61/all/8/72/105.json')
def test_compare_z10_bounds(self): from tilequeue.tile import coord_to_mercator_bounds from tilequeue.tile import deserialize_coord coord = deserialize_coord('10/100/100') merc_bounds = coord_to_mercator_bounds(coord) tile_meters_wide = merc_bounds[2] - merc_bounds[0] exp_meters_per_pixel_dim = tile_meters_wide / 256 act_meters_per_pixel_dim = self._call_fut(10) self.assertAlmostEquals( exp_meters_per_pixel_dim, act_meters_per_pixel_dim, places=0)
def test_queue_mappings(self): q1 = object() q2 = object() q3 = object() specs = ( (None, None, 'tile_queue_1', q1, None), (0, 10, 'tile_queue_2', q2, None), (10, 16, 'tile_queue_3', q3, 10), ) qm = self.make_queue_mapper(specs) q1_id, q2_id, q3_id = range(3) self.assertIs(q1, qm.get_queue(q1_id)) self.assertIs(q2, qm.get_queue(q2_id)) self.assertIs(q3, qm.get_queue(q3_id)) ordered_queue_result = list(qm.queues_in_priority_order()) self.assertEquals(3, len(ordered_queue_result)) r1_id, r1_q = ordered_queue_result[0] r2_id, r2_q = ordered_queue_result[1] r3_id, r3_q = ordered_queue_result[2] self.assertIs(q1, r1_q) self.assertEquals(q1_id, r1_id) self.assertIs(q2, r2_q) self.assertEquals(q2_id, r2_id) self.assertIs(q3, r3_q) self.assertEquals(q3_id, r3_id) from tilequeue.tile import deserialize_coord # verify that the queue ids line up with those that have zooms # specified coord_groups = list(qm.group([deserialize_coord('5/0/0')])) self.assertEquals(1, len(coord_groups)) self.assertEquals(1, coord_groups[0].queue_id) coord_groups = list(qm.group([deserialize_coord('15/0/0')])) self.assertEquals(1, len(coord_groups)) self.assertEquals(2, coord_groups[0].queue_id)
def test_compare_z10_bounds(self): from tilequeue.tile import coord_to_mercator_bounds from tilequeue.tile import deserialize_coord coord = deserialize_coord('10/100/100') merc_bounds = coord_to_mercator_bounds(coord) tile_meters_wide = merc_bounds[2] - merc_bounds[0] exp_meters_per_pixel_dim = tile_meters_wide / 256 act_meters_per_pixel_dim = self._call_fut(10) self.assertAlmostEquals(exp_meters_per_pixel_dim, act_meters_per_pixel_dim, places=0)
def test_read_single(self): from mock import MagicMock from tilequeue.tile import deserialize_coord from tilequeue.tile import serialize_coord sqs_queue = MagicMock() coord = deserialize_coord('1/1/1') sqs_queue.get_messages = self._make_get_messages( [serialize_coord(coord)]) msq = self._make_one([sqs_queue]) coord_msgs = msq.read() self.assertEqual(1, len(coord_msgs)) self.assertEqual(coord, coord_msgs[0].coord)
def test_job_done_no_queue_name(self): from mock import MagicMock from tilequeue.tile import CoordMessage from tilequeue.tile import deserialize_coord coord = deserialize_coord('1/1/1') sqs_queue = MagicMock() sqs_queue.name = 'q1' sqs_queue.delete_message = self._mock_delete_message cm = CoordMessage(coord, 'msg_handle') msq = self._make_one([sqs_queue]) with self.assertRaises(AssertionError): msq.job_done(cm)
def read(self, max_to_read=1): coord_messages = [] messages = self.sqs_queue.get_messages(num_messages=max_to_read, attributes=["SentTimestamp"]) for message in messages: data = message.get_body() coord = deserialize_coord(data) if coord is None: # log? continue timestamp = float(message.attributes.get('SentTimestamp')) coord_message = CoordMessage(coord, message, timestamp) coord_messages.append(coord_message) return coord_messages
def test_single_queue_mapper(self): from mock import MagicMock from tilequeue.tile import deserialize_coord tile_queue_mock = MagicMock() queue_mapper = self.make_queue_mapper('queue_name', tile_queue_mock) coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')] coord_groups = list(queue_mapper.group(coords)) self.assertEquals(2, len(coord_groups)) cg1, cg2 = coord_groups self.assertEquals('queue_name', cg1.queue_id) self.assertEquals('queue_name', cg2.queue_id) self.assertEquals(1, len(cg1.coords)) self.assertEquals(coords[0], cg1.coords[0]) self.assertEquals(1, len(cg2.coords)) self.assertEquals(coords[1], cg2.coords[0]) self.assertIs(tile_queue_mock, queue_mapper.get_queue('queue_name')) qs = queue_mapper.queues_in_priority_order() self.assertEquals(1, len(qs)) qn, q = qs[0] self.assertEquals('queue_name', qn) self.assertIs(tile_queue_mock, q)
def test_job_done_single(self): from mock import MagicMock from tilequeue.tile import CoordMessage from tilequeue.tile import deserialize_coord coord = deserialize_coord('1/1/1') sqs_queue = MagicMock() sqs_queue.name = 'q1' sqs_queue.delete_message = self._mock_delete_message cm = CoordMessage(coord, 'msg_handle', dict(queue_name=sqs_queue.name)) msq = self._make_one([sqs_queue]) msq.job_done(cm) self.assertIsNotNone(self.mock_handle) self.assertEqual(self.mock_handle, 'msg_handle')
def test_enqueue_batch(self): from mock import MagicMock from tilequeue.tile import deserialize_coord sqs_queue1 = MagicMock() sqs_queue1.name = 'q1' sqs_queue1.write_batch = self._mock_write_batch sqs_queue2 = MagicMock() sqs_queue2.name = 'q2' sqs_queue2.write_batch = self._mock_write_batch msq = self._make_one([sqs_queue1, sqs_queue2]) msq.enqueue_batch([ deserialize_coord('1/1/1'), deserialize_coord('1/0/0') ]) self.assertIsNotNone(self.mock_write_batch) self.assertEqual(2, len(self.mock_write_batch)) coord_str1, coord_str2 = [x[1] for x in self.mock_write_batch] self.assertEqual(coord_str1, '1/1/1') self.assertEqual(coord_str2, '1/0/0')
def test_track_and_done(self): from tilequeue.tile import deserialize_coord from tilequeue.queue.message import QueueHandle queue_id = 1 queue_handle = QueueHandle(queue_id, 'handle') coords = [deserialize_coord('1/1/1')] coord_handles = self.tracker.track(queue_handle, coords) self.assertEqual(1, len(coord_handles)) coord_handle = coord_handles[0] self.assertIs(queue_handle, coord_handle) track_result = self.tracker.done(coord_handle) self.assertIs(queue_handle, track_result.queue_handle) self.assertTrue(track_result.all_done) self.assertFalse(track_result.parent_tile)
def _assert_no_id_in_props(self, features, merge_fn): from tilequeue.process import Context from tilequeue.tile import deserialize_coord layer_name = 'layername' feature_layer = dict( features=features, layer_datum=dict(name=layer_name), ) feature_layers = [feature_layer] ctx = Context(feature_layers=feature_layers, tile_coord=deserialize_coord('0/0/0'), unpadded_bounds=None, params=dict(source_layer=layer_name), resources=None) merged_feature_layer = merge_fn(ctx) merged_features = merged_feature_layer['features'] self.assertEquals(1, len(merged_features)) merged_feature = merged_features[0] props = merged_feature[1] self.assertTrue('id' not in props)
def test_tags(self): from tilequeue.store import KeyFormatType from tilequeue.store import S3 from tilequeue.store import S3TileKeyGenerator s3_client = self._make_stub_s3_client() tags = None tile_key_gen = S3TileKeyGenerator( key_format_type=KeyFormatType.hash_prefix) store = S3(s3_client, 'bucket', 'prefix', False, 60, None, 'public-read', tags, tile_key_gen) tile_data = 'data' from tilequeue.tile import deserialize_coord coord = deserialize_coord('14/1/2') from tilequeue.format import mvt_format store.write_tile(tile_data, coord, mvt_format) self.assertIsNone(store.s3_client.put_props.get('Tagging')) store.tags = dict(prefix='foo', run_id='bar') store.write_tile(tile_data, coord, mvt_format) self.assertEquals('prefix=foo&run_id=bar', store.s3_client.put_props.get('Tagging'))
def _call_fut(self, building_shapes, building_part_shapes): from tilequeue.tile import deserialize_coord from tilequeue.process import Context building_features = [] building_id = 1 for building_shape in building_shapes: building_props = dict( id=building_id, kind='building', ) building_feature = building_shape, building_props, building_id building_features.append(building_feature) building_id += 1 part_features = [] building_part_id = building_id for part_shape in building_part_shapes: part_props = dict( id=building_part_id, kind='building_part', ) part_feature = part_shape, part_props, building_part_id part_features.append(part_feature) building_part_id += 1 building_features = building_features + part_features building_feature_layer = dict( features=building_features, layer_datum=dict(name='buildings'), ) feature_layers = [building_feature_layer] ctx = Context(feature_layers=feature_layers, tile_coord=deserialize_coord('0/0/0'), unpadded_bounds=None, params=dict(source_layer='buildings'), resources=None) from vectordatasource.transform import buildings_unify buildings_unify(ctx) return building_feature_layer['features']
def test_no_merge_preserve_props(self): import shapely.geometry from tilequeue.process import Context from tilequeue.tile import deserialize_coord from vectordatasource.transform import merge_polygon_features buildings = [] for i in (0, 100): id = i + 1 points = [ (1 + i, 1 + i), (10 + i, 10 + i), (1 + i, 10 + i), (1 + i, 1 + i), ] shape = shapely.geometry.Polygon(points) props = dict( id=id, kind='building', unique_value='value-%d' % id, ) building = shape, props, id buildings.append(building) layer_name = 'buildings' feature_layer = dict( features=buildings, layer_datum=dict(name=layer_name), ) feature_layers = [feature_layer] ctx = Context(feature_layers=feature_layers, tile_coord=deserialize_coord('0/0/0'), unpadded_bounds=None, params=dict(source_layer=layer_name), resources=None) merged_feature_layer = merge_polygon_features(ctx) merged_features = merged_feature_layer['features'] self.assertEquals(2, len(merged_features)) for f in merged_features: props = f[1] self.assertTrue('id' in props)
def test_marshall_single_coord(self): from tilequeue.tile import deserialize_coord result = self.msg_marshaller.marshall([deserialize_coord('1/1/1')]) self.assertEqual('1/1/1', result)
def test_unmarshall_multiple(self): from tilequeue.tile import deserialize_coord actual = self.msg_marshaller.unmarshall('1/1/1,2/2/2') self.assertEqual(2, len(actual)) self.assertEqual(actual[0], deserialize_coord('1/1/1')) self.assertEqual(actual[1], deserialize_coord('2/2/2'))
from boto import connect_s3 from boto.s3.bucket import Bucket from tilequeue.tile import deserialize_coord from tilequeue.tile import serialize_coord import fileinput bucket_name = 'mapzen-tiles-assets' key_name = 'test/integration-test-coords.txt' coords_to_store = set() for line in fileinput.input(): coord = deserialize_coord(line.strip()) assert coord if coord.zoom == 0: uplifted_coord = coord else: uplifted_coord = coord.zoomBy(-1).container() coords_to_store.add(uplifted_coord) sorted_coords = sorted(coords_to_store) coords_str = '\n'.join(map(serialize_coord, sorted_coords)) conn = connect_s3() bucket = Bucket(conn, bucket_name) key = bucket.new_key(key_name) key.set_contents_from_string( coords_str, headers={'Content-Type': 'text/plain'}, )