def test_read_multiple(self):
from mock import MagicMock
from tilequeue.tile import deserialize_coord
from tilequeue.tile import serialize_coord
coord1 = deserialize_coord('1/1/1')
coord2 = deserialize_coord('2/2/2')
sqs_queue1 = MagicMock()
sqs_queue2 = MagicMock()
sqs_queue1.name = 'q1'
sqs_queue1.get_messages = self._make_get_messages(
[serialize_coord(coord1)])
sqs_queue2.name = 'q2'
sqs_queue2.get_messages = self._make_get_messages(
[serialize_coord(coord2)])
msq = self._make_one(
[sqs_queue1, sqs_queue2],
_test_q1_q2_get_queue_name_for_zoom,
)
coord_msgs = msq.read()
self.assertEqual(2, len(coord_msgs))
self.assertEqual(coord1, coord_msgs[0].coord)
self.assertEqual(coord2, coord_msgs[1].coord)
def test_write_coords(self):
from tilequeue.tile import deserialize_coord
coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')]
queue_writer = self.make_queue_writer()
n_enqueued, n_inflight = queue_writer.enqueue_batch(coords)
self.assertEquals(2, n_enqueued)
self.assertEquals(0, n_inflight)
def test_job_done_multiple(self):
from mock import MagicMock
from tilequeue.tile import CoordMessage
from tilequeue.tile import deserialize_coord
coord1 = deserialize_coord('1/1/1')
sqs_queue1 = MagicMock()
sqs_queue1.name = 'q1'
sqs_queue1.delete_message = self._mock_delete_message
coord2 = deserialize_coord('2/2/2')
sqs_queue2 = MagicMock()
sqs_queue2.name = 'q2'
sqs_queue2.delete_message = self._mock_delete_message
msq = self._make_one(
[sqs_queue1, sqs_queue2],
_test_q1_q2_get_queue_name_for_zoom,
)
cm1 = CoordMessage(coord1, 'msg_handle1',
metadata=dict(queue_name=sqs_queue1.name))
msq.job_done(cm1)
self.assertIsNotNone(self.mock_handle)
self.assertEqual(self.mock_handle, 'msg_handle1')
self.mock_handle = None
cm2 = CoordMessage(coord2, 'msg_handle2',
metadata=dict(queue_name=sqs_queue2.name))
msq.job_done(cm2)
self.assertIsNotNone(self.mock_handle)
self.assertEqual(self.mock_handle, 'msg_handle2')
self.mock_handle = None
def __init__(self, filename, msg_marshaller):
self.queue = []
with open(filename, 'r') as fh:
for line in fh:
coord = deserialize_coord(line)
payload = msg_marshaller.marshall([coord])
self.queue.append(payload)
def missing_tiles(missing_bucket, rawr_bucket, date_prefix, region,
key_format_type, config, group_by_zoom,
tile_coords_generator, tile_verifier):
from make_meta_tiles import MissingTileFinder
if bool(tile_coords_generator):
return {
c
for c in tile_coords_generator.generate_tiles_coordinates(
[group_by_zoom])
}
else:
present = set()
finder = MissingTileFinder(missing_bucket, rawr_bucket, date_prefix,
date_prefix, region, key_format_type,
config, group_by_zoom,
tile_coords_generator, tile_verifier)
with finder.present_tiles() as present_file:
with open(present_file) as fh:
for line in fh:
coord = deserialize_coord(line)
if coord.zoom == group_by_zoom:
present.add(coord)
missing = set(all_tiles_at(group_by_zoom)) - set(present)
return missing
def _call_fut(self, shape, bounds):
from tilequeue.process import Context
from tilequeue.tile import deserialize_coord
from vectordatasource.transform import rank_features
props = dict(foo='bar')
feature = shape, props, 1
feature_layer = dict(
features=[feature],
layer_datum=dict(name='layer-name'),
)
params = dict(
source_layer='layer-name',
rank_key='rank',
items_matching=dict(foo='bar'),
)
ctx = Context(
feature_layers=[feature_layer],
tile_coord=deserialize_coord('0/0/0'),
unpadded_bounds=bounds,
params=params,
resources=None,
)
rank_features(ctx)
rank = props.get('rank')
return rank
def __init__(self, filename, msg_marshaller):
self.queue = []
with open(filename, 'r') as fh:
for line in fh:
coord = deserialize_coord(line)
payload = msg_marshaller.marshall([coord])
self.queue.append(payload)
def test_track_and_done_not_including_parent(self):
from tilequeue.tile import deserialize_coord
from tilequeue.queue.message import QueueHandle
queue_id = 1
queue_handle = QueueHandle(queue_id, 'handle')
coords = map(deserialize_coord, ('2/2/3', '2/2/2'))
parent_tile = deserialize_coord('1/1/1')
self._assert_track_done(coords, queue_handle, parent_tile)
def test_group_coord_out_of_range(self):
from tilequeue.tile import deserialize_coord
specs = (
(0, 10, 'tile_queue_1', object(), None),
(10, 16, 'tile_queue_2', object(), 10),
)
qm = self.make_queue_mapper(specs)
coords = [deserialize_coord('20/0/0')]
coord_groups = list(qm.group(coords))
self.assertEquals(0, len(coord_groups))
coords = map(deserialize_coord, ['20/0/0', '1/1/1', '16/0/0'])
coord_groups = list(qm.group(coords))
self.assertEquals(1, len(coord_groups))
self.assertEquals([deserialize_coord('1/1/1')], coord_groups[0].coords)
self.assertEquals(0, coord_groups[0].queue_id)
def test_track_and_done_not_including_parent(self):
from tilequeue.tile import deserialize_coord
from tilequeue.queue.message import QueueHandle
queue_id = 1
queue_handle = QueueHandle(queue_id, 'handle')
coords = map(deserialize_coord, ('2/2/3', '2/2/2'))
parent_tile = deserialize_coord('1/1/1')
self._assert_track_done(coords, queue_handle, parent_tile)
def test_group_coord_out_of_range(self):
from tilequeue.tile import deserialize_coord
specs = (
(0, 10, 'tile_queue_1', object(), None),
(10, 16, 'tile_queue_2', object(), 10),
)
qm = self.make_queue_mapper(specs)
coords = [deserialize_coord('20/0/0')]
coord_groups = list(qm.group(coords))
self.assertEquals(0, len(coord_groups))
coords = map(deserialize_coord, ['20/0/0', '1/1/1', '16/0/0'])
coord_groups = list(qm.group(coords))
self.assertEquals(1, len(coord_groups))
self.assertEquals([deserialize_coord('1/1/1')], coord_groups[0].coords)
self.assertEquals(0, coord_groups[0].queue_id)
def load_set_from_fp(fp):
toi_set = set()
for coord_str in fp:
coord = deserialize_coord(coord_str)
coord_int = coord_marshall_int(coord)
toi_set.add(coord_int)
return toi_set
def unmarshall(self, payload):
coord_strs = payload.split(',')
coords = []
for coord_str in coord_strs:
coord_str = coord_str.strip()
if coord_str:
coord = deserialize_coord(coord_str)
assert coord
coords.append(coord)
return coords
def test_enqueue(self):
from mock import MagicMock
from tilequeue.tile import deserialize_coord
sqs_queue = MagicMock()
sqs_queue.name = 'q1'
msq = self._make_one([sqs_queue])
sqs_queue.write = self._mock_write
msq.enqueue(deserialize_coord('1/1/1'))
self.assertIsNotNone(self.mock_written)
self.assertEqual(self.mock_written.get_body(), '1/1/1')
def test_group_coords(self):
from tilequeue.tile import deserialize_coord
specs = (
(0, 10, 'tile_queue_1', object(), None),
(10, 16, 'tile_queue_2', object(), 10),
)
qm = self.make_queue_mapper(specs)
coord_strs = (
'1/1/1',
'9/0/0',
'10/0/0',
'14/65536/65536',
'15/0/0',
)
coords = map(deserialize_coord, coord_strs)
coord_groups = list(qm.group(coords))
assert len(coord_groups) == 4
cg1, cg2, cg3, cg4 = coord_groups
lo_zoom_queue_id = 0
hi_zoom_queue_id = 1
# low zooms are grouped separately
self.assertEquals(1, len(cg1.coords))
self.assertEquals([deserialize_coord('1/1/1')], cg1.coords)
self.assertEquals(lo_zoom_queue_id, cg1.queue_id)
self.assertEquals(1, len(cg2.coords))
self.assertEquals([deserialize_coord('9/0/0')], cg2.coords)
self.assertEquals(lo_zoom_queue_id, cg2.queue_id)
# common z10 parents are grouped together
self.assertEquals(2, len(cg3.coords))
self.assertEquals(map(deserialize_coord, ['10/0/0', '15/0/0']),
cg3.coords)
self.assertEquals(hi_zoom_queue_id, cg3.queue_id)
# different z10 parent grouped separately, even though it
# should be sent to the same queue
self.assertEquals(1, len(cg4.coords))
self.assertEquals([deserialize_coord('14/65536/65536')], cg4.coords)
self.assertEquals(hi_zoom_queue_id, cg4.queue_id)
def test_track_and_done_asserts_on_duplicates(self):
from tilequeue.tile import deserialize_coord
from tilequeue.queue.message import QueueHandle
queue_id = 1
queue_handle = QueueHandle(queue_id, 'handle')
coords = map(deserialize_coord, ('2/2/2', '2/2/2'))
parent_tile = deserialize_coord('1/1/1')
with self.assertRaises(AssertionError):
self.tracker.track(queue_handle, coords, parent_tile)
def test_track_and_done_asserts_on_duplicates(self):
from tilequeue.tile import deserialize_coord
from tilequeue.queue.message import QueueHandle
queue_id = 1
queue_handle = QueueHandle(queue_id, 'handle')
coords = map(deserialize_coord, ('2/2/2', '2/2/2'))
parent_tile = deserialize_coord('1/1/1')
with self.assertRaises(AssertionError):
self.tracker.track(queue_handle, coords, parent_tile)
def test_group_coords(self):
from tilequeue.tile import deserialize_coord
specs = (
(0, 10, 'tile_queue_1', object(), None),
(10, 16, 'tile_queue_2', object(), 10),
)
qm = self.make_queue_mapper(specs)
coord_strs = (
'1/1/1',
'9/0/0',
'10/0/0',
'14/65536/65536',
'15/0/0',
)
coords = map(deserialize_coord, coord_strs)
coord_groups = list(qm.group(coords))
assert len(coord_groups) == 4
cg1, cg2, cg3, cg4 = coord_groups
lo_zoom_queue_id = 0
hi_zoom_queue_id = 1
# low zooms are grouped separately
self.assertEquals(1, len(cg1.coords))
self.assertEquals([deserialize_coord('1/1/1')], cg1.coords)
self.assertEquals(lo_zoom_queue_id, cg1.queue_id)
self.assertEquals(1, len(cg2.coords))
self.assertEquals([deserialize_coord('9/0/0')], cg2.coords)
self.assertEquals(lo_zoom_queue_id, cg2.queue_id)
# common z10 parents are grouped together
self.assertEquals(2, len(cg3.coords))
self.assertEquals(map(deserialize_coord, ['10/0/0', '15/0/0']),
cg3.coords)
self.assertEquals(hi_zoom_queue_id, cg3.queue_id)
# different z10 parent grouped separately, even though it
# should be sent to the same queue
self.assertEquals(1, len(cg4.coords))
self.assertEquals([deserialize_coord('14/65536/65536')], cg4.coords)
self.assertEquals(hi_zoom_queue_id, cg4.queue_id)
def test_example_coord(self):
from tilequeue.tile import deserialize_coord
from tilequeue.format import json_format
from tilequeue.store import KeyFormatType
from tilequeue.store import S3TileKeyGenerator
coord = deserialize_coord('8/72/105')
prefix = '20160121'
tile_key_gen = S3TileKeyGenerator(
key_format_type=KeyFormatType.hash_prefix)
tile_key = tile_key_gen(prefix, coord, json_format.extension)
self.assertEqual(tile_key, 'b57e9/20160121/8/72/105.json')
def read(self, max_to_read=1, timeout_seconds=20):
with self.lock:
coords = []
for _ in range(max_to_read):
coord = self.fp.readline()
if coord:
coords.append(CoordMessage(deserialize_coord(coord), None))
else:
break
return coords
def test_example_coord(self):
from tilequeue.tile import deserialize_coord
from tilequeue.format import json_format
from tilequeue.store import KeyFormatType
from tilequeue.store import S3TileKeyGenerator
coord = deserialize_coord('8/72/105')
prefix = '20160121'
tile_key_gen = S3TileKeyGenerator(
key_format_type=KeyFormatType.hash_prefix)
tile_key = tile_key_gen(prefix, coord, json_format.extension)
self.assertEqual(tile_key, 'b57e9/20160121/8/72/105.json')
def test_s3_path(self):
from tilequeue.tile import deserialize_coord
from tilequeue.store import KeyFormatType
from tilequeue.store import S3TileKeyGenerator
coord = deserialize_coord('10/1/2')
prefix = '19851026'
extension = 'zip'
tile_key_gen = S3TileKeyGenerator(
key_format_type=KeyFormatType.hash_prefix)
key = tile_key_gen(prefix, coord, extension)
self.assertEqual('c35b6/19851026/10/1/2.zip', key)
def test_no_path(self):
from tilequeue.store import s3_tile_key
from tilequeue.tile import deserialize_coord
from tilequeue.format import json_format
coord = deserialize_coord('8/72/105')
date_str = '20160121'
path = ''
layer = 'all'
tile_key = s3_tile_key(date_str, path, layer, coord,
json_format.extension)
self.assertEqual(tile_key, '20160121/cfc61/all/8/72/105.json')
def read(self, max_to_read=1, timeout_seconds=20):
with self.lock:
coords = []
for _ in range(max_to_read):
coord = self.fp.readline()
if coord:
coords.append(CoordMessage(deserialize_coord(coord), None))
else:
break
return coords
def test_compare_z10_bounds(self):
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('10/100/100')
merc_bounds = coord_to_mercator_bounds(coord)
tile_meters_wide = merc_bounds[2] - merc_bounds[0]
exp_meters_per_pixel_dim = tile_meters_wide / 256
act_meters_per_pixel_dim = self._call_fut(10)
self.assertAlmostEquals(
exp_meters_per_pixel_dim, act_meters_per_pixel_dim, places=0)
def test_queue_mappings(self):
q1 = object()
q2 = object()
q3 = object()
specs = (
(None, None, 'tile_queue_1', q1, None),
(0, 10, 'tile_queue_2', q2, None),
(10, 16, 'tile_queue_3', q3, 10),
)
qm = self.make_queue_mapper(specs)
q1_id, q2_id, q3_id = range(3)
self.assertIs(q1, qm.get_queue(q1_id))
self.assertIs(q2, qm.get_queue(q2_id))
self.assertIs(q3, qm.get_queue(q3_id))
ordered_queue_result = list(qm.queues_in_priority_order())
self.assertEquals(3, len(ordered_queue_result))
r1_id, r1_q = ordered_queue_result[0]
r2_id, r2_q = ordered_queue_result[1]
r3_id, r3_q = ordered_queue_result[2]
self.assertIs(q1, r1_q)
self.assertEquals(q1_id, r1_id)
self.assertIs(q2, r2_q)
self.assertEquals(q2_id, r2_id)
self.assertIs(q3, r3_q)
self.assertEquals(q3_id, r3_id)
from tilequeue.tile import deserialize_coord
# verify that the queue ids line up with those that have zooms
# specified
coord_groups = list(qm.group([deserialize_coord('5/0/0')]))
self.assertEquals(1, len(coord_groups))
self.assertEquals(1, coord_groups[0].queue_id)
coord_groups = list(qm.group([deserialize_coord('15/0/0')]))
self.assertEquals(1, len(coord_groups))
self.assertEquals(2, coord_groups[0].queue_id)
def test_compare_z10_bounds(self):
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('10/100/100')
merc_bounds = coord_to_mercator_bounds(coord)
tile_meters_wide = merc_bounds[2] - merc_bounds[0]
exp_meters_per_pixel_dim = tile_meters_wide / 256
act_meters_per_pixel_dim = self._call_fut(10)
self.assertAlmostEquals(exp_meters_per_pixel_dim,
act_meters_per_pixel_dim,
places=0)
def test_read_single(self):
from mock import MagicMock
from tilequeue.tile import deserialize_coord
from tilequeue.tile import serialize_coord
sqs_queue = MagicMock()
coord = deserialize_coord('1/1/1')
sqs_queue.get_messages = self._make_get_messages(
[serialize_coord(coord)])
msq = self._make_one([sqs_queue])
coord_msgs = msq.read()
self.assertEqual(1, len(coord_msgs))
self.assertEqual(coord, coord_msgs[0].coord)
def test_queue_mappings(self):
q1 = object()
q2 = object()
q3 = object()
specs = (
(None, None, 'tile_queue_1', q1, None),
(0, 10, 'tile_queue_2', q2, None),
(10, 16, 'tile_queue_3', q3, 10),
)
qm = self.make_queue_mapper(specs)
q1_id, q2_id, q3_id = range(3)
self.assertIs(q1, qm.get_queue(q1_id))
self.assertIs(q2, qm.get_queue(q2_id))
self.assertIs(q3, qm.get_queue(q3_id))
ordered_queue_result = list(qm.queues_in_priority_order())
self.assertEquals(3, len(ordered_queue_result))
r1_id, r1_q = ordered_queue_result[0]
r2_id, r2_q = ordered_queue_result[1]
r3_id, r3_q = ordered_queue_result[2]
self.assertIs(q1, r1_q)
self.assertEquals(q1_id, r1_id)
self.assertIs(q2, r2_q)
self.assertEquals(q2_id, r2_id)
self.assertIs(q3, r3_q)
self.assertEquals(q3_id, r3_id)
from tilequeue.tile import deserialize_coord
# verify that the queue ids line up with those that have zooms
# specified
coord_groups = list(qm.group([deserialize_coord('5/0/0')]))
self.assertEquals(1, len(coord_groups))
self.assertEquals(1, coord_groups[0].queue_id)
coord_groups = list(qm.group([deserialize_coord('15/0/0')]))
self.assertEquals(1, len(coord_groups))
self.assertEquals(2, coord_groups[0].queue_id)
def test_job_done_no_queue_name(self):
from mock import MagicMock
from tilequeue.tile import CoordMessage
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('1/1/1')
sqs_queue = MagicMock()
sqs_queue.name = 'q1'
sqs_queue.delete_message = self._mock_delete_message
cm = CoordMessage(coord, 'msg_handle')
msq = self._make_one([sqs_queue])
with self.assertRaises(AssertionError):
msq.job_done(cm)
def read(self, max_to_read=1):
coord_messages = []
messages = self.sqs_queue.get_messages(num_messages=max_to_read,
attributes=["SentTimestamp"])
for message in messages:
data = message.get_body()
coord = deserialize_coord(data)
if coord is None:
# log?
continue
timestamp = float(message.attributes.get('SentTimestamp'))
coord_message = CoordMessage(coord, message, timestamp)
coord_messages.append(coord_message)
return coord_messages
def test_single_queue_mapper(self):
from mock import MagicMock
from tilequeue.tile import deserialize_coord
tile_queue_mock = MagicMock()
queue_mapper = self.make_queue_mapper('queue_name', tile_queue_mock)
coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')]
coord_groups = list(queue_mapper.group(coords))
self.assertEquals(2, len(coord_groups))
cg1, cg2 = coord_groups
self.assertEquals('queue_name', cg1.queue_id)
self.assertEquals('queue_name', cg2.queue_id)
self.assertEquals(1, len(cg1.coords))
self.assertEquals(coords[0], cg1.coords[0])
self.assertEquals(1, len(cg2.coords))
self.assertEquals(coords[1], cg2.coords[0])
self.assertIs(tile_queue_mock, queue_mapper.get_queue('queue_name'))
qs = queue_mapper.queues_in_priority_order()
self.assertEquals(1, len(qs))
qn, q = qs[0]
self.assertEquals('queue_name', qn)
self.assertIs(tile_queue_mock, q)
def test_job_done_single(self):
from mock import MagicMock
from tilequeue.tile import CoordMessage
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('1/1/1')
sqs_queue = MagicMock()
sqs_queue.name = 'q1'
sqs_queue.delete_message = self._mock_delete_message
cm = CoordMessage(coord, 'msg_handle', dict(queue_name=sqs_queue.name))
msq = self._make_one([sqs_queue])
msq.job_done(cm)
self.assertIsNotNone(self.mock_handle)
self.assertEqual(self.mock_handle, 'msg_handle')
def test_enqueue_batch(self):
from mock import MagicMock
from tilequeue.tile import deserialize_coord
sqs_queue1 = MagicMock()
sqs_queue1.name = 'q1'
sqs_queue1.write_batch = self._mock_write_batch
sqs_queue2 = MagicMock()
sqs_queue2.name = 'q2'
sqs_queue2.write_batch = self._mock_write_batch
msq = self._make_one([sqs_queue1, sqs_queue2])
msq.enqueue_batch([
deserialize_coord('1/1/1'),
deserialize_coord('1/0/0')
])
self.assertIsNotNone(self.mock_write_batch)
self.assertEqual(2, len(self.mock_write_batch))
coord_str1, coord_str2 = [x[1] for x in self.mock_write_batch]
self.assertEqual(coord_str1, '1/1/1')
self.assertEqual(coord_str2, '1/0/0')
def test_single_queue_mapper(self):
from mock import MagicMock
from tilequeue.tile import deserialize_coord
tile_queue_mock = MagicMock()
queue_mapper = self.make_queue_mapper('queue_name', tile_queue_mock)
coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')]
coord_groups = list(queue_mapper.group(coords))
self.assertEquals(2, len(coord_groups))
cg1, cg2 = coord_groups
self.assertEquals('queue_name', cg1.queue_id)
self.assertEquals('queue_name', cg2.queue_id)
self.assertEquals(1, len(cg1.coords))
self.assertEquals(coords[0], cg1.coords[0])
self.assertEquals(1, len(cg2.coords))
self.assertEquals(coords[1], cg2.coords[0])
self.assertIs(tile_queue_mock, queue_mapper.get_queue('queue_name'))
qs = queue_mapper.queues_in_priority_order()
self.assertEquals(1, len(qs))
qn, q = qs[0]
self.assertEquals('queue_name', qn)
self.assertIs(tile_queue_mock, q)
def test_track_and_done(self):
from tilequeue.tile import deserialize_coord
from tilequeue.queue.message import QueueHandle
queue_id = 1
queue_handle = QueueHandle(queue_id, 'handle')
coords = [deserialize_coord('1/1/1')]
coord_handles = self.tracker.track(queue_handle, coords)
self.assertEqual(1, len(coord_handles))
coord_handle = coord_handles[0]
self.assertIs(queue_handle, coord_handle)
track_result = self.tracker.done(coord_handle)
self.assertIs(queue_handle, track_result.queue_handle)
self.assertTrue(track_result.all_done)
self.assertFalse(track_result.parent_tile)
def test_track_and_done(self):
from tilequeue.tile import deserialize_coord
from tilequeue.queue.message import QueueHandle
queue_id = 1
queue_handle = QueueHandle(queue_id, 'handle')
coords = [deserialize_coord('1/1/1')]
coord_handles = self.tracker.track(queue_handle, coords)
self.assertEqual(1, len(coord_handles))
coord_handle = coord_handles[0]
self.assertIs(queue_handle, coord_handle)
track_result = self.tracker.done(coord_handle)
self.assertIs(queue_handle, track_result.queue_handle)
self.assertTrue(track_result.all_done)
self.assertFalse(track_result.parent_tile)
def _assert_no_id_in_props(self, features, merge_fn):
from tilequeue.process import Context
from tilequeue.tile import deserialize_coord
layer_name = 'layername'
feature_layer = dict(
features=features,
layer_datum=dict(name=layer_name),
)
feature_layers = [feature_layer]
ctx = Context(feature_layers=feature_layers,
tile_coord=deserialize_coord('0/0/0'),
unpadded_bounds=None,
params=dict(source_layer=layer_name),
resources=None)
merged_feature_layer = merge_fn(ctx)
merged_features = merged_feature_layer['features']
self.assertEquals(1, len(merged_features))
merged_feature = merged_features[0]
props = merged_feature[1]
self.assertTrue('id' not in props)
def test_tags(self):
from tilequeue.store import KeyFormatType
from tilequeue.store import S3
from tilequeue.store import S3TileKeyGenerator
s3_client = self._make_stub_s3_client()
tags = None
tile_key_gen = S3TileKeyGenerator(
key_format_type=KeyFormatType.hash_prefix)
store = S3(s3_client, 'bucket', 'prefix', False, 60, None,
'public-read', tags, tile_key_gen)
tile_data = 'data'
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('14/1/2')
from tilequeue.format import mvt_format
store.write_tile(tile_data, coord, mvt_format)
self.assertIsNone(store.s3_client.put_props.get('Tagging'))
store.tags = dict(prefix='foo', run_id='bar')
store.write_tile(tile_data, coord, mvt_format)
self.assertEquals('prefix=foo&run_id=bar',
store.s3_client.put_props.get('Tagging'))
def test_tags(self):
from tilequeue.store import KeyFormatType
from tilequeue.store import S3
from tilequeue.store import S3TileKeyGenerator
s3_client = self._make_stub_s3_client()
tags = None
tile_key_gen = S3TileKeyGenerator(
key_format_type=KeyFormatType.hash_prefix)
store = S3(s3_client, 'bucket', 'prefix', False, 60, None,
'public-read', tags, tile_key_gen)
tile_data = 'data'
from tilequeue.tile import deserialize_coord
coord = deserialize_coord('14/1/2')
from tilequeue.format import mvt_format
store.write_tile(tile_data, coord, mvt_format)
self.assertIsNone(store.s3_client.put_props.get('Tagging'))
store.tags = dict(prefix='foo', run_id='bar')
store.write_tile(tile_data, coord, mvt_format)
self.assertEquals('prefix=foo&run_id=bar',
store.s3_client.put_props.get('Tagging'))
def _call_fut(self, building_shapes, building_part_shapes):
from tilequeue.tile import deserialize_coord
from tilequeue.process import Context
building_features = []
building_id = 1
for building_shape in building_shapes:
building_props = dict(
id=building_id,
kind='building',
)
building_feature = building_shape, building_props, building_id
building_features.append(building_feature)
building_id += 1
part_features = []
building_part_id = building_id
for part_shape in building_part_shapes:
part_props = dict(
id=building_part_id,
kind='building_part',
)
part_feature = part_shape, part_props, building_part_id
part_features.append(part_feature)
building_part_id += 1
building_features = building_features + part_features
building_feature_layer = dict(
features=building_features,
layer_datum=dict(name='buildings'),
)
feature_layers = [building_feature_layer]
ctx = Context(feature_layers=feature_layers,
tile_coord=deserialize_coord('0/0/0'),
unpadded_bounds=None,
params=dict(source_layer='buildings'),
resources=None)
from vectordatasource.transform import buildings_unify
buildings_unify(ctx)
return building_feature_layer['features']
def test_no_merge_preserve_props(self):
import shapely.geometry
from tilequeue.process import Context
from tilequeue.tile import deserialize_coord
from vectordatasource.transform import merge_polygon_features
buildings = []
for i in (0, 100):
id = i + 1
points = [
(1 + i, 1 + i),
(10 + i, 10 + i),
(1 + i, 10 + i),
(1 + i, 1 + i),
]
shape = shapely.geometry.Polygon(points)
props = dict(
id=id,
kind='building',
unique_value='value-%d' % id,
)
building = shape, props, id
buildings.append(building)
layer_name = 'buildings'
feature_layer = dict(
features=buildings,
layer_datum=dict(name=layer_name),
)
feature_layers = [feature_layer]
ctx = Context(feature_layers=feature_layers,
tile_coord=deserialize_coord('0/0/0'),
unpadded_bounds=None,
params=dict(source_layer=layer_name),
resources=None)
merged_feature_layer = merge_polygon_features(ctx)
merged_features = merged_feature_layer['features']
self.assertEquals(2, len(merged_features))
for f in merged_features:
props = f[1]
self.assertTrue('id' in props)
def test_marshall_single_coord(self):
from tilequeue.tile import deserialize_coord
result = self.msg_marshaller.marshall([deserialize_coord('1/1/1')])
self.assertEqual('1/1/1', result)
def test_unmarshall_multiple(self):
from tilequeue.tile import deserialize_coord
actual = self.msg_marshaller.unmarshall('1/1/1,2/2/2')
self.assertEqual(2, len(actual))
self.assertEqual(actual[0], deserialize_coord('1/1/1'))
self.assertEqual(actual[1], deserialize_coord('2/2/2'))
from boto import connect_s3
from boto.s3.bucket import Bucket
from tilequeue.tile import deserialize_coord
from tilequeue.tile import serialize_coord
import fileinput
bucket_name = 'mapzen-tiles-assets'
key_name = 'test/integration-test-coords.txt'
coords_to_store = set()
for line in fileinput.input():
coord = deserialize_coord(line.strip())
assert coord
if coord.zoom == 0:
uplifted_coord = coord
else:
uplifted_coord = coord.zoomBy(-1).container()
coords_to_store.add(uplifted_coord)
sorted_coords = sorted(coords_to_store)
coords_str = '\n'.join(map(serialize_coord, sorted_coords))
conn = connect_s3()
bucket = Bucket(conn, bucket_name)
key = bucket.new_key(key_name)
key.set_contents_from_string(
coords_str,
headers={'Content-Type': 'text/plain'},
)
