def test_create_messages(self):
job_id = 8
num_tiles = 16
exp_upload_queue = 'upload-test-queue'
exp_ingest_queue = 'ingest-test-queue'
args = {
'job_id': job_id,
'project_info': [5, 3, 2], # Collection/Experiment/Channel ids
'z_chunk_size': 16,
'resolution': 0,
'upload_queue': exp_upload_queue,
'ingest_queue': exp_ingest_queue
}
backend = BossBackend(None)
x1, y1, z1 = (0, 0, 0)
x2, y2, z2 = (1024, 1024, 16)
res = 0
t = 2
chunk_key1 = backend.encode_chunk_key(num_tiles, args['project_info'],
res, x1, y1, z1, t)
chunk_key2 = backend.encode_chunk_key(num_tiles, args['project_info'],
res, x2, y2, z2, t)
with NamedTemporaryFile(mode='wt', suffix='.csv',
delete=False) as output_csv:
csv_file = output_csv.name
output_csv.write('{},{},{}'.format(chunk_key1, job_id, num_tiles))
output_csv.write('{},{},{}'.format(chunk_key2, job_id, num_tiles))
try:
actual = _create_messages(args, csv_file)
for i, raw_msg in enumerate(actual):
msg = json.loads(raw_msg)
self.assertEqual(job_id, msg['job_id'])
self.assertEqual(exp_upload_queue, msg['upload_queue_arn'])
self.assertEqual(exp_ingest_queue, msg['ingest_queue_arn'])
if i < 16:
self.assertEqual(chunk_key1, msg['chunk_key'])
exp_tile_key = backend.encode_tile_key(
args['project_info'], res, x1, y1, z1 + i, t)
self.assertEqual(exp_tile_key, msg['tile_key'])
elif i < 32:
self.assertEqual(chunk_key2, msg['chunk_key'])
exp_tile_key = backend.encode_tile_key(
args['project_info'], res, x2, y2, z2 + i - 16, t)
self.assertEqual(exp_tile_key, msg['tile_key'])
else:
self.fail('Too many messages returned')
finally:
os.remove(csv_file)
def test_encode_chunk_key(self):
"""Test encoding an object key"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
params = {"collection": 1,
"experiment": 2,
"channel": 3,
"resolution": 0,
"x_index": 5,
"y_index": 6,
"z_index": 1,
"t_index": 0,
"num_tiles": 16,
}
proj = [str(params['collection']), str(params['experiment']), str(params['channel'])]
key = b.encode_chunk_key(params['num_tiles'], proj,
params['resolution'],
params['x_index'],
params['y_index'],
params['z_index'],
params['t_index'],
)
assert key == six.u("77ff984241a0d6aa443d8724a816866d&16&1&2&3&0&5&6&1&0")
def create_messages(args):
"""Create all of the tile messages to be enqueued
Args:
args (dict): Same arguments as populate_upload_queue()
Returns:
list: List of strings containing Json data
"""
tile_size = lambda v: args[v + "_tile_size"]
range_ = lambda v: range(args[v + '_start'], args[v + '_stop'], tile_size(v))
# DP NOTE: configuration is not actually used by encode_*_key method
backend = BossBackend(None)
msgs = []
for t in range_('t'):
for z in range_('z'):
for y in range_('y'):
for x in range_('x'):
chunk_x = int(x/tile_size('x'))
chunk_y = int(y/tile_size('y'))
chunk_z = int(z/tile_size('z'))
num_of_tiles = min(tile_size('z'), args['z_stop'] - z)
chunk_key = backend.encode_chunk_key(num_of_tiles,
args['project_info'],
args['resolution'],
chunk_x,
chunk_y,
chunk_z,
t)
for tile in range(z, z + num_of_tiles):
tile_key = backend.encode_tile_key(args['project_info'],
args['resolution'],
chunk_x,
chunk_y,
tile,
t)
msg = {
'job_id': args['job_id'],
'upload_queue_arn': args['upload_queue'],
'ingest_queue_arn': args['ingest_queue'],
'chunk_key': chunk_key,
'tile_key': tile_key,
}
yield json.dumps(msg)
def test_decode_chunk_key(self):
"""Test encoding an object key"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
params = {
"collection": 1,
"experiment": 2,
"channel": 3,
"resolution": 0,
"x_index": 5,
"y_index": 6,
"z_index": 1,
"t_index": 0,
"num_tiles": 0,
}
proj = [
str(params['collection']),
str(params['experiment']),
str(params['channel'])
]
key = b.encode_chunk_key(
params['num_tiles'],
proj,
params['resolution'],
params['x_index'],
params['y_index'],
params['z_index'],
params['t_index'],
)
parts = b.decode_chunk_key(key)
assert parts["collection"] == params['collection']
assert parts["experiment"] == params['experiment']
assert parts["channel"] == params['channel']
assert parts["resolution"] == params['resolution']
assert parts["x_index"] == params['x_index']
assert parts["y_index"] == params['y_index']
assert parts["z_index"] == params['z_index']
assert parts["t_index"] == params['t_index']
assert parts["num_tiles"] == params['num_tiles']
def test_check_tiles(self):
table = 'foo'
db_host = 'bar'
job = {
'collection': 90,
'experiment': 141,
'channel': 985,
'task_id': 1082,
'resolution': 0,
'z_chunk_size': 16,
'upload_queue': 'foo',
'ingest_queue': 'bar',
'ingest_type': TILE_INGEST,
}
chunk_x = 94
chunk_y = 40
chunk_z = int(2128 / job['z_chunk_size'])
# Simulate the tile map in the Dynamo tile index.
tile_map = {
'51b03272fb38672209b1891a5b6e20b8&90&141&985&0&94&40&2128&0': {
'N': '1'
},
'1743c07e50c1a642e357d6738cab13e9&90&141&985&0&94&40&2129&0': {
'N': '1'
},
'de1a408932bcfc3d36b5b4a7607d0964&90&141&985&0&94&40&2130&0': {
'N': '1'
},
'12223001b0bc18b3a8cb3463b09552d8&90&141&985&0&94&40&2131&0': {
'N': '1'
},
'82ae0d1f7775f29e30491036fd0335e2&90&141&985&0&94&40&2132&0': {
'N': '1'
},
'4b0c2f87566cbcdffb9b89ee873c0949&90&141&985&0&94&40&2134&0': {
'N': '1'
},
'6f13405cda1b04907b9ed3f9795eb9fb&90&141&985&0&94&40&2135&0': {
'N': '1'
},
'ab42d5b88996f9a0076c51f13f082173&90&141&985&0&94&40&2136&0': {
'N': '1'
},
'fb3bdfff8982a8250a9f21c30098d2e5&90&141&985&0&94&40&2137&0': {
'N': '1'
},
'2853c92ea467cf0e3f571bba11a63473&90&141&985&0&94&40&2138&0': {
'N': '1'
},
'6700434c27e4fb23de45fe0df9f2162f&90&141&985&0&94&40&2139&0': {
'N': '1'
},
'2fb7f8223dae87bf8d9c6bd73f7e3d2a&90&141&985&0&94&40&2141&0': {
'N': '1'
},
'6483f9efb8da500d010e1ea70b956ebe&90&141&985&0&94&40&2142&0': {
'N': '1'
},
'b8f59755d83f8501e387fb15329ee7ee&90&141&985&0&94&40&2143&0': {
'N': '1'
},
}
# The tile map is missing tiles with z=2133 and 2140.
missing_tiles = [
'16a888bbb0457cb8e6bfce882b74afff&90&141&985&0&94&40&2133&0',
'4a1c5be8336960d75fb4c4366544a9d3&90&141&985&0&94&40&2140&0'
]
cs = ChunkScanner(self.dynamo, self.sqs, table, db_host, job,
self.resource, self.x_size, self.y_size,
self.kvio_settings, self.stateio_config,
self.objectio_config)
backend = BossBackend(None)
chunk_key = backend.encode_chunk_key(16, cs._get_project_info(),
job['resolution'], chunk_x,
chunk_y, chunk_z)
msg_template = {
'job_id': job['task_id'],
'upload_queue_arn': 'foo',
'ingest_queue_arn': 'bar',
'chunk_key': chunk_key
}
# Expect messages for tile upload queue for tiles with z=2133 and 2140.
exp = [
json.dumps(dict(msg_template, tile_key=tile))
for tile in missing_tiles
]
actual = [tile for tile in cs.check_tiles(chunk_key, tile_map)]
self.assertCountEqual(exp, actual)