def test_create_messages(self): job_id = 8 num_tiles = 16 exp_upload_queue = 'upload-test-queue' exp_ingest_queue = 'ingest-test-queue' args = { 'job_id': job_id, 'project_info': [5, 3, 2], # Collection/Experiment/Channel ids 'z_chunk_size': 16, 'resolution': 0, 'upload_queue': exp_upload_queue, 'ingest_queue': exp_ingest_queue } backend = BossBackend(None) x1, y1, z1 = (0, 0, 0) x2, y2, z2 = (1024, 1024, 16) res = 0 t = 2 chunk_key1 = backend.encode_chunk_key(num_tiles, args['project_info'], res, x1, y1, z1, t) chunk_key2 = backend.encode_chunk_key(num_tiles, args['project_info'], res, x2, y2, z2, t) with NamedTemporaryFile(mode='wt', suffix='.csv', delete=False) as output_csv: csv_file = output_csv.name output_csv.write('{},{},{}'.format(chunk_key1, job_id, num_tiles)) output_csv.write('{},{},{}'.format(chunk_key2, job_id, num_tiles)) try: actual = _create_messages(args, csv_file) for i, raw_msg in enumerate(actual): msg = json.loads(raw_msg) self.assertEqual(job_id, msg['job_id']) self.assertEqual(exp_upload_queue, msg['upload_queue_arn']) self.assertEqual(exp_ingest_queue, msg['ingest_queue_arn']) if i < 16: self.assertEqual(chunk_key1, msg['chunk_key']) exp_tile_key = backend.encode_tile_key( args['project_info'], res, x1, y1, z1 + i, t) self.assertEqual(exp_tile_key, msg['tile_key']) elif i < 32: self.assertEqual(chunk_key2, msg['chunk_key']) exp_tile_key = backend.encode_tile_key( args['project_info'], res, x2, y2, z2 + i - 16, t) self.assertEqual(exp_tile_key, msg['tile_key']) else: self.fail('Too many messages returned') finally: os.remove(csv_file)
def test_encode_chunk_key(self): """Test encoding an object key""" b = BossBackend(self.example_config_data) b.setup(self.api_token) params = {"collection": 1, "experiment": 2, "channel": 3, "resolution": 0, "x_index": 5, "y_index": 6, "z_index": 1, "t_index": 0, "num_tiles": 16, } proj = [str(params['collection']), str(params['experiment']), str(params['channel'])] key = b.encode_chunk_key(params['num_tiles'], proj, params['resolution'], params['x_index'], params['y_index'], params['z_index'], params['t_index'], ) assert key == six.u("77ff984241a0d6aa443d8724a816866d&16&1&2&3&0&5&6&1&0")
def create_messages(args): """Create all of the tile messages to be enqueued Args: args (dict): Same arguments as populate_upload_queue() Returns: list: List of strings containing Json data """ tile_size = lambda v: args[v + "_tile_size"] range_ = lambda v: range(args[v + '_start'], args[v + '_stop'], tile_size(v)) # DP NOTE: configuration is not actually used by encode_*_key method backend = BossBackend(None) msgs = [] for t in range_('t'): for z in range_('z'): for y in range_('y'): for x in range_('x'): chunk_x = int(x/tile_size('x')) chunk_y = int(y/tile_size('y')) chunk_z = int(z/tile_size('z')) num_of_tiles = min(tile_size('z'), args['z_stop'] - z) chunk_key = backend.encode_chunk_key(num_of_tiles, args['project_info'], args['resolution'], chunk_x, chunk_y, chunk_z, t) for tile in range(z, z + num_of_tiles): tile_key = backend.encode_tile_key(args['project_info'], args['resolution'], chunk_x, chunk_y, tile, t) msg = { 'job_id': args['job_id'], 'upload_queue_arn': args['upload_queue'], 'ingest_queue_arn': args['ingest_queue'], 'chunk_key': chunk_key, 'tile_key': tile_key, } yield json.dumps(msg)
def test_decode_chunk_key(self): """Test encoding an object key""" b = BossBackend(self.example_config_data) b.setup(self.api_token) params = { "collection": 1, "experiment": 2, "channel": 3, "resolution": 0, "x_index": 5, "y_index": 6, "z_index": 1, "t_index": 0, "num_tiles": 0, } proj = [ str(params['collection']), str(params['experiment']), str(params['channel']) ] key = b.encode_chunk_key( params['num_tiles'], proj, params['resolution'], params['x_index'], params['y_index'], params['z_index'], params['t_index'], ) parts = b.decode_chunk_key(key) assert parts["collection"] == params['collection'] assert parts["experiment"] == params['experiment'] assert parts["channel"] == params['channel'] assert parts["resolution"] == params['resolution'] assert parts["x_index"] == params['x_index'] assert parts["y_index"] == params['y_index'] assert parts["z_index"] == params['z_index'] assert parts["t_index"] == params['t_index'] assert parts["num_tiles"] == params['num_tiles']
def test_check_tiles(self): table = 'foo' db_host = 'bar' job = { 'collection': 90, 'experiment': 141, 'channel': 985, 'task_id': 1082, 'resolution': 0, 'z_chunk_size': 16, 'upload_queue': 'foo', 'ingest_queue': 'bar', 'ingest_type': TILE_INGEST, } chunk_x = 94 chunk_y = 40 chunk_z = int(2128 / job['z_chunk_size']) # Simulate the tile map in the Dynamo tile index. tile_map = { '51b03272fb38672209b1891a5b6e20b8&90&141&985&0&94&40&2128&0': { 'N': '1' }, '1743c07e50c1a642e357d6738cab13e9&90&141&985&0&94&40&2129&0': { 'N': '1' }, 'de1a408932bcfc3d36b5b4a7607d0964&90&141&985&0&94&40&2130&0': { 'N': '1' }, '12223001b0bc18b3a8cb3463b09552d8&90&141&985&0&94&40&2131&0': { 'N': '1' }, '82ae0d1f7775f29e30491036fd0335e2&90&141&985&0&94&40&2132&0': { 'N': '1' }, '4b0c2f87566cbcdffb9b89ee873c0949&90&141&985&0&94&40&2134&0': { 'N': '1' }, '6f13405cda1b04907b9ed3f9795eb9fb&90&141&985&0&94&40&2135&0': { 'N': '1' }, 'ab42d5b88996f9a0076c51f13f082173&90&141&985&0&94&40&2136&0': { 'N': '1' }, 'fb3bdfff8982a8250a9f21c30098d2e5&90&141&985&0&94&40&2137&0': { 'N': '1' }, '2853c92ea467cf0e3f571bba11a63473&90&141&985&0&94&40&2138&0': { 'N': '1' }, '6700434c27e4fb23de45fe0df9f2162f&90&141&985&0&94&40&2139&0': { 'N': '1' }, '2fb7f8223dae87bf8d9c6bd73f7e3d2a&90&141&985&0&94&40&2141&0': { 'N': '1' }, '6483f9efb8da500d010e1ea70b956ebe&90&141&985&0&94&40&2142&0': { 'N': '1' }, 'b8f59755d83f8501e387fb15329ee7ee&90&141&985&0&94&40&2143&0': { 'N': '1' }, } # The tile map is missing tiles with z=2133 and 2140. missing_tiles = [ '16a888bbb0457cb8e6bfce882b74afff&90&141&985&0&94&40&2133&0', '4a1c5be8336960d75fb4c4366544a9d3&90&141&985&0&94&40&2140&0' ] cs = ChunkScanner(self.dynamo, self.sqs, table, db_host, job, self.resource, self.x_size, self.y_size, self.kvio_settings, self.stateio_config, self.objectio_config) backend = BossBackend(None) chunk_key = backend.encode_chunk_key(16, cs._get_project_info(), job['resolution'], chunk_x, chunk_y, chunk_z) msg_template = { 'job_id': job['task_id'], 'upload_queue_arn': 'foo', 'ingest_queue_arn': 'bar', 'chunk_key': chunk_key } # Expect messages for tile upload queue for tiles with z=2133 and 2140. exp = [ json.dumps(dict(msg_template, tile_key=tile)) for tile in missing_tiles ] actual = [tile for tile in cs.check_tiles(chunk_key, tile_map)] self.assertCountEqual(exp, actual)