def _create_messages(args, index_csv): """ Create all of the tile messages to be enqueued. Args: args (dict): Same arguments as patch_upload_queue(). index_csv (str): CSV file with tile info. Generates: (list): List of strings containing Json data """ # DP NOTE: configuration is not actually used by encode_*_key method backend = BossBackend(None) chunk_key_list = [] with open(index_csv, "rt") as data: lines = data.readlines() for line in lines: parts = line.split(",") chunk_key_list.append(parts[0]) msgs = [] for base_chunk_key in chunk_key_list: parts = backend.decode_chunk_key(base_chunk_key) chunk_x = parts['x_index'] chunk_y = parts['y_index'] chunk_z = parts['z_index'] t = parts['t_index'] num_of_tiles = parts['num_tiles'] chunk_key = base_chunk_key z_start = chunk_z * args['z_chunk_size'] for tile in range(z_start, z_start + num_of_tiles): tile_key = backend.encode_tile_key(args['project_info'], args['resolution'], chunk_x, chunk_y, tile, t) msg = { 'job_id': args['job_id'], 'upload_queue_arn': args['upload_queue'], 'ingest_queue_arn': args['ingest_queue'], 'chunk_key': chunk_key, 'tile_key': tile_key, } yield json.dumps(msg)
def test_decode_chunk_key(self): """Test encoding an object key""" b = BossBackend(self.example_config_data) b.setup(self.api_token) params = { "collection": 1, "experiment": 2, "channel": 3, "resolution": 0, "x_index": 5, "y_index": 6, "z_index": 1, "t_index": 0, "num_tiles": 0, } proj = [ str(params['collection']), str(params['experiment']), str(params['channel']) ] key = b.encode_chunk_key( params['num_tiles'], proj, params['resolution'], params['x_index'], params['y_index'], params['z_index'], params['t_index'], ) parts = b.decode_chunk_key(key) assert parts["collection"] == params['collection'] assert parts["experiment"] == params['experiment'] assert parts["channel"] == params['channel'] assert parts["resolution"] == params['resolution'] assert parts["x_index"] == params['x_index'] assert parts["y_index"] == params['y_index'] assert parts["z_index"] == params['z_index'] assert parts["t_index"] == params['t_index'] assert parts["num_tiles"] == params['num_tiles']
def check_tiles(self, chunk_key, tiles): """ Check the chunk's tile map for missing tiles. If any are missing, generate the proper stringified JSON for putting those missing tiles back in the tile upload queue. Args: chunk_key (str): Identifies chunk of tiles. tiles (): List of tiles uploaded for the chunk. Yields: (str): JSON string for sending to SQS tile upload queue. """ # Only using encode|decode_*_key methods, so don't need to provide a # config. ingest_backend = BossBackend(None) chunk_key_parts = ingest_backend.decode_chunk_key(chunk_key) chunk_x = chunk_key_parts['x_index'] chunk_y = chunk_key_parts['y_index'] chunk_z = chunk_key_parts['z_index'] t = chunk_key_parts['t_index'] num_tiles = chunk_key_parts['num_tiles'] z_start = chunk_z * self.job['z_chunk_size'] for tile_z in range(z_start, z_start + num_tiles): # First arg is a list of [collection, experiment, channel] ids. tile_key = ingest_backend.encode_tile_key(self._get_project_info(), self.job['resolution'], chunk_x, chunk_y, tile_z, t) if tile_key in tiles: continue msg = { 'job_id': self.job['task_id'], 'upload_queue_arn': self.job['upload_queue'], 'ingest_queue_arn': self.job['ingest_queue'], 'chunk_key': chunk_key, 'tile_key': tile_key } log.info( f'Re-enqueuing tile: {tile_key} belonging to chunk: {chunk_key}' ) yield json.dumps(msg)