def _create_messages(args, index_csv):
"""
Create all of the tile messages to be enqueued.
Args:
args (dict): Same arguments as patch_upload_queue().
index_csv (str): CSV file with tile info.
Generates:
(list): List of strings containing Json data
"""
# DP NOTE: configuration is not actually used by encode_*_key method
backend = BossBackend(None)
chunk_key_list = []
with open(index_csv, "rt") as data:
lines = data.readlines()
for line in lines:
parts = line.split(",")
chunk_key_list.append(parts[0])
msgs = []
for base_chunk_key in chunk_key_list:
parts = backend.decode_chunk_key(base_chunk_key)
chunk_x = parts['x_index']
chunk_y = parts['y_index']
chunk_z = parts['z_index']
t = parts['t_index']
num_of_tiles = parts['num_tiles']
chunk_key = base_chunk_key
z_start = chunk_z * args['z_chunk_size']
for tile in range(z_start, z_start + num_of_tiles):
tile_key = backend.encode_tile_key(args['project_info'],
args['resolution'], chunk_x,
chunk_y, tile, t)
msg = {
'job_id': args['job_id'],
'upload_queue_arn': args['upload_queue'],
'ingest_queue_arn': args['ingest_queue'],
'chunk_key': chunk_key,
'tile_key': tile_key,
}
yield json.dumps(msg)
def test_decode_chunk_key(self):
"""Test encoding an object key"""
b = BossBackend(self.example_config_data)
b.setup(self.api_token)
params = {
"collection": 1,
"experiment": 2,
"channel": 3,
"resolution": 0,
"x_index": 5,
"y_index": 6,
"z_index": 1,
"t_index": 0,
"num_tiles": 0,
}
proj = [
str(params['collection']),
str(params['experiment']),
str(params['channel'])
]
key = b.encode_chunk_key(
params['num_tiles'],
proj,
params['resolution'],
params['x_index'],
params['y_index'],
params['z_index'],
params['t_index'],
)
parts = b.decode_chunk_key(key)
assert parts["collection"] == params['collection']
assert parts["experiment"] == params['experiment']
assert parts["channel"] == params['channel']
assert parts["resolution"] == params['resolution']
assert parts["x_index"] == params['x_index']
assert parts["y_index"] == params['y_index']
assert parts["z_index"] == params['z_index']
assert parts["t_index"] == params['t_index']
assert parts["num_tiles"] == params['num_tiles']
def check_tiles(self, chunk_key, tiles):
"""
Check the chunk's tile map for missing tiles. If any are missing,
generate the proper stringified JSON for putting those missing tiles
back in the tile upload queue.
Args:
chunk_key (str): Identifies chunk of tiles.
tiles (): List of tiles uploaded for the chunk.
Yields:
(str): JSON string for sending to SQS tile upload queue.
"""
# Only using encode|decode_*_key methods, so don't need to provide a
# config.
ingest_backend = BossBackend(None)
chunk_key_parts = ingest_backend.decode_chunk_key(chunk_key)
chunk_x = chunk_key_parts['x_index']
chunk_y = chunk_key_parts['y_index']
chunk_z = chunk_key_parts['z_index']
t = chunk_key_parts['t_index']
num_tiles = chunk_key_parts['num_tiles']
z_start = chunk_z * self.job['z_chunk_size']
for tile_z in range(z_start, z_start + num_tiles):
# First arg is a list of [collection, experiment, channel] ids.
tile_key = ingest_backend.encode_tile_key(self._get_project_info(),
self.job['resolution'],
chunk_x, chunk_y, tile_z,
t)
if tile_key in tiles:
continue
msg = {
'job_id': self.job['task_id'],
'upload_queue_arn': self.job['upload_queue'],
'ingest_queue_arn': self.job['ingest_queue'],
'chunk_key': chunk_key,
'tile_key': tile_key
}
log.info(
f'Re-enqueuing tile: {tile_key} belonging to chunk: {chunk_key}'
)
yield json.dumps(msg)