def delete_tiles(self, ingest_job):
"""
Delete all remaining tiles from the tile index database and tile bucket
Args:
ingest_job: Ingest job model
Returns:
None
Raises:
BossError : For exceptions that happen while deleting the tiles and index
"""
try:
# Get all the chunks for a job
tiledb = BossTileIndexDB(ingest_job.collection + '&' +
ingest_job.experiment)
tilebucket = TileBucket(ingest_job.collection + '&' +
ingest_job.experiment)
chunks = list(tiledb.getTaskItems(ingest_job.id))
for chunk in chunks:
chunk_key = chunk['chunk_key']
# delete each tile in the chunk
for key in chunk['tile_uploaded_map']:
response = tilebucket.deleteObject(key)
tiledb.deleteCuboid(chunk['chunk_key'], ingest_job.id)
except Exception as e:
raise BossError(
"Exception while deleteing tiles for the ingest job {}. {}".
format(ingest_job.id, e), ErrorCodes.BOSS_SYSTEM_ERROR)
def delete_tiles(self, ingest_job):
"""
Delete all remaining tiles from the tile index database and tile bucket
5/24/2018 - This code depends on a GSI for the tile index. The GSI was
removed because its key didn't shard well. Cleanup will now be handled
by TTL policies applied to the tile bucket and the tile index. This
method will be removed once that code is merged.
Args:
ingest_job: Ingest job model
Returns:
None
Raises:
BossError : For exceptions that happen while deleting the tiles and index
"""
try:
# Get all the chunks for a job
tiledb = BossTileIndexDB(ingest_job.collection + '&' + ingest_job.experiment)
tilebucket = TileBucket(ingest_job.collection + '&' + ingest_job.experiment)
chunks = list(tiledb.getTaskItems(ingest_job.id))
for chunk in chunks:
# delete each tile in the chunk
for key in chunk['tile_uploaded_map']:
response = tilebucket.deleteObject(key)
tiledb.deleteCuboid(chunk['chunk_key'], ingest_job.id)
except Exception as e:
raise BossError("Exception while deleteing tiles for the ingest job {}. {}".format(ingest_job.id, e),
ErrorCodes.BOSS_SYSTEM_ERROR)
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
warnings.simplefilter('ignore')
with open('nddynamo/schemas/boss_tile_index.json') as fp:
schema = json.load(fp)
BossTileIndexDB.createTable(schema,
endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
self.tileindex_db = BossTileIndexDB(
nd_proj.project_name, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def setUpClass(cls):
# Silence warnings about open boto3 sessions.
warnings.filterwarnings('ignore')
cls.job_id = 123
cls.nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0,
cls.job_id)
TileBucket.createBucket()
cls.tile_bucket = TileBucket(cls.nd_proj.project_name)
warnings.simplefilter('ignore')
#with open('/Users/manavpj1/repos/boss/django/bossingest/test/boss_tile_index.json') as fp:
# schema = json.load(fp)
#BossTileIndexDB.createTable(schema, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
cls.tileindex_db = BossTileIndexDB(
cls.nd_proj.project_name,
endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def _query_tile_index(paginator, tile_index_name, job_id, i):
"""
Query tile index table for chunks with the given job_id and suffix.
Args:
paginator (boto3.Paginator):
tile_index_name (str): Name of DynamoDB tile index table.
job_id (int): Id of ingest job.
i (int): Suffix for job_id (which Dynamo partition to check).
Returns:
(list[response])
"""
return paginator.paginate(
TableName=tile_index_name,
IndexName=TASK_INDEX,
KeyConditionExpression="appended_task_id = :task_id_val",
ExpressionAttributeValues={
":task_id_val": {
"S": BossTileIndexDB.generate_appended_task_id(job_id, i)
}
})
def add_tile_entry(ingest_job):
"""
Put a fake chunk in the tile index for the given job. Cleans up fake chunk
when it goes out of scope.
Args:
ingest_job (dict): Job data as returned by the POST method.
Yields:
(None)
"""
tiledb = BossTileIndexDB(ingest_job['collection'] + '&' +
ingest_job['experiment'])
chunk_key = '{}&16&1&2&3&0&0&0&0&0'.format(randint(0, 2000))
tiledb.createCuboidEntry(chunk_key, ingest_job['id'])
# Mark some tiles as uploaded.
for i in range(12):
tiledb.markTileAsUploaded(chunk_key, 'fake_tile_key_{}'.format(i),
ingest_job['id'])
yield
# Cleanup.
print('deleting fake chunk')
tiledb.deleteCuboid(chunk_key, ingest_job['id'])
def add_tile_entry(ingest_job):
"""
Put a fake chunk in the tile index for the given job. Cleans up fake chunk
when it goes out of scope.
Args:
ingest_job (dict): Job data as returned by the POST method.
Yields:
(None)
"""
tiledb = BossTileIndexDB(ingest_job['collection'] + '&' + ingest_job['experiment'])
chunk_key = '{}&16&1&2&3&0&0&0&0&0'.format(randint(0, 2000))
tiledb.createCuboidEntry(chunk_key, ingest_job['id'])
# Mark some tiles as uploaded.
for i in range(12):
tiledb.markTileAsUploaded(chunk_key, 'fake_tile_key_{}'.format(i), ingest_job['id'])
yield
# Cleanup.
print('deleting fake chunk')
tiledb.deleteCuboid(chunk_key, ingest_job['id'])
# "ingest_queue": XX,
# "ingest_lambda":XX,
# "KVIO_SETTINGS": XX,
# "STATEIO_CONFIG": XX,
# "OBJECTIO_CONFIG": XX
# },
# 'tile_size_x': "{}".format(self.config.config_data["ingest_job"]["tile_size"]["x"]),
# 'tile_size_y': "{}".format(self.config.config_data["ingest_job"]["tile_size"]["y"])
# }
# TODO: DMK not sure if you actually need to set the job_id in proj_info
# Set the job id
proj_info.job_id = metadata["ingest_job"]
# update value in the dynamo table
tile_index_db = BossTileIndexDB(proj_info.project_name)
chunk = tile_index_db.getCuboid(metadata["chunk_key"],
int(metadata["ingest_job"]))
if chunk:
print("Updating tile index for chunk_key: {}".format(
metadata["chunk_key"]))
chunk_ready = tile_index_db.markTileAsUploaded(metadata["chunk_key"],
tile_key,
int(metadata["ingest_job"]))
else:
# First tile in the chunk
print("Creating first entry for chunk_key: {}".format(
metadata["chunk_key"]))
try:
tile_index_db.createCuboidEntry(metadata["chunk_key"],
int(metadata["ingest_job"]))
def handler(event, context):
# Load settings
SETTINGS = BossSettings.load()
# Used as a guard against trying to delete the SQS message when lambda is
# triggered by SQS.
sqs_triggered = 'Records' in event and len(event['Records']) > 0
if sqs_triggered :
# Lambda invoked by an SQS trigger.
msg_data = json.loads(event['Records'][0]['body'])
# Load the project info from the chunk key you are processing
chunk_key = msg_data['chunk_key']
proj_info = BossIngestProj.fromSupercuboidKey(chunk_key)
proj_info.job_id = msg_data['ingest_job']
else:
# Standard async invoke of this lambda.
# Load the project info from the chunk key you are processing
proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"])
proj_info.job_id = event["ingest_job"]
# Get message from SQS ingest queue, try for ~2 seconds
rx_cnt = 0
msg_data = None
msg_id = None
msg_rx_handle = None
while rx_cnt < 6:
ingest_queue = IngestQueue(proj_info)
msg = [x for x in ingest_queue.receiveMessage()]
if msg:
msg = msg[0]
print("MESSAGE: {}".format(msg))
print(len(msg))
msg_id = msg[0]
msg_rx_handle = msg[1]
msg_data = json.loads(msg[2])
print("MESSAGE DATA: {}".format(msg_data))
break
else:
rx_cnt += 1
print("No message found. Try {} of 6".format(rx_cnt))
time.sleep(1)
if not msg_id:
# No tiles ready to ingest.
print("No ingest message available")
return
# Get the chunk key of the tiles to ingest.
chunk_key = msg_data['chunk_key']
tile_error_queue = TileErrorQueue(proj_info)
print("Ingesting Chunk {}".format(chunk_key))
tiles_in_chunk = int(chunk_key.split('&')[1])
# Setup SPDB instance
sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"],
msg_data['parameters']["STATEIO_CONFIG"],
msg_data['parameters']["OBJECTIO_CONFIG"])
# Get tile list from Tile Index Table
tile_index_db = BossTileIndexDB(proj_info.project_name)
# tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk.
tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"]))
if tile_index_result is None:
# If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards.
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to chunk key missing from tile index table")
return
# Sort the tile keys
print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"]))
tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()]
if len(tile_key_list) < tiles_in_chunk:
print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list)))
if not sqs_triggered:
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
return
tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1]))
tile_key_list = ["&".join(x) for x in tile_key_list]
print("Sorted Tile Keys: {}".format(tile_key_list))
# Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits
resource_dict = msg_data['parameters']['resource']
_, exp_name, ch_name = resource_dict["boss_key"].split("&")
resource_dict["channel"]["name"] = ch_name
resource_dict["channel"]["description"] = ""
resource_dict["channel"]["sources"] = []
resource_dict["channel"]["related"] = []
resource_dict["channel"]["default_time_sample"] = 0
resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED"
resource_dict["experiment"]["name"] = exp_name
resource_dict["experiment"]["description"] = ""
resource_dict["experiment"]["num_time_samples"] = 1
resource_dict["experiment"]["time_step"] = None
resource_dict["experiment"]["time_step_unit"] = None
resource_dict["coord_frame"]["name"] = "cf"
resource_dict["coord_frame"]["name"] = ""
resource_dict["coord_frame"]["x_start"] = 0
resource_dict["coord_frame"]["x_stop"] = 100000
resource_dict["coord_frame"]["y_start"] = 0
resource_dict["coord_frame"]["y_stop"] = 100000
resource_dict["coord_frame"]["z_start"] = 0
resource_dict["coord_frame"]["z_stop"] = 100000
resource_dict["coord_frame"]["voxel_unit"] = "nanometers"
# Setup the resource
resource = BossResourceBasic()
resource.from_dict(resource_dict)
dtype = resource.get_numpy_data_type()
# read all tiles from bucket into a slab
tile_bucket = TileBucket(proj_info.project_name)
data = []
num_z_slices = 0
for tile_key in tile_key_list:
try:
image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key)
except KeyError:
print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format(
tile_key))
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to missing tile in bucket")
return
image_bytes = BytesIO(image_data)
image_size = image_bytes.getbuffer().nbytes
# Get tiles size from metadata, need to shape black tile if actual tile is corrupt.
if 'x_size' in metadata:
tile_size_x = metadata['x_size']
else:
print('MetadataMissing: x_size not in tile metadata: using 1024.')
tile_size_x = 1024
if 'y_size' in metadata:
tile_size_y = metadata['y_size']
else:
print('MetadataMissing: y_size not in tile metadata: using 1024.')
tile_size_y = 1024
if image_size == 0:
print('TileError: Zero length tile, using black instead: {}'.format(tile_key))
error_msg = 'Zero length tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
else:
try:
tile_img = np.asarray(Image.open(image_bytes), dtype=dtype)
except TypeError as te:
print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}'
.format(image_size, tile_key))
error_msg = 'Incomplete tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
except OSError as oe:
print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}'
.format(image_size, tile_key, oe))
error_msg = 'OSError: {}'.format(oe)
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
data.append(tile_img)
num_z_slices += 1
# Make 3D array of image data. It should be in XYZ at this point
chunk_data = np.array(data)
del data
tile_dims = chunk_data.shape
# Break into Cube instances
print("Tile Dims: {}".format(tile_dims))
print("Num Z Slices: {}".format(num_z_slices))
num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0]))
num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1]))
print("Num X Cuboids: {}".format(num_x_cuboids))
print("Num Y Cuboids: {}".format(num_y_cuboids))
chunk_key_parts = BossUtil.decode_chunk_key(chunk_key)
t_index = chunk_key_parts['t_index']
for x_idx in range(0, num_x_cuboids):
for y_idx in range(0, num_y_cuboids):
# TODO: check time series support
cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution])
cube.zeros()
# Compute Morton ID
# TODO: verify Morton indices correct!
print(chunk_key_parts)
morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids)
morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids)
print("Morton X: {}".format(morton_x_ind))
print("Morton Y: {}".format(morton_y_ind))
morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])])
# Insert sub-region from chunk_data into cuboid
x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0]
x_end = x_start + CUBOIDSIZE[proj_info.resolution][0]
x_end = min(x_end, tile_dims[2])
y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1]
y_end = y_start + CUBOIDSIZE[proj_info.resolution][1]
y_end = min(y_end, tile_dims[1])
z_end = CUBOIDSIZE[proj_info.resolution][2]
# TODO: get sub-array w/o making a copy.
print("Yrange: {}".format(y_end - y_start))
print("Xrange: {}".format(x_end - x_start))
print("X start: {}".format(x_start))
print("X stop: {}".format(x_end))
cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices,
y_start:y_end, x_start:x_end]
# Create object key
object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index)
print("Object Key: {}".format(object_key))
# Put object in S3
sp.objectio.put_objects([object_key], [cube.to_blosc()])
# Add object to index
sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"]))
# Update id indices if this is an annotation channel
# We no longer index during ingest.
#if resource.data['channel']['type'] == 'annotation':
# try:
# sp.objectio.update_id_indices(
# resource, proj_info.resolution, [object_key], [cube.data])
# except SpdbError as ex:
# sns_client = boto3.client('sns')
# topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"]
# msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format(
# ex.message,
# resource.data['collection']['name'],
# resource.data['experiment']['name'],
# resource.data['channel']['name'])
# sns_client.publish(
# TopicArn=topic_arn,
# Subject='Object services misuse',
# Message=msg)
lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME)
names = AWSNames.create_from_lambda_name(context.function_name)
delete_tiles_data = {
'tile_key_list': tile_key_list,
'region': SETTINGS.REGION_NAME,
'bucket': tile_bucket.bucket.name
}
# Delete tiles from tile bucket.
lambda_client.invoke(
FunctionName=names.delete_tile_objs_lambda,
InvocationType='Event',
Payload=json.dumps(delete_tiles_data).encode()
)
delete_tile_entry_data = {
'tile_index': tile_index_db.table.name,
'region': SETTINGS.REGION_NAME,
'chunk_key': chunk_key,
'task_id': msg_data['ingest_job']
}
# Delete entry from tile index.
lambda_client.invoke(
FunctionName=names.delete_tile_index_entry_lambda,
InvocationType='Event',
Payload=json.dumps(delete_tile_entry_data).encode()
)
if not sqs_triggered:
# Delete message since it was processed successfully
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
def process(msg, context, region):
"""
Process a single message.
Args:
msg (dict): Contents described at the top of the file.
context (Context): Lambda context object.
region (str): Lambda execution region.
"""
job_id = int(msg['ingest_job'])
chunk_key = msg['chunk_key']
tile_key = msg['tile_key']
print("Tile key: {}".format(tile_key))
proj_info = BossIngestProj.fromTileKey(tile_key)
# Set the job id
proj_info.job_id = msg['ingest_job']
print("Data: {}".format(msg))
# update value in the dynamo table
tile_index_db = BossTileIndexDB(proj_info.project_name)
chunk = tile_index_db.getCuboid(chunk_key, job_id)
if chunk:
if tile_index_db.cuboidReady(chunk_key, chunk["tile_uploaded_map"]):
print("Chunk already has all its tiles: {}".format(chunk_key))
# Go ahead and setup to fire another ingest lambda so this tile
# entry will be deleted on successful execution of the ingest lambda.
chunk_ready = True
else:
print("Updating tile index for chunk_key: {}".format(chunk_key))
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
else:
# First tile in the chunk
print("Creating first entry for chunk_key: {}".format(chunk_key))
try:
tile_index_db.createCuboidEntry(chunk_key, job_id)
except ClientError as err:
# Under _exceptional_ circumstances, it's possible for another lambda
# to beat the current instance to creating the initial cuboid entry
# in the index.
error_code = err.response['Error'].get('Code', 'Unknown')
if error_code == 'ConditionalCheckFailedException':
print('Chunk key entry already created - proceeding.')
else:
raise
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
# ingest the chunk if we have all the tiles
if chunk_ready:
print("CHUNK READY SENDING MESSAGE: {}".format(chunk_key))
# insert a new job in the insert queue if we have all the tiles
ingest_queue = IngestQueue(proj_info)
ingest_queue.sendMessage(json.dumps(msg))
# Invoke Ingest lambda function
names = AWSNames.create_from_lambda_name(context.function_name)
lambda_client = boto3.client('lambda', region_name=region)
lambda_client.invoke(
FunctionName=names.tile_ingest_lambda,
InvocationType='Event',
Payload=json.dumps(msg).encode())
else:
print("Chunk not ready for ingest yet: {}".format(chunk_key))
print("DONE!")
def test_upload_tile_index_table(self):
""""""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
ingest_job = ingest_mgmr.create_ingest_job()
assert (ingest_job.id is not None)
# Get the chunks in this job
# Get the project information
bosskey = ingest_job.collection + '&' + ingest_job.experiment + '&' + ingest_job.channel_layer
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
proj_name = ingest_job.collection + '&' + ingest_job.experiment
tile_index_db = BossTileIndexDB(proj_name)
tilebucket = TileBucket(str(col_id) + '&' + str(exp_id))
for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
# For each time step, compute the chunks and tile keys
for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
for y in range(ingest_job.y_start, ingest_job.y_stop,
ingest_job.tile_size_y):
for x in range(ingest_job.x_start, ingest_job.x_stop,
ingest_job.tile_size_x):
# compute the chunk indices
chunk_x = int(x / ingest_job.tile_size_x)
chunk_y = int(y / ingest_job.tile_size_y)
chunk_z = int(z / 16)
# Compute the number of tiles in the chunk
if ingest_job.z_stop - z >= 16:
num_of_tiles = 16
else:
num_of_tiles = ingest_job.z_stop - z
# Generate the chunk key
chunk_key = (BossBackend(
ingest_mgmr.config)).encode_chunk_key(
num_of_tiles, project_info,
ingest_job.resolution, chunk_x, chunk_y,
chunk_z, time_step)
# Upload the chunk to the tile index db
tile_index_db.createCuboidEntry(
chunk_key, ingest_job.id)
key_map = {}
for tile in range(0, num_of_tiles):
# get the object key and upload it
#tile_key = tilebucket.encodeObjectKey(ch_id, ingest_job.resolution,
# chunk_x, chunk_y, tile, time_step)
tile_key = 'fakekey' + str(tile)
tile_index_db.markTileAsUploaded(
chunk_key, tile_key)
# for each chunk key, delete entries from the tile_bucket
# Check if data has been uploaded
chunks = list(tile_index_db.getTaskItems(ingest_job.id))
assert (len(chunks) != 0)
ingest_mgmr.delete_tiles(ingest_job)
chunks = list(tile_index_db.getTaskItems(ingest_job.id))
assert (len(chunks) == 0)
class Test_BossTileIndexDB(unittest.TestCase):
"""
Note that the chunk keys used, for testing, do not have real hash keys.
The rest of the chunk key is valid.
"""
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
warnings.simplefilter("ignore")
with open("ndingest/nddynamo/schemas/boss_tile_index.json") as fp:
schema = json.load(fp)
BossTileIndexDB.createTable(schema,
endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
self.tileindex_db = BossTileIndexDB(
nd_proj.project_name, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def tearDown(self):
BossTileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def test_cuboidReady_false(self):
fake_map = {"o": 1}
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
self.assertFalse(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_cuboidReady_true(self):
fake_map = {
"s1": 1,
"s2": 1,
"s3": 1,
"s4": 1,
"s5": 1,
"s6": 1,
"s7": 1,
"s8": 1,
"s9": 1,
"s10": 1,
"s11": 1,
"s12": 1,
"s13": 1,
"s14": 1,
"s15": 1,
"s16": 1,
}
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
self.assertTrue(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_cuboidReady_small_cuboid_true(self):
"""Test case where the number of tiles is smaller than a cuboid in the z direction."""
fake_map = {
"s1": 1,
"s2": 1,
"s3": 1,
"s4": 1,
"s5": 1,
"s6": 1,
"s7": 1,
"s8": 1,
}
num_tiles = 8
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
self.assertTrue(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_cuboidReady_small_cuboid_false(self):
"""Test case where the number of tiles is smaller than a cuboid in the z direction."""
fake_map = {
"s1": 1,
"s2": 1,
"s3": 1,
"s4": 1,
"s5": 1,
"s6": 1,
"s7": 1
}
num_tiles = 8
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
self.assertFalse(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_createCuboidEntry(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
task_id = 21
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
actual = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(chunk_key, actual["chunk_key"])
self.assertEqual({}, actual[TILE_UPLOADED_MAP_KEY])
self.assertIn("expires", actual)
self.assertEqual(task_id, actual["task_id"])
self.assertTrue(actual["appended_task_id"].startswith(
"{}_".format(task_id)))
def test_markTileAsUploaded(self):
# Cuboid must first have an entry before one of its tiles may be marked
# as uploaded.
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
task_id = 231
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
self.tileindex_db.markTileAsUploaded(chunk_key, "fakekey&sss", task_id)
expected = {"fakekey&sss": 1}
resp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(expected, resp[TILE_UPLOADED_MAP_KEY])
def test_markTileAsUploaded_multiple(self):
# Cuboid must first have an entry before one of its tiles may be marked
# as uploaded.
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
task_id = 231
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
self.tileindex_db.markTileAsUploaded(chunk_key, "fakekey&sss", task_id)
expected_first = {"fakekey&sss": 1}
resp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(expected_first, resp[TILE_UPLOADED_MAP_KEY])
expected_second = {"fakekey&sss": 1, "fakekey&ttt": 1}
self.tileindex_db.markTileAsUploaded(chunk_key, "fakekey&ttt", task_id)
resp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertCountEqual(expected_second, resp[TILE_UPLOADED_MAP_KEY])
def test_deleteItem(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = "<hash>&{}&111&222&333&0&0&0&0&0".format(num_tiles)
task_id = 231
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
preDelResp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(chunk_key, preDelResp["chunk_key"])
self.tileindex_db.deleteCuboid(chunk_key, task_id)
postDelResp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertIsNone(postDelResp)
def test_getTaskItems(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key1 = "<hash>&{}&111&222&333&0&0&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key1, task_id=3)
chunk_key2 = "<hash>&{}&111&222&333&0&1&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key2, task_id=3)
chunk_key3 = "<hash>&{}&111&222&333&0&2&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key3, task_id=3)
# Cuboid for a different upload job.
chunk_key4 = "<hash>&{}&555&666&777&0&0&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key4, task_id=5)
expected = [
{
"task_id": 3,
TILE_UPLOADED_MAP_KEY: {},
"chunk_key": chunk_key1
},
{
"task_id": 3,
TILE_UPLOADED_MAP_KEY: {},
"chunk_key": chunk_key2
},
{
"task_id": 3,
TILE_UPLOADED_MAP_KEY: {},
"chunk_key": chunk_key3
},
]
actual = list(self.tileindex_db.getTaskItems(3))
filtered = [{
"task_id": i["task_id"],
TILE_UPLOADED_MAP_KEY: i[TILE_UPLOADED_MAP_KEY],
"chunk_key": i["chunk_key"],
} for i in actual]
six.assertCountEqual(self, expected, filtered)
def test_getTaskItems_force_multiple_queries(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
job = 3
chunk_key1 = "<hash>&{}&111&222&333&0&0&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key1, task_id=job)
chunk_key2 = "<hash>&{}&111&222&333&0&1&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key2, task_id=job)
chunk_key3 = "<hash>&{}&111&222&333&0&2&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key3, task_id=job)
# Cuboid for a different upload job.
chunk_key4 = "<hash>&{}&555&666&777&0&0&0&z&t".format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key4, task_id=5)
expected = [
{
"task_id": job,
TILE_UPLOADED_MAP_KEY: {},
"chunk_key": chunk_key1
},
{
"task_id": job,
TILE_UPLOADED_MAP_KEY: {},
"chunk_key": chunk_key2
},
{
"task_id": job,
TILE_UPLOADED_MAP_KEY: {},
"chunk_key": chunk_key3
},
]
# Limit only 1 read per query so multiple queries required.
query_limit = 1
actual = list(self.tileindex_db.getTaskItems(job, query_limit))
filtered = [{
"task_id": i["task_id"],
TILE_UPLOADED_MAP_KEY: i[TILE_UPLOADED_MAP_KEY],
"chunk_key": i["chunk_key"],
} for i in actual]
six.assertCountEqual(self, expected, filtered)
def test_createCuboidAlreadyExistsRaises(self):
"""Raise an error if the chunk key already exists in the index."""
chunk_key = "foo"
task_id = 9999999
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
with self.assertRaises(botocore.exceptions.ClientError) as err:
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
error_code = err.response["Error"].get("Code", "Unknown")
self.assertEqual("ConditionalCheckFailedException", error_code)
if not msg_id:
# Nothing to flush. Exit.
sys.exit("No ingest message available")
# Get the write-cuboid key to flush
chunk_key = msg_data['chunk_key']
print("Ingesting Chunk {}".format(chunk_key))
# Setup SPDB instance
sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"],
msg_data['parameters']["STATEIO_CONFIG"],
msg_data['parameters']["OBJECTIO_CONFIG"])
# Get tile list from Tile Index Table
tile_index_db = BossTileIndexDB(proj_info.project_name)
# tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk.
tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"]))
if tile_index_result is None:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
sys.exit("Aborting due to chunk key missing from tile index table")
# Sort the tile keys
print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"]))
tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()]
tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1]))
tile_key_list = ["&".join(x) for x in tile_key_list]
print("Sorted Tile Keys: {}".format(tile_key_list))
# Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits
def tearDown(self):
BossTileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
class Test_BossTileIndexDB(unittest.TestCase):
"""
Note that the chunk keys used, for testing, do not have real hash keys.
The rest of the chunk key is valid.
"""
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
warnings.simplefilter('ignore')
with open('nddynamo/schemas/boss_tile_index.json') as fp:
schema = json.load(fp)
BossTileIndexDB.createTable(schema,
endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
self.tileindex_db = BossTileIndexDB(
nd_proj.project_name, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def tearDown(self):
BossTileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def test_cuboidReady_false(self):
fake_map = {'o': 1}
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
self.assertFalse(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_cuboidReady_true(self):
fake_map = {
's1': 1,
's2': 1,
's3': 1,
's4': 1,
's5': 1,
's6': 1,
's7': 1,
's8': 1,
's9': 1,
's10': 1,
's11': 1,
's12': 1,
's13': 1,
's14': 1,
's15': 1,
's16': 1
}
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
self.assertTrue(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_cuboidReady_small_cuboid_true(self):
"""Test case where the number of tiles is smaller than a cuboid in the z direction."""
fake_map = {
's1': 1,
's2': 1,
's3': 1,
's4': 1,
's5': 1,
's6': 1,
's7': 1,
's8': 1
}
num_tiles = 8
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
self.assertTrue(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_cuboidReady_small_cuboid_false(self):
"""Test case where the number of tiles is smaller than a cuboid in the z direction."""
fake_map = {
's1': 1,
's2': 1,
's3': 1,
's4': 1,
's5': 1,
's6': 1,
's7': 1
}
num_tiles = 8
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
self.assertFalse(self.tileindex_db.cuboidReady(chunk_key, fake_map))
def test_createCuboidEntry(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
task_id = 21
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
preDelResp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(chunk_key, preDelResp['chunk_key'])
self.assertEqual({}, preDelResp['tile_uploaded_map'])
def test_markTileAsUploaded(self):
# Cuboid must first have an entry before one of its tiles may be marked
# as uploaded.
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
task_id = 231
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
self.tileindex_db.markTileAsUploaded(chunk_key, 'fakekey&sss', task_id)
expected = {'fakekey&sss': 1}
resp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(expected, resp['tile_uploaded_map'])
def test_markTileAsUploaded_multiple(self):
# Cuboid must first have an entry before one of its tiles may be marked
# as uploaded.
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
task_id = 231
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
self.tileindex_db.markTileAsUploaded(chunk_key, 'fakekey&sss', task_id)
expected_first = {'fakekey&sss': 1}
resp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(expected_first, resp['tile_uploaded_map'])
expected_second = {'fakekey&sss': 1, 'fakekey&ttt': 1}
self.tileindex_db.markTileAsUploaded(chunk_key, 'fakekey&ttt', task_id)
resp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertCountEqual(expected_second, resp['tile_uploaded_map'])
def test_deleteItem(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key = '<hash>&{}&111&222&333&0&0&0&0&0'.format(num_tiles)
task_id = 231
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
preDelResp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertEqual(chunk_key, preDelResp['chunk_key'])
self.tileindex_db.deleteCuboid(chunk_key, task_id)
postDelResp = self.tileindex_db.getCuboid(chunk_key, task_id)
self.assertIsNone(postDelResp)
def test_getTaskItems(self):
num_tiles = settings.SUPER_CUBOID_SIZE[2]
chunk_key1 = '<hash>&{}&111&222&333&0&0&0&z&t'.format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key1, task_id=3)
chunk_key2 = '<hash>&{}&111&222&333&0&1&0&z&t'.format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key2, task_id=3)
chunk_key3 = '<hash>&{}&111&222&333&0&2&0&z&t'.format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key3, task_id=3)
# Cuboid for a different upload job.
chunk_key4 = '<hash>&{}&555&666&777&0&0&0&z&t'.format(num_tiles)
self.tileindex_db.createCuboidEntry(chunk_key4, task_id=5)
expected = [{
'task_id': 3,
'tile_uploaded_map': {},
'chunk_key': chunk_key1
}, {
'task_id': 3,
'tile_uploaded_map': {},
'chunk_key': chunk_key2
}, {
'task_id': 3,
'tile_uploaded_map': {},
'chunk_key': chunk_key3
}]
actual = list(self.tileindex_db.getTaskItems(3))
six.assertCountEqual(self, expected, actual)
def test_createCuboidAlreadyExistsRaises(self):
"""Raise an error if the chunk key already exists in the index."""
chunk_key = 'foo'
task_id = 9999999
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
with self.assertRaises(botocore.exceptions.ClientError) as err:
self.tileindex_db.createCuboidEntry(chunk_key, task_id)
error_code = err.response['Error'].get('Code', 'Unknown')
self.assertEqual('ConditionalCheckFailedException', error_code)
def process(msg, context, region):
"""
Process a single message.
Args:
msg (dict): Contents described at the top of the file.
context (Context): Lambda context object.
region (str): Lambda execution region.
"""
job_id = int(msg['ingest_job'])
chunk_key = msg['chunk_key']
tile_key = msg['tile_key']
print("Tile key: {}".format(tile_key))
proj_info = BossIngestProj.fromTileKey(tile_key)
# Set the job id
proj_info.job_id = msg['ingest_job']
print("Data: {}".format(msg))
# update value in the dynamo table
tile_index_db = BossTileIndexDB(proj_info.project_name)
chunk = tile_index_db.getCuboid(chunk_key, job_id)
if chunk:
if tile_index_db.cuboidReady(chunk_key, chunk["tile_uploaded_map"]):
print("Chunk already has all its tiles: {}".format(chunk_key))
# Go ahead and setup to fire another ingest lambda so this tile
# entry will be deleted on successful execution of the ingest lambda.
chunk_ready = True
else:
print("Updating tile index for chunk_key: {}".format(chunk_key))
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
else:
# First tile in the chunk
print("Creating first entry for chunk_key: {}".format(chunk_key))
try:
tile_index_db.createCuboidEntry(chunk_key, job_id)
except ClientError as err:
# Under _exceptional_ circumstances, it's possible for another lambda
# to beat the current instance to creating the initial cuboid entry
# in the index.
error_code = err.response['Error'].get('Code', 'Unknown')
if error_code == 'ConditionalCheckFailedException':
print('Chunk key entry already created - proceeding.')
else:
raise
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
# ingest the chunk if we have all the tiles
if chunk_ready:
print("CHUNK READY SENDING MESSAGE: {}".format(chunk_key))
# insert a new job in the insert queue if we have all the tiles
ingest_queue = IngestQueue(proj_info)
ingest_queue.sendMessage(json.dumps(msg))
# Invoke Ingest lambda function
names = AWSNames.from_lambda(context.function_name)
lambda_client = boto3.client('lambda', region_name=region)
lambda_client.invoke(
FunctionName=names.tile_ingest.lambda_,
InvocationType='Event',
Payload=json.dumps(msg).encode())
else:
print("Chunk not ready for ingest yet: {}".format(chunk_key))
print("DONE!")
def handler(event, context):
# Load settings
SETTINGS = BossSettings.load()
# Used as a guard against trying to delete the SQS message when lambda is
# triggered by SQS.
sqs_triggered = 'Records' in event and len(event['Records']) > 0
if sqs_triggered :
# Lambda invoked by an SQS trigger.
msg_data = json.loads(event['Records'][0]['body'])
# Load the project info from the chunk key you are processing
chunk_key = msg_data['chunk_key']
proj_info = BossIngestProj.fromSupercuboidKey(chunk_key)
proj_info.job_id = msg_data['ingest_job']
else:
# Standard async invoke of this lambda.
# Load the project info from the chunk key you are processing
proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"])
proj_info.job_id = event["ingest_job"]
# Get message from SQS ingest queue, try for ~2 seconds
rx_cnt = 0
msg_data = None
msg_id = None
msg_rx_handle = None
while rx_cnt < 6:
ingest_queue = IngestQueue(proj_info)
try:
msg = [x for x in ingest_queue.receiveMessage()]
# StopIteration may be converted to a RunTimeError.
except (StopIteration, RuntimeError):
msg = None
if msg:
msg = msg[0]
print("MESSAGE: {}".format(msg))
print(len(msg))
msg_id = msg[0]
msg_rx_handle = msg[1]
msg_data = json.loads(msg[2])
print("MESSAGE DATA: {}".format(msg_data))
break
else:
rx_cnt += 1
print("No message found. Try {} of 6".format(rx_cnt))
time.sleep(1)
if not msg_id:
# No tiles ready to ingest.
print("No ingest message available")
return
# Get the chunk key of the tiles to ingest.
chunk_key = msg_data['chunk_key']
tile_error_queue = TileErrorQueue(proj_info)
print("Ingesting Chunk {}".format(chunk_key))
tiles_in_chunk = int(chunk_key.split('&')[1])
# Setup SPDB instance
sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"],
msg_data['parameters']["STATEIO_CONFIG"],
msg_data['parameters']["OBJECTIO_CONFIG"])
# Get tile list from Tile Index Table
tile_index_db = BossTileIndexDB(proj_info.project_name)
# tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk.
tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"]))
if tile_index_result is None:
# If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards.
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to chunk key missing from tile index table")
return
# Sort the tile keys
print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"]))
tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()]
if len(tile_key_list) < tiles_in_chunk:
print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list)))
if not sqs_triggered:
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
return
tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1]))
tile_key_list = ["&".join(x) for x in tile_key_list]
print("Sorted Tile Keys: {}".format(tile_key_list))
# Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits
resource_dict = msg_data['parameters']['resource']
_, exp_name, ch_name = resource_dict["boss_key"].split("&")
resource_dict["channel"]["name"] = ch_name
resource_dict["channel"]["description"] = ""
resource_dict["channel"]["sources"] = []
resource_dict["channel"]["related"] = []
resource_dict["channel"]["default_time_sample"] = 0
resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED"
resource_dict["experiment"]["name"] = exp_name
resource_dict["experiment"]["description"] = ""
resource_dict["experiment"]["num_time_samples"] = 1
resource_dict["experiment"]["time_step"] = None
resource_dict["experiment"]["time_step_unit"] = None
resource_dict["coord_frame"]["name"] = "cf"
resource_dict["coord_frame"]["name"] = ""
resource_dict["coord_frame"]["x_start"] = 0
resource_dict["coord_frame"]["x_stop"] = 100000
resource_dict["coord_frame"]["y_start"] = 0
resource_dict["coord_frame"]["y_stop"] = 100000
resource_dict["coord_frame"]["z_start"] = 0
resource_dict["coord_frame"]["z_stop"] = 100000
resource_dict["coord_frame"]["voxel_unit"] = "nanometers"
# Setup the resource
resource = BossResourceBasic()
resource.from_dict(resource_dict)
dtype = resource.get_numpy_data_type()
# read all tiles from bucket into a slab
tile_bucket = TileBucket(proj_info.project_name)
data = []
num_z_slices = 0
for tile_key in tile_key_list:
try:
image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key)
except KeyError:
print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format(
tile_key))
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to missing tile in bucket")
return
image_bytes = BytesIO(image_data)
image_size = image_bytes.getbuffer().nbytes
# Get tiles size from metadata, need to shape black tile if actual tile is corrupt.
if 'x_size' in metadata:
tile_size_x = metadata['x_size']
else:
print('MetadataMissing: x_size not in tile metadata: using 1024.')
tile_size_x = 1024
if 'y_size' in metadata:
tile_size_y = metadata['y_size']
else:
print('MetadataMissing: y_size not in tile metadata: using 1024.')
tile_size_y = 1024
if image_size == 0:
print('TileError: Zero length tile, using black instead: {}'.format(tile_key))
error_msg = 'Zero length tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
else:
try:
# DP NOTE: Issues when specifying dtype in the asarray function with Pillow ver 8.3.1.
# Fixed by separating array instantiation and dtype assignment.
tile_img = np.asarray(Image.open(image_bytes))
tile_img = tile_img.astype(dtype)
except TypeError as te:
print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}'
.format(image_size, tile_key))
error_msg = 'Incomplete tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
except OSError as oe:
print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}'
.format(image_size, tile_key, oe))
error_msg = 'OSError: {}'.format(oe)
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
data.append(tile_img)
num_z_slices += 1
# Make 3D array of image data. It should be in XYZ at this point
chunk_data = np.array(data)
del data
tile_dims = chunk_data.shape
# Break into Cube instances
print("Tile Dims: {}".format(tile_dims))
print("Num Z Slices: {}".format(num_z_slices))
num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0]))
num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1]))
print("Num X Cuboids: {}".format(num_x_cuboids))
print("Num Y Cuboids: {}".format(num_y_cuboids))
chunk_key_parts = BossUtil.decode_chunk_key(chunk_key)
t_index = chunk_key_parts['t_index']
for x_idx in range(0, num_x_cuboids):
for y_idx in range(0, num_y_cuboids):
# TODO: check time series support
cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution])
cube.zeros()
# Compute Morton ID
# TODO: verify Morton indices correct!
print(chunk_key_parts)
morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids)
morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids)
print("Morton X: {}".format(morton_x_ind))
print("Morton Y: {}".format(morton_y_ind))
morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])])
# Insert sub-region from chunk_data into cuboid
x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0]
x_end = x_start + CUBOIDSIZE[proj_info.resolution][0]
x_end = min(x_end, tile_dims[2])
y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1]
y_end = y_start + CUBOIDSIZE[proj_info.resolution][1]
y_end = min(y_end, tile_dims[1])
z_end = CUBOIDSIZE[proj_info.resolution][2]
# TODO: get sub-array w/o making a copy.
print("Yrange: {}".format(y_end - y_start))
print("Xrange: {}".format(x_end - x_start))
print("X start: {}".format(x_start))
print("X stop: {}".format(x_end))
cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices,
y_start:y_end, x_start:x_end]
# Create object key
object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index)
print("Object Key: {}".format(object_key))
# Put object in S3
sp.objectio.put_objects([object_key], [cube.to_blosc()])
# Add object to index
sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"]))
# Update id indices if this is an annotation channel
# We no longer index during ingest.
#if resource.data['channel']['type'] == 'annotation':
# try:
# sp.objectio.update_id_indices(
# resource, proj_info.resolution, [object_key], [cube.data])
# except SpdbError as ex:
# sns_client = boto3.client('sns')
# topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"]
# msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format(
# ex.message,
# resource.data['collection']['name'],
# resource.data['experiment']['name'],
# resource.data['channel']['name'])
# sns_client.publish(
# TopicArn=topic_arn,
# Subject='Object services misuse',
# Message=msg)
lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME)
names = AWSNames.from_lambda(context.function_name)
delete_tiles_data = {
'tile_key_list': tile_key_list,
'region': SETTINGS.REGION_NAME,
'bucket': tile_bucket.bucket.name
}
# Delete tiles from tile bucket.
lambda_client.invoke(
FunctionName=names.delete_tile_objs.lambda_,
InvocationType='Event',
Payload=json.dumps(delete_tiles_data).encode()
)
delete_tile_entry_data = {
'tile_index': tile_index_db.table.name,
'region': SETTINGS.REGION_NAME,
'chunk_key': chunk_key,
'task_id': msg_data['ingest_job']
}
# Delete entry from tile index.
lambda_client.invoke(
FunctionName=names.delete_tile_index_entry.lambda_,
InvocationType='Event',
Payload=json.dumps(delete_tile_entry_data).encode()
)
if not sqs_triggered:
# Delete message since it was processed successfully
ingest_queue.deleteMessage(msg_id, msg_rx_handle)