class Test_Ingest_Queue: def setup_class(self): """Setup class parameters""" if "SQS_ENDPOINT" in dir(settings): self.endpoint_url = settings.SQS_ENDPOINT else: self.endpoint_url = None IngestQueue.createQueue(nd_proj, endpoint_url=self.endpoint_url) self.ingest_queue = IngestQueue(nd_proj, endpoint_url=self.endpoint_url) def teardown_class(self): """Teardown parameters""" IngestQueue.deleteQueue(nd_proj, endpoint_url=self.endpoint_url) def test_Message(self): """Testing the upload queue""" supercuboid_key = "kasthuri11&image&0&0" self.ingest_queue.sendMessage(supercuboid_key) for ( message_id, receipt_handle, message_body, ) in self.ingest_queue.receiveMessage(): assert supercuboid_key == message_body response = self.ingest_queue.deleteMessage(message_id, receipt_handle) assert "Successful" in response
def delete_ingest_queue(self): """ Returns: """ IngestQueue.deleteQueue(self.nd_proj, endpoint_url=None)
def createIngestJob(self, user_id, config_data): """Create an ingest job based on the posted config data""" config_data = json.loads(config_data) # validate schema if self.validateConfig(config_data): try: # create the upload queue UploadQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.upload_queue = UploadQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.upload_queue = self.upload_queue.url # create the ingest queue IngestQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.ingest_queue = IngestQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT).url # create the cleanup queue CleanupQueue.createQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) self.ingest_job.cleanup_queue = CleanupQueue( self.nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT).url self.generateUploadTasks() self.ingest_job.user_id = user_id self.ingest_job.save() return NDIngestJob.serialize(self.ingest_job._job) except Exception as e: print(e) raise NDWSError(e)
def teardown_class(self): """Teardown class parameters""" # cleanup tilebucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_key = self.tile_bucket.encodeObjectKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, ) self.tile_bucket.deleteObject(tile_key) morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, self.tiles) self.cuboid_bucket.deleteObject(supercuboid_key) # delete created entities TileIndexDB.deleteTable(endpoint_url="http://localhost:8000") CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000") IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") TileBucket.deleteBucket(endpoint_url="http://localhost:4567") try: CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567") except Exception as e: pass
def setup_class(self): """Setup class parameters""" if 'SQS_ENDPOINT' in dir(settings): self.endpoint_url = settings.SQS_ENDPOINT else: self.endpoint_url = None IngestQueue.createQueue(nd_proj, endpoint_url=self.endpoint_url) self.ingest_queue = IngestQueue(nd_proj, endpoint_url=self.endpoint_url)
def delete_ingest_queue(self): """ Delete the current ingest queue Returns: None """ IngestQueue.deleteQueue(self.nd_proj, endpoint_url=None)
def create_ingest_queue(self): """ Returns: """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) return queue
def create_ingest_queue(self): """ Create an ingest queue for an ingest job using the ndingest library Returns: IngestQueue : Returns a ingest queue object """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) return queue
def delete_ingest_queue(self): """ Delete the current ingest queue and removes it as an event source of the ingest lambda if it's connected. Returns: None """ queue = IngestQueue(self.nd_proj) self.remove_sqs_event_source_from_lambda(queue.arn, INGEST_LAMBDA) IngestQueue.deleteQueue(self.nd_proj)
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568")
def create_ingest_queue(self): """ Create an ingest queue for an ingest job using the ndingest library Returns: IngestQueue : Returns a ingest queue object """ IngestQueue.createQueue(self.nd_proj, endpoint_url=None) queue = IngestQueue(self.nd_proj, endpoint_url=None) timeout = self.get_ingest_lambda_timeout(INGEST_LAMBDA) # Ensure visibility timeout is greater than the ingest lambda that pulls # from it with a bit of buffer. queue.queue.set_attributes(Attributes={'VisibilityTimeout': str(timeout + 20)}) return queue
def test_setup_ingest(self): """Method to test the setup_ingest method""" try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data) assert (ingest_job is not None) # Check if the queue's exist proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel, ingest_job.resolution, ingest_job.id) ingest_mgmr.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) assert (upload_queue is not None) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) assert (ingest_queue is not None) ingest_mgmr.remove_ingest_credentials(ingest_job.id) except: raise finally: ingest_mgmr.delete_upload_queue() ingest_mgmr.delete_ingest_queue()
def test_post_new_ingest_job(self): """ Test view to create a new ingest job """ config_data = self.setup_helper.get_ingest_config_data_dict() config_data = json.loads(json.dumps(config_data)) # # Post the data url = '/' + version + '/ingest/' response = self.client.post(url, data=config_data, format='json') assert (response.status_code == 201) # Check if the queue's exist ingest_job = response.json() proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel_layer'], 0, ingest_job['id']) self.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) assert (upload_queue is not None) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) assert (ingest_queue is not None) # # Delete the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.delete(url) assert (response.status_code == 204)
def test_not_creator_admin(self): """Method to test only creators or admins can interact with ingest jobs""" config_data = self.setup_helper.get_ingest_config_data_dict() config_data = json.loads(json.dumps(config_data)) # # Post the data url = '/' + version + '/ingest/' response = self.client.post(url, data=config_data, format='json') assert (response.status_code == 201) # Check if the queue's exist ingest_job = response.json() proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'], 0, ingest_job['id']) self.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) assert (upload_queue is not None) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) assert (ingest_queue is not None) # Log in as the admin and create a job self.client.force_login(self.superuser) # Test joining the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.get(url) assert (response.status_code == 200) assert (response.json()['ingest_job']['id'] == ingest_job['id']) assert("credentials" in response.json()) # # Delete the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.delete(url) assert (response.status_code == 204)
def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url=settings.DYNAMO_ENDPOINT) except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url=settings.DYNAMO_ENDPOINT) # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) # create the upload queue UploadQueue.createQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) self.upload_queue = UploadQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) tile_bucket = TileBucket(nd_proj.project_name, endpoint_url=settings.S3_ENDPOINT) [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message = serializer.encodeUploadMessage( nd_proj.project_name, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, ) # insert message in the upload queue self.upload_queue.sendMessage(message) # receive message and upload object for ( message_id, receipt_handle, message_body, ) in self.upload_queue.receiveMessage(): tile_handle = cStringIO.StringIO() tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, self.z_tile, message_id, receipt_handle, )
def test_post_new_ingest_job(self): """ Test view to create a new ingest job """ config_data = self.setup_helper.get_ingest_config_data_dict() config_data = json.loads(json.dumps(config_data)) # # Post the data url = '/' + version + '/ingest/' response = self.client.post(url, data=config_data, format='json') self.assertEqual(201, response.status_code) # Check if the queue's exist ingest_job = response.json() proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'], 0, ingest_job['id']) self.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) self.assertIsNotNone(upload_queue) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) self.assertIsNotNone(ingest_queue) # Test joining the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.get(url) self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id']) self.assertIn("credentials", response.json()) # # Delete the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.delete(url) self.assertEqual(204, response.status_code) # Verify Queues are removed with self.assertRaises(ClientError): UploadQueue(nd_proj, endpoint_url=None) with self.assertRaises(ClientError): IngestQueue(nd_proj, endpoint_url=None) # Verify the job is deleted url = '/' + version + '/ingest/{}/status'.format(ingest_job['id']) response = self.client.get(url) self.assertEqual(response.status_code, 404)
def test_complete_ingest_job(self): """ Test view to create a new ingest job """ config_data = self.setup_helper.get_ingest_config_data_dict() config_data = json.loads(json.dumps(config_data)) # # Post the data url = '/' + version + '/ingest/' response = self.client.post(url, data=config_data, format='json') self.assertEqual(response.status_code, 201) # Check if the queues exist ingest_job = response.json() proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'], 0, ingest_job['id']) self.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) self.assertIsNotNone(upload_queue) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) self.assertIsNotNone(ingest_queue) # Test joining the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.get(url) self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id']) self.assertIn("credentials", response.json()) # Complete the job url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id']) response = self.client.post(url) # Can't complete until it is done self.assertEqual(400, response.status_code) # Wait for job to complete print('trying to join job') for cnt in range(0, 30): # Try joining to kick the status url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) self.client.get(url) url = '/' + version + '/ingest/{}/status'.format(ingest_job['id']) response = self.client.get(url) if response.json()["status"] == IngestJob.UPLOADING: break time.sleep(10) print('completing') # Complete the job url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id']) response = self.client.post(url) self.assertEqual(204, response.status_code)
def deleteIngestJob(self, job_id): """Delete an ingest job based on job id""" try: ingest_job = NDIngestJob.fromId(job_id) nd_proj = NDIngestProj(ingest_job.project, ingest_job.channel, ingest_job.resolution) # delete the upload queue UploadQueue.deleteQueue( nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) # delete the ingest queue IngestQueue.deleteQueue( nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) # delete the cleanup queue CleanupQueue.deleteQueue( nd_proj, endpoint_url=ndingest_settings.SQS_ENDPOINT) ingest_job.status = INGEST_STATUS_DELETED ingest_job.save() except Exception as e: print(e) raise
def get_ingest_job_ingest_queue(self, ingest_job): """ Return the ingest queue for an ingest job Args: ingest_job: Ingest job model Returns: Ndingest.ingestqueue """ proj_class = BossIngestProj.load() self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel, ingest_job.resolution, ingest_job.id) queue = IngestQueue(self.nd_proj, endpoint_url=None) return queue
def delete_lambda_event_source_queues(self): """ Remove SQS queues connected to ingest lambdas that are connected as event sources. """ if int(self.job['ingest_type']) != TILE_INGEST: return try: config = BossConfig() INGEST_LAMBDA = config["aws"]["tile_ingest_lambda"] TILE_UPLOADED_LAMBDA = config["aws"]["tile_uploaded_lambda"] except Exception as ex: log.error(f'Failed to get lambda names from boss.config: {ex}') return self.remove_sqs_event_source_from_lambda( TileIndexQueue(self.nd_proj).arn, TILE_UPLOADED_LAMBDA) self.remove_sqs_event_source_from_lambda( IngestQueue(self.nd_proj).arn, INGEST_LAMBDA)
def process(msg, context, region): """ Process a single message. Args: msg (dict): Contents described at the top of the file. context (Context): Lambda context object. region (str): Lambda execution region. """ job_id = int(msg['ingest_job']) chunk_key = msg['chunk_key'] tile_key = msg['tile_key'] print("Tile key: {}".format(tile_key)) proj_info = BossIngestProj.fromTileKey(tile_key) # Set the job id proj_info.job_id = msg['ingest_job'] print("Data: {}".format(msg)) # update value in the dynamo table tile_index_db = BossTileIndexDB(proj_info.project_name) chunk = tile_index_db.getCuboid(chunk_key, job_id) if chunk: if tile_index_db.cuboidReady(chunk_key, chunk["tile_uploaded_map"]): print("Chunk already has all its tiles: {}".format(chunk_key)) # Go ahead and setup to fire another ingest lambda so this tile # entry will be deleted on successful execution of the ingest lambda. chunk_ready = True else: print("Updating tile index for chunk_key: {}".format(chunk_key)) chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id) else: # First tile in the chunk print("Creating first entry for chunk_key: {}".format(chunk_key)) try: tile_index_db.createCuboidEntry(chunk_key, job_id) except ClientError as err: # Under _exceptional_ circumstances, it's possible for another lambda # to beat the current instance to creating the initial cuboid entry # in the index. error_code = err.response['Error'].get('Code', 'Unknown') if error_code == 'ConditionalCheckFailedException': print('Chunk key entry already created - proceeding.') else: raise chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id) # ingest the chunk if we have all the tiles if chunk_ready: print("CHUNK READY SENDING MESSAGE: {}".format(chunk_key)) # insert a new job in the insert queue if we have all the tiles ingest_queue = IngestQueue(proj_info) ingest_queue.sendMessage(json.dumps(msg)) # Invoke Ingest lambda function names = AWSNames.from_lambda(context.function_name) lambda_client = boto3.client('lambda', region_name=region) lambda_client.invoke( FunctionName=names.tile_ingest.lambda_, InvocationType='Event', Payload=json.dumps(msg).encode()) else: print("Chunk not ready for ingest yet: {}".format(chunk_key)) print("DONE!")
class Test_IngestLambda: def setup_class(self): """Setup class parameters""" # create the tile index table. skip if it exists try: TileIndexDB.createTable(endpoint_url="http://localhost:8000") CuboidIndexDB.createTable(endpoint_url="http://localhost:8000") except Exception as e: pass self.tileindex_db = TileIndexDB(nd_proj.project_name, endpoint_url="http://localhost:8000") # create the tile bucket TileBucket.createBucket(endpoint_url="http://localhost:4567") self.tile_bucket = TileBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") self.tiles = [self.x_tile, self.y_tile, self.z_tile] = [0, 0, 0] message_id = "testing" receipt_handle = "123456" # insert SUPER_CUBOID_SIZE tiles in the bucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_handle = cStringIO.StringIO() self.tile_bucket.putObject( tile_handle, nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, message_id, receipt_handle, ) # creating the cuboid bucket CuboidBucket.createBucket(endpoint_url="http://localhost:4567") self.cuboid_bucket = CuboidBucket(nd_proj.project_name, endpoint_url="http://localhost:4567") # create the ingest queue IngestQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") self.ingest_queue = IngestQueue(nd_proj, endpoint_url="http://localhost:4568") # send message to the ingest queue morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, morton_index) response = self.ingest_queue.sendMessage(supercuboid_key) # create the cleanup queue CleanupQueue.createQueue(nd_proj, endpoint_url="http://localhost:4568") def teardown_class(self): """Teardown class parameters""" # cleanup tilebucket for z_index in (self.z_tile, settings.SUPER_CUBOID_SIZE[2], 1): tile_key = self.tile_bucket.encodeObjectKey( nd_proj.channel_name, nd_proj.resolution, self.x_tile, self.y_tile, z_index, ) self.tile_bucket.deleteObject(tile_key) morton_index = XYZMorton(self.tiles) supercuboid_key = self.cuboid_bucket.generateSupercuboidKey( nd_proj.channel_name, nd_proj.resolution, self.tiles) self.cuboid_bucket.deleteObject(supercuboid_key) # delete created entities TileIndexDB.deleteTable(endpoint_url="http://localhost:8000") CuboidIndexDB.deleteTable(endpoint_url="http://localhost:8000") IngestQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") CleanupQueue.deleteQueue(nd_proj, endpoint_url="http://localhost:4568") TileBucket.deleteBucket(endpoint_url="http://localhost:4567") try: CuboidBucket.deleteBucket(endpoint_url="http://localhost:4567") except Exception as e: pass def test_Uploadevent(self): """Testing the event""" # creating an emulambda function func = emulambda.import_lambda("ingestlambda.lambda_handler") # creating an emulambda event event = emulambda.parse_event( open("../ndlambda/functions/ingest/ingest_event.json").read()) # calling the emulambda function to invoke a lambda emulambda.invoke_lambda(func, event, None, 0, None) # testing if the supercuboid was inserted in the bucket morton_index = XYZMorton(self.tiles) cuboid = self.cuboid_bucket.getObject(nd_proj.channel_name, nd_proj.resolution, morton_index) # testing if the message was removed from the ingest queue for message in self.ingest_queue.receiveMessage(): # KL TODO write the message id into the JSON event file directly print(message)
def test_complete_ingest_job(self): """ Test view to create a new ingest job """ config_data = self.setup_helper.get_ingest_config_data_dict() config_data = json.loads(json.dumps(config_data)) # # Post the data url = '/' + version + '/ingest/' response = self.client.post(url, data=config_data, format='json') assert (response.status_code == 201) # Check if the queue's exist ingest_job = response.json() proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'], 0, ingest_job['id']) self.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) assert (upload_queue is not None) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) assert (ingest_queue is not None) # Test joining the job url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) response = self.client.get(url) assert(response.json()['ingest_job']['id'] == ingest_job['id']) assert("credentials" in response.json()) # Complete the job url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id']) response = self.client.post(url) # Can't complete until it is done assert(response.status_code == 400) # Wait for job to complete for cnt in range(0, 30): # Try joining to kick the status url = '/' + version + '/ingest/{}/'.format(ingest_job['id']) self.client.get(url) url = '/' + version + '/ingest/{}/status'.format(ingest_job['id']) response = self.client.get(url) if response.json()["status"] == 1: break time.sleep(10) # Complete the job url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id']) response = self.client.post(url) assert(response.status_code == 204) # Verify Queues are removed with self.assertRaises(ClientError): UploadQueue(nd_proj, endpoint_url=None) with self.assertRaises(ClientError): IngestQueue(nd_proj, endpoint_url=None) # Verify status has changed url = '/' + version + '/ingest/{}/status'.format(ingest_job['id']) response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["status"], 2)
def teardown_class(self): """Teardown parameters""" IngestQueue.deleteQueue(nd_proj, endpoint_url=self.endpoint_url)
event = json.loads(json_event) # Load the project info from the chunk key you are processing proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"]) proj_info.job_id = event["ingest_job"] # Handle up to 2 messages before quitting (helps deal with making sure all messages get processed) run_cnt = 0 while run_cnt < 1: # Adjusted count down to 1 as lambda is crashing with full memory when pulling off more than 1. # Get message from SQS flush queue, try for ~2 seconds rx_cnt = 0 msg_data = None msg_id = None msg_rx_handle = None while rx_cnt < 6: ingest_queue = IngestQueue(proj_info) msg = [x for x in ingest_queue.receiveMessage()] if msg: msg = msg[0] print("MESSAGE: {}".format(msg)) print(len(msg)) msg_id = msg[0] msg_rx_handle = msg[1] msg_data = json.loads(msg[2]) print("MESSAGE DATA: {}".format(msg_data)) break else: rx_cnt += 1 print("No message found. Try {} of 6".format(rx_cnt)) time.sleep(1)
def handler(event, context): # Load settings SETTINGS = BossSettings.load() # Used as a guard against trying to delete the SQS message when lambda is # triggered by SQS. sqs_triggered = 'Records' in event and len(event['Records']) > 0 if sqs_triggered : # Lambda invoked by an SQS trigger. msg_data = json.loads(event['Records'][0]['body']) # Load the project info from the chunk key you are processing chunk_key = msg_data['chunk_key'] proj_info = BossIngestProj.fromSupercuboidKey(chunk_key) proj_info.job_id = msg_data['ingest_job'] else: # Standard async invoke of this lambda. # Load the project info from the chunk key you are processing proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"]) proj_info.job_id = event["ingest_job"] # Get message from SQS ingest queue, try for ~2 seconds rx_cnt = 0 msg_data = None msg_id = None msg_rx_handle = None while rx_cnt < 6: ingest_queue = IngestQueue(proj_info) msg = [x for x in ingest_queue.receiveMessage()] if msg: msg = msg[0] print("MESSAGE: {}".format(msg)) print(len(msg)) msg_id = msg[0] msg_rx_handle = msg[1] msg_data = json.loads(msg[2]) print("MESSAGE DATA: {}".format(msg_data)) break else: rx_cnt += 1 print("No message found. Try {} of 6".format(rx_cnt)) time.sleep(1) if not msg_id: # No tiles ready to ingest. print("No ingest message available") return # Get the chunk key of the tiles to ingest. chunk_key = msg_data['chunk_key'] tile_error_queue = TileErrorQueue(proj_info) print("Ingesting Chunk {}".format(chunk_key)) tiles_in_chunk = int(chunk_key.split('&')[1]) # Setup SPDB instance sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"], msg_data['parameters']["STATEIO_CONFIG"], msg_data['parameters']["OBJECTIO_CONFIG"]) # Get tile list from Tile Index Table tile_index_db = BossTileIndexDB(proj_info.project_name) # tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk. tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"])) if tile_index_result is None: # If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards. if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to chunk key missing from tile index table") return # Sort the tile keys print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"])) tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()] if len(tile_key_list) < tiles_in_chunk: print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list))) if not sqs_triggered: ingest_queue.deleteMessage(msg_id, msg_rx_handle) return tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1])) tile_key_list = ["&".join(x) for x in tile_key_list] print("Sorted Tile Keys: {}".format(tile_key_list)) # Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits resource_dict = msg_data['parameters']['resource'] _, exp_name, ch_name = resource_dict["boss_key"].split("&") resource_dict["channel"]["name"] = ch_name resource_dict["channel"]["description"] = "" resource_dict["channel"]["sources"] = [] resource_dict["channel"]["related"] = [] resource_dict["channel"]["default_time_sample"] = 0 resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED" resource_dict["experiment"]["name"] = exp_name resource_dict["experiment"]["description"] = "" resource_dict["experiment"]["num_time_samples"] = 1 resource_dict["experiment"]["time_step"] = None resource_dict["experiment"]["time_step_unit"] = None resource_dict["coord_frame"]["name"] = "cf" resource_dict["coord_frame"]["name"] = "" resource_dict["coord_frame"]["x_start"] = 0 resource_dict["coord_frame"]["x_stop"] = 100000 resource_dict["coord_frame"]["y_start"] = 0 resource_dict["coord_frame"]["y_stop"] = 100000 resource_dict["coord_frame"]["z_start"] = 0 resource_dict["coord_frame"]["z_stop"] = 100000 resource_dict["coord_frame"]["voxel_unit"] = "nanometers" # Setup the resource resource = BossResourceBasic() resource.from_dict(resource_dict) dtype = resource.get_numpy_data_type() # read all tiles from bucket into a slab tile_bucket = TileBucket(proj_info.project_name) data = [] num_z_slices = 0 for tile_key in tile_key_list: try: image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key) except KeyError: print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format( tile_key)) if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to missing tile in bucket") return image_bytes = BytesIO(image_data) image_size = image_bytes.getbuffer().nbytes # Get tiles size from metadata, need to shape black tile if actual tile is corrupt. if 'x_size' in metadata: tile_size_x = metadata['x_size'] else: print('MetadataMissing: x_size not in tile metadata: using 1024.') tile_size_x = 1024 if 'y_size' in metadata: tile_size_y = metadata['y_size'] else: print('MetadataMissing: y_size not in tile metadata: using 1024.') tile_size_y = 1024 if image_size == 0: print('TileError: Zero length tile, using black instead: {}'.format(tile_key)) error_msg = 'Zero length tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) else: try: tile_img = np.asarray(Image.open(image_bytes), dtype=dtype) except TypeError as te: print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}' .format(image_size, tile_key)) error_msg = 'Incomplete tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) except OSError as oe: print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}' .format(image_size, tile_key, oe)) error_msg = 'OSError: {}'.format(oe) enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) data.append(tile_img) num_z_slices += 1 # Make 3D array of image data. It should be in XYZ at this point chunk_data = np.array(data) del data tile_dims = chunk_data.shape # Break into Cube instances print("Tile Dims: {}".format(tile_dims)) print("Num Z Slices: {}".format(num_z_slices)) num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0])) num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1])) print("Num X Cuboids: {}".format(num_x_cuboids)) print("Num Y Cuboids: {}".format(num_y_cuboids)) chunk_key_parts = BossUtil.decode_chunk_key(chunk_key) t_index = chunk_key_parts['t_index'] for x_idx in range(0, num_x_cuboids): for y_idx in range(0, num_y_cuboids): # TODO: check time series support cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution]) cube.zeros() # Compute Morton ID # TODO: verify Morton indices correct! print(chunk_key_parts) morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids) morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids) print("Morton X: {}".format(morton_x_ind)) print("Morton Y: {}".format(morton_y_ind)) morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])]) # Insert sub-region from chunk_data into cuboid x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0] x_end = x_start + CUBOIDSIZE[proj_info.resolution][0] x_end = min(x_end, tile_dims[2]) y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1] y_end = y_start + CUBOIDSIZE[proj_info.resolution][1] y_end = min(y_end, tile_dims[1]) z_end = CUBOIDSIZE[proj_info.resolution][2] # TODO: get sub-array w/o making a copy. print("Yrange: {}".format(y_end - y_start)) print("Xrange: {}".format(x_end - x_start)) print("X start: {}".format(x_start)) print("X stop: {}".format(x_end)) cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices, y_start:y_end, x_start:x_end] # Create object key object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index) print("Object Key: {}".format(object_key)) # Put object in S3 sp.objectio.put_objects([object_key], [cube.to_blosc()]) # Add object to index sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"])) # Update id indices if this is an annotation channel # We no longer index during ingest. #if resource.data['channel']['type'] == 'annotation': # try: # sp.objectio.update_id_indices( # resource, proj_info.resolution, [object_key], [cube.data]) # except SpdbError as ex: # sns_client = boto3.client('sns') # topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"] # msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format( # ex.message, # resource.data['collection']['name'], # resource.data['experiment']['name'], # resource.data['channel']['name']) # sns_client.publish( # TopicArn=topic_arn, # Subject='Object services misuse', # Message=msg) lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME) names = AWSNames.create_from_lambda_name(context.function_name) delete_tiles_data = { 'tile_key_list': tile_key_list, 'region': SETTINGS.REGION_NAME, 'bucket': tile_bucket.bucket.name } # Delete tiles from tile bucket. lambda_client.invoke( FunctionName=names.delete_tile_objs_lambda, InvocationType='Event', Payload=json.dumps(delete_tiles_data).encode() ) delete_tile_entry_data = { 'tile_index': tile_index_db.table.name, 'region': SETTINGS.REGION_NAME, 'chunk_key': chunk_key, 'task_id': msg_data['ingest_job'] } # Delete entry from tile index. lambda_client.invoke( FunctionName=names.delete_tile_index_entry_lambda, InvocationType='Event', Payload=json.dumps(delete_tile_entry_data).encode() ) if not sqs_triggered: # Delete message since it was processed successfully ingest_queue.deleteMessage(msg_id, msg_rx_handle)
# to beat the current instance to creating the initial cuboid entry # in the index. error_code = err.response['Error'].get('Code', 'Unknown') if error_code == 'ConditionalCheckFailedException': print('Chunk key entry already created - proceeding.') else: raise chunk_ready = tile_index_db.markTileAsUploaded(metadata["chunk_key"], tile_key, int(metadata["ingest_job"])) # ingest the chunk if we have all the tiles if chunk_ready: print("CHUNK READY SENDING MESSAGE: {}".format(metadata["chunk_key"])) # insert a new job in the insert queue if we have all the tiles ingest_queue = IngestQueue(proj_info) ingest_queue.sendMessage(json.dumps(metadata)) # Invoke Ingest lambda function metadata["lambda-name"] = "ingest" lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME) response = lambda_client.invoke( FunctionName=metadata["parameters"]["ingest_lambda"], InvocationType='Event', Payload=json.dumps(metadata).encode()) else: print("Chunk not ready for ingest yet: {}".format(metadata["chunk_key"])) # Delete message from upload queue upload_queue = UploadQueue(proj_info) upload_queue.deleteMessage(message_id, receipt_handle)
def process(msg, context, region): """ Process a single message. Args: msg (dict): Contents described at the top of the file. context (Context): Lambda context object. region (str): Lambda execution region. """ job_id = int(msg['ingest_job']) chunk_key = msg['chunk_key'] tile_key = msg['tile_key'] print("Tile key: {}".format(tile_key)) proj_info = BossIngestProj.fromTileKey(tile_key) # Set the job id proj_info.job_id = msg['ingest_job'] print("Data: {}".format(msg)) # update value in the dynamo table tile_index_db = BossTileIndexDB(proj_info.project_name) chunk = tile_index_db.getCuboid(chunk_key, job_id) if chunk: if tile_index_db.cuboidReady(chunk_key, chunk["tile_uploaded_map"]): print("Chunk already has all its tiles: {}".format(chunk_key)) # Go ahead and setup to fire another ingest lambda so this tile # entry will be deleted on successful execution of the ingest lambda. chunk_ready = True else: print("Updating tile index for chunk_key: {}".format(chunk_key)) chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id) else: # First tile in the chunk print("Creating first entry for chunk_key: {}".format(chunk_key)) try: tile_index_db.createCuboidEntry(chunk_key, job_id) except ClientError as err: # Under _exceptional_ circumstances, it's possible for another lambda # to beat the current instance to creating the initial cuboid entry # in the index. error_code = err.response['Error'].get('Code', 'Unknown') if error_code == 'ConditionalCheckFailedException': print('Chunk key entry already created - proceeding.') else: raise chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id) # ingest the chunk if we have all the tiles if chunk_ready: print("CHUNK READY SENDING MESSAGE: {}".format(chunk_key)) # insert a new job in the insert queue if we have all the tiles ingest_queue = IngestQueue(proj_info) ingest_queue.sendMessage(json.dumps(msg)) # Invoke Ingest lambda function names = AWSNames.create_from_lambda_name(context.function_name) lambda_client = boto3.client('lambda', region_name=region) lambda_client.invoke( FunctionName=names.tile_ingest_lambda, InvocationType='Event', Payload=json.dumps(msg).encode()) else: print("Chunk not ready for ingest yet: {}".format(chunk_key)) print("DONE!")
def teardown_class(self): """Teardown class parameters""" TileIndexDB.deleteTable(endpoint_url=settings.DYNAMO_ENDPOINT) IngestQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT) UploadQueue.deleteQueue(nd_proj, endpoint_url=settings.SQS_ENDPOINT)
def handler(event, context): # Load settings SETTINGS = BossSettings.load() # Used as a guard against trying to delete the SQS message when lambda is # triggered by SQS. sqs_triggered = 'Records' in event and len(event['Records']) > 0 if sqs_triggered : # Lambda invoked by an SQS trigger. msg_data = json.loads(event['Records'][0]['body']) # Load the project info from the chunk key you are processing chunk_key = msg_data['chunk_key'] proj_info = BossIngestProj.fromSupercuboidKey(chunk_key) proj_info.job_id = msg_data['ingest_job'] else: # Standard async invoke of this lambda. # Load the project info from the chunk key you are processing proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"]) proj_info.job_id = event["ingest_job"] # Get message from SQS ingest queue, try for ~2 seconds rx_cnt = 0 msg_data = None msg_id = None msg_rx_handle = None while rx_cnt < 6: ingest_queue = IngestQueue(proj_info) try: msg = [x for x in ingest_queue.receiveMessage()] # StopIteration may be converted to a RunTimeError. except (StopIteration, RuntimeError): msg = None if msg: msg = msg[0] print("MESSAGE: {}".format(msg)) print(len(msg)) msg_id = msg[0] msg_rx_handle = msg[1] msg_data = json.loads(msg[2]) print("MESSAGE DATA: {}".format(msg_data)) break else: rx_cnt += 1 print("No message found. Try {} of 6".format(rx_cnt)) time.sleep(1) if not msg_id: # No tiles ready to ingest. print("No ingest message available") return # Get the chunk key of the tiles to ingest. chunk_key = msg_data['chunk_key'] tile_error_queue = TileErrorQueue(proj_info) print("Ingesting Chunk {}".format(chunk_key)) tiles_in_chunk = int(chunk_key.split('&')[1]) # Setup SPDB instance sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"], msg_data['parameters']["STATEIO_CONFIG"], msg_data['parameters']["OBJECTIO_CONFIG"]) # Get tile list from Tile Index Table tile_index_db = BossTileIndexDB(proj_info.project_name) # tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk. tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"])) if tile_index_result is None: # If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards. if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to chunk key missing from tile index table") return # Sort the tile keys print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"])) tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()] if len(tile_key_list) < tiles_in_chunk: print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list))) if not sqs_triggered: ingest_queue.deleteMessage(msg_id, msg_rx_handle) return tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1])) tile_key_list = ["&".join(x) for x in tile_key_list] print("Sorted Tile Keys: {}".format(tile_key_list)) # Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits resource_dict = msg_data['parameters']['resource'] _, exp_name, ch_name = resource_dict["boss_key"].split("&") resource_dict["channel"]["name"] = ch_name resource_dict["channel"]["description"] = "" resource_dict["channel"]["sources"] = [] resource_dict["channel"]["related"] = [] resource_dict["channel"]["default_time_sample"] = 0 resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED" resource_dict["experiment"]["name"] = exp_name resource_dict["experiment"]["description"] = "" resource_dict["experiment"]["num_time_samples"] = 1 resource_dict["experiment"]["time_step"] = None resource_dict["experiment"]["time_step_unit"] = None resource_dict["coord_frame"]["name"] = "cf" resource_dict["coord_frame"]["name"] = "" resource_dict["coord_frame"]["x_start"] = 0 resource_dict["coord_frame"]["x_stop"] = 100000 resource_dict["coord_frame"]["y_start"] = 0 resource_dict["coord_frame"]["y_stop"] = 100000 resource_dict["coord_frame"]["z_start"] = 0 resource_dict["coord_frame"]["z_stop"] = 100000 resource_dict["coord_frame"]["voxel_unit"] = "nanometers" # Setup the resource resource = BossResourceBasic() resource.from_dict(resource_dict) dtype = resource.get_numpy_data_type() # read all tiles from bucket into a slab tile_bucket = TileBucket(proj_info.project_name) data = [] num_z_slices = 0 for tile_key in tile_key_list: try: image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key) except KeyError: print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format( tile_key)) if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to missing tile in bucket") return image_bytes = BytesIO(image_data) image_size = image_bytes.getbuffer().nbytes # Get tiles size from metadata, need to shape black tile if actual tile is corrupt. if 'x_size' in metadata: tile_size_x = metadata['x_size'] else: print('MetadataMissing: x_size not in tile metadata: using 1024.') tile_size_x = 1024 if 'y_size' in metadata: tile_size_y = metadata['y_size'] else: print('MetadataMissing: y_size not in tile metadata: using 1024.') tile_size_y = 1024 if image_size == 0: print('TileError: Zero length tile, using black instead: {}'.format(tile_key)) error_msg = 'Zero length tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) else: try: # DP NOTE: Issues when specifying dtype in the asarray function with Pillow ver 8.3.1. # Fixed by separating array instantiation and dtype assignment. tile_img = np.asarray(Image.open(image_bytes)) tile_img = tile_img.astype(dtype) except TypeError as te: print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}' .format(image_size, tile_key)) error_msg = 'Incomplete tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) except OSError as oe: print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}' .format(image_size, tile_key, oe)) error_msg = 'OSError: {}'.format(oe) enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) data.append(tile_img) num_z_slices += 1 # Make 3D array of image data. It should be in XYZ at this point chunk_data = np.array(data) del data tile_dims = chunk_data.shape # Break into Cube instances print("Tile Dims: {}".format(tile_dims)) print("Num Z Slices: {}".format(num_z_slices)) num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0])) num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1])) print("Num X Cuboids: {}".format(num_x_cuboids)) print("Num Y Cuboids: {}".format(num_y_cuboids)) chunk_key_parts = BossUtil.decode_chunk_key(chunk_key) t_index = chunk_key_parts['t_index'] for x_idx in range(0, num_x_cuboids): for y_idx in range(0, num_y_cuboids): # TODO: check time series support cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution]) cube.zeros() # Compute Morton ID # TODO: verify Morton indices correct! print(chunk_key_parts) morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids) morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids) print("Morton X: {}".format(morton_x_ind)) print("Morton Y: {}".format(morton_y_ind)) morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])]) # Insert sub-region from chunk_data into cuboid x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0] x_end = x_start + CUBOIDSIZE[proj_info.resolution][0] x_end = min(x_end, tile_dims[2]) y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1] y_end = y_start + CUBOIDSIZE[proj_info.resolution][1] y_end = min(y_end, tile_dims[1]) z_end = CUBOIDSIZE[proj_info.resolution][2] # TODO: get sub-array w/o making a copy. print("Yrange: {}".format(y_end - y_start)) print("Xrange: {}".format(x_end - x_start)) print("X start: {}".format(x_start)) print("X stop: {}".format(x_end)) cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices, y_start:y_end, x_start:x_end] # Create object key object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index) print("Object Key: {}".format(object_key)) # Put object in S3 sp.objectio.put_objects([object_key], [cube.to_blosc()]) # Add object to index sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"])) # Update id indices if this is an annotation channel # We no longer index during ingest. #if resource.data['channel']['type'] == 'annotation': # try: # sp.objectio.update_id_indices( # resource, proj_info.resolution, [object_key], [cube.data]) # except SpdbError as ex: # sns_client = boto3.client('sns') # topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"] # msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format( # ex.message, # resource.data['collection']['name'], # resource.data['experiment']['name'], # resource.data['channel']['name']) # sns_client.publish( # TopicArn=topic_arn, # Subject='Object services misuse', # Message=msg) lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME) names = AWSNames.from_lambda(context.function_name) delete_tiles_data = { 'tile_key_list': tile_key_list, 'region': SETTINGS.REGION_NAME, 'bucket': tile_bucket.bucket.name } # Delete tiles from tile bucket. lambda_client.invoke( FunctionName=names.delete_tile_objs.lambda_, InvocationType='Event', Payload=json.dumps(delete_tiles_data).encode() ) delete_tile_entry_data = { 'tile_index': tile_index_db.table.name, 'region': SETTINGS.REGION_NAME, 'chunk_key': chunk_key, 'task_id': msg_data['ingest_job'] } # Delete entry from tile index. lambda_client.invoke( FunctionName=names.delete_tile_index_entry.lambda_, InvocationType='Event', Payload=json.dumps(delete_tile_entry_data).encode() ) if not sqs_triggered: # Delete message since it was processed successfully ingest_queue.deleteMessage(msg_id, msg_rx_handle)