def test_setup_ingest(self):
"""Method to test the setup_ingest method"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.user.id,
self.example_config_data)
assert (ingest_job is not None)
# Check if the queue's exist
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job.collection, ingest_job.experiment,
ingest_job.channel, ingest_job.resolution,
ingest_job.id)
ingest_mgmr.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
ingest_mgmr.remove_ingest_credentials(ingest_job.id)
except:
raise
finally:
ingest_mgmr.delete_upload_queue()
ingest_mgmr.delete_ingest_queue()
def test_not_creator_admin(self):
"""Method to test only creators or admins can interact with ingest jobs"""
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
assert (response.status_code == 201)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
# Log in as the admin and create a job
self.client.force_login(self.superuser)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
assert (response.status_code == 200)
assert (response.json()['ingest_job']['id'] == ingest_job['id'])
assert("credentials" in response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
assert (response.status_code == 204)
def test_not_creator_admin(self):
"""Method to test only creators or admins can interact with ingest jobs"""
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(201, response.status_code)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Log in as the admin and create a job
self.client.force_login(self.superuser)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(200, response.status_code)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
self.assertEqual(204, response.status_code)
def test_generate_upload_tasks(self):
""""""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data)
ingest_mgmr.generate_upload_tasks(ingest_job.id)
assert (ingest_job.collection == 'my_col_1')
assert (ingest_job.experiment == 'my_exp_1')
assert (ingest_job.channel == 'my_ch_1')
# Pull the messages off the queue
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = UploadQueue(nd_proj, endpoint_url=None)
tmp = queue.receiveMessage(number_of_messages=10)
# receive message from the queue
for message_id, receipt_handle, message_body in tmp:
assert(message_body['job_id'] == ingest_job.id)
# delete message from the queue
response = queue.deleteMessage(message_id, receipt_handle)
assert ('Successful' in response)
ingest_mgmr.remove_ingest_credentials(ingest_job.id)
except:
raise
finally:
ingest_mgmr.delete_upload_queue()
ingest_mgmr.delete_ingest_queue()
def test_post_new_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
assert (response.status_code == 201)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'],
ingest_job['experiment'],
ingest_job['channel_layer'], 0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
assert (response.status_code == 204)
def setup_ingest(self, creator, config_data):
"""
Setup the ingest job. This is the primary method for the ingest manager.
It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest
Args:
creator: The validated user from the request to create the ingest jon
config_data : Config data to create the ingest job
Returns:
IngestJob : data model containing the ingest job
Raises:
BossError : For all exceptions that happen
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name,
self.experiment.name,
self.channel.name, self.resolution,
self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
self.generate_upload_tasks()
tile_bucket = TileBucket(self.job.collection + '&' +
self.job.experiment)
self.create_ingest_credentials(upload_queue, tile_bucket)
# TODO create channel if needed
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError(
"Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
def test_complete_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(response.status_code, 201)
# Check if the queues exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'],
ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
# Can't complete until it is done
self.assertEqual(400, response.status_code)
# Wait for job to complete
print('trying to join job')
for cnt in range(0, 30):
# Try joining to kick the status
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
self.client.get(url)
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
if response.json()["status"] == IngestJob.UPLOADING:
break
time.sleep(10)
print('completing')
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
self.assertEqual(204, response.status_code)
def setup_ingest(self, creator, config_data):
"""
Args:
Returns:
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name,
self.experiment.name,
self.channel_layer.name,
self.resolution, self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
self.generate_upload_tasks()
tile_bucket = TileBucket(self.job.collection + '&' +
self.job.experiment)
self.create_ingest_credentials(upload_queue, tile_bucket)
# Update status
self.job.status = 1
self.job.save()
# TODO create channel if needed
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError(
"Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
def test_complete_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(response.status_code, 201)
# Check if the queues exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
# Can't complete until it is done
self.assertEqual(400, response.status_code)
# Wait for job to complete
print('trying to join job')
for cnt in range(0, 30):
# Try joining to kick the status
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
self.client.get(url)
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
if response.json()["status"] == IngestJob.UPLOADING:
break
time.sleep(10)
print('completing')
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
self.assertEqual(204, response.status_code)
def setUpClass(cls):
# Suppress warnings about Boto3's unclosed sockets.
warnings.simplefilter('ignore')
# Use ndingest in test mode.
os.environ['NDINGEST_TEST'] = '1'
cls.job_id = 125
cls.nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0,
cls.job_id)
UploadQueue.createQueue(cls.nd_proj)
cls.upload_queue = UploadQueue(cls.nd_proj)
def get_ingest_job_tile_error_queue(self, ingest_job):
"""
Return the tile index queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
ndingest.TileIndexQueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = TileErrorQueue(self.nd_proj, endpoint_url=None)
return queue
def get_ingest_job_upload_queue(self, ingest_job):
"""
Return the upload queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
Ndingest.uploadqueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = UploadQueue(self.nd_proj, endpoint_url=None)
return queue
def get_ingest_job_tile_error_queue(self, ingest_job):
"""
Return the tile index queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
ndingest.TileIndexQueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = TileErrorQueue(self.nd_proj, endpoint_url=None)
return queue
def get_ingest_job_ingest_queue(self, ingest_job):
"""
Return the ingest queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
Ndingest.ingestqueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = IngestQueue(self.nd_proj, endpoint_url=None)
return queue
def cleanup_ingest_job(self, ingest_job, job_status):
"""
Delete or complete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
in the tile bucket for this job id. It does not delete the ingest job datamodel but changes its state.
Args:
ingest_job: Ingest job to cleanup
job_status(int): Status to update to
Returns:
(int): ingest job id for the job that was successfully deleted
Raises:
BossError : If the the job id is not valid or any exception happens in deletion process
"""
try:
# cleanup ingest job
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection,
ingest_job.experiment,
ingest_job.channel,
ingest_job.resolution, ingest_job.id)
# delete the ingest and upload_queue
self.delete_upload_queue()
if ingest_job.ingest_type != IngestJob.VOLUMETRIC_INGEST:
self.delete_ingest_queue()
# delete any pending entries in the tile index database and tile bucket
# Commented out due to removal of tile index's GSI.
# self.delete_tiles(ingest_job)
ingest_job.status = job_status
ingest_job.ingest_queue = None
ingest_job.upload_queue = None
ingest_job.end_date = timezone.now()
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job.id)
except Exception as e:
raise BossError("Unable to cleanup the upload queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(ingest_job.id),
ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job.id
def delete_ingest_job(self, ingest_job_id):
"""
Delete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
in the tile bucket for this job id. It does not delete the ingest job datamodel but marks it as deleted.
Args:
ingest_job_id: Ingest job id to delete
Returns:
Int : ingest job id for the job that was successfully deleted
Raises:
BossError : If the the job id is not valid or any exception happens in deletion process
"""
try:
# delete ingest job
ingest_job = IngestJob.objects.get(id=ingest_job_id)
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection,
ingest_job.experiment,
ingest_job.channel,
ingest_job.resolution, ingest_job.id)
# delete the ingest and upload_queue
self.delete_upload_queue()
self.delete_ingest_queue()
# delete any pending entries in the tile index database and tile bucket
self.delete_tiles(ingest_job)
ingest_job.status = 3
ingest_job.ingest_queue = None
ingest_job.upload_queue = None
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job_id)
except Exception as e:
raise BossError("Unable to delete the upload queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(ingest_job_id),
ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job_id
def cleanup_ingest_job(self, ingest_job, job_status):
"""
Delete or complete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
in the tile bucket for this job id. It does not delete the ingest job datamodel but changes its state.
Args:
ingest_job: Ingest job to cleanup
job_status(int): Status to update to
Returns:
(int): ingest job id for the job that was successfully deleted
Raises:
BossError : If the the job id is not valid or any exception happens in deletion process
"""
try:
# cleanup ingest job
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
# delete the queues
self.delete_upload_queue()
if ingest_job.ingest_type != IngestJob.VOLUMETRIC_INGEST:
self.delete_ingest_queue()
self.delete_tile_index_queue()
self.delete_tile_error_queue()
# delete any pending entries in the tile index database and tile bucket
# Commented out due to removal of tile index's GSI.
# self.delete_tiles(ingest_job)
ingest_job.status = job_status
ingest_job.ingest_queue = None
ingest_job.upload_queue = None
ingest_job.end_date = timezone.now()
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job.id)
except Exception as e:
raise BossError("Unable to cleanup the upload queue.{}".format(e), ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError("Ingest job with id {} does not exist".format(ingest_job.id), ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job.id
def test_post_new_volumetric_ingest_job(self):
""" Test view to create a new volumetric_ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict_volumetric(
)
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(201, response.status_code)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'],
ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
# There shouldn't be an ingest queue for a volumetric ingest
with self.assertRaises(ClientError):
IngestQueue(nd_proj, endpoint_url=None)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
self.assertEqual(204, response.status_code)
# Verify Queues are removed
with self.assertRaises(ClientError):
UploadQueue(nd_proj, endpoint_url=None)
# Verify the job is deleted
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
def enqueue_msgs(fp):
"""Parse given messages and send to SQS queue.
Args:
fp (file-like-object): File-like-object containing a header and messages.
"""
read_header = False
msgs = []
upload_queue = None
lineNum = 0
for line in fp:
lineNum += 1
if not read_header:
header = json.loads(line)
if 'upload_queue_url' not in header:
raise KeyError('Expected upload_queue_url in header')
if 'ingest_queue_url' not in header:
raise KeyError('Expected ingest_queue_url in header')
if 'job_id' not in header:
raise KeyError('Expected job_id in header')
read_header = True
continue
try:
msgs.append(parse_line(header, line))
except:
print('Error parsing line {}: {}'.format(lineNum, line))
if len(msgs) == 1 and upload_queue is None:
# Instantiate the upload queue object.
asDict = json.loads(msgs[0])
boss_ingest_proj = BossIngestProj.fromTileKey(asDict['tile_key'])
boss_ingest_proj.job_id = header['job_id']
upload_queue = UploadQueue(boss_ingest_proj)
if len(msgs) >= MAX_BATCH_MSGS:
# Enqueue messages.
upload_queue.sendBatchMessages(msgs)
msgs = []
if len(msgs) > 0:
# Final enqueue messages of remaining messages.
upload_queue.sendBatchMessages(msgs)
def enqueue_msgs(fp):
"""Parse given messages and send to SQS queue.
Args:
fp (file-like-object): File-like-object containing a header and messages.
"""
read_header = False
msgs = []
upload_queue = None
lineNum = 0
for line in fp:
lineNum += 1
if not read_header:
header = json.loads(line)
if 'upload_queue_url' not in header:
raise KeyError('Expected upload_queue_url in header')
if 'ingest_queue_url' not in header:
raise KeyError('Expected ingest_queue_url in header')
if 'job_id' not in header:
raise KeyError('Expected job_id in header')
read_header = True
continue
try:
msgs.append(parse_line(header, line))
except:
print('Error parsing line {}: {}'.format(lineNum, line))
if len(msgs) == 1 and upload_queue is None:
# Instantiate the upload queue object.
asDict = json.loads(msgs[0])
boss_ingest_proj = BossIngestProj.fromTileKey(asDict['tile_key'])
boss_ingest_proj.job_id = header['job_id']
upload_queue = UploadQueue(boss_ingest_proj)
if len(msgs) >= MAX_BATCH_MSGS:
# Enqueue messages.
upload_queue.sendBatchMessages(msgs)
msgs = []
if len(msgs) > 0:
# Final enqueue messages of remaining messages.
upload_queue.sendBatchMessages(msgs)
def test_post_new_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(201, response.status_code)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
self.assertEqual(204, response.status_code)
# Verify Queues are removed
with self.assertRaises(ClientError):
UploadQueue(nd_proj, endpoint_url=None)
with self.assertRaises(ClientError):
IngestQueue(nd_proj, endpoint_url=None)
# Verify the job is deleted
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
def setUpClass(cls):
# Silence warnings about open boto3 sessions.
warnings.filterwarnings('ignore')
cls.job_id = 123
cls.nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0,
cls.job_id)
TileBucket.createBucket()
cls.tile_bucket = TileBucket(cls.nd_proj.project_name)
warnings.simplefilter('ignore')
#with open('/Users/manavpj1/repos/boss/django/bossingest/test/boss_tile_index.json') as fp:
# schema = json.load(fp)
#BossTileIndexDB.createTable(schema, endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
cls.tileindex_db = BossTileIndexDB(
cls.nd_proj.project_name,
endpoint_url=settings.DYNAMO_TEST_ENDPOINT)
def boss_util_fixtures(tile_bucket, sqs):
job_id = 123
nd_proj = BossIngestProj("testCol", "kasthuri11", "image", 0, job_id)
from ndingest.ndqueue.uploadqueue import UploadQueue
UploadQueue.createQueue(nd_proj)
upload_queue = UploadQueue(nd_proj)
from ndingest.ndqueue.tileindexqueue import TileIndexQueue
TileIndexQueue.createQueue(nd_proj)
tile_index_queue = TileIndexQueue(nd_proj)
def get_test_data():
return (nd_proj, upload_queue, tile_index_queue, tile_bucket)
yield get_test_data
UploadQueue.deleteQueue(nd_proj)
TileIndexQueue.deleteQueue(nd_proj)
def delete_ingest_job(self, ingest_job_id):
"""
Args:
ingest_job_id:
Returns:
"""
try:
# delete ingest job
ingest_job = IngestJob.objects.get(id=ingest_job_id)
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection,
ingest_job.experiment,
ingest_job.channel_layer,
ingest_job.resolution, ingest_job.id)
# delete the ingest and upload_queue
self.delete_upload_queue()
self.delete_ingest_queue()
# delete any pending entries in the tile index database and tile bucket
self.delete_tiles(ingest_job)
ingest_job.status = 3
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job_id)
except Exception as e:
raise BossError("Unable to delete the upload queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(ingest_job_id),
ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job_id
def __init__(self, status, db_host, job):
"""
Args:
status (str): Mark ingest job as 'deleted' or 'complete'.
db_host (str): Host of MySQL database.
job (dict):
collection (int): Collection id.
experiment (int): Experiment id.
channel (int): Channel id.
task_id (int): The ingest job's id.
resolution (int): Resolution of chunk.
ingest_type (int): Tile (0) or volumetric ingest (1).
"""
self._status_map = {
COMPLETE_STATUS: COMPLETE_DB,
DELETED_STATUS: DELETED_DB
}
self.db_host = db_host
self.job = job
# Validate job parameter.
for field in IngestCleaner.JOB_FIELDS:
if field not in job:
raise KeyError('Job must have {}'.format(field))
# Validate status parameter.
self.status = status
if status.lower() not in IngestCleaner.STATUS_VALUES:
raise BadStatusError('{} is not a valid status.'.format(status))
proj_class = BossIngestProj.load()
# coll/exp/chan are specified as strings, but not necessarily to give
# actual names when using ndingest's delete functionality.
self.nd_proj = proj_class(job['collection'], job['experiment'],
job['channel'], job['resolution'],
job['task_id'])
def setup_ingest(self, creator, config_data):
"""
Setup the ingest job. This is the primary method for the ingest manager.
It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest
Args:
creator: The validated user from the request to create the ingest jon
config_data : Config data to create the ingest job
Returns:
IngestJob : data model containing the ingest job
Raises:
BossError : For all exceptions that happen
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name, self.experiment.name, self.channel.name,
self.resolution, self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
if self.job.ingest_type == IngestJob.TILE_INGEST:
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
tile_index_queue = self.create_tile_index_queue()
self.add_trigger_tile_uploaded_lambda_from_queue(tile_index_queue.arn)
self.create_tile_error_queue()
elif self.job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# Will the management console be ok with ingest_queue being null?
pass
# Call the step function to populate the queue.
self.job.step_function_arn = self.populate_upload_queue(self.job)
# Compute # of tiles or chunks in the job
x_extent = self.job.x_stop - self.job.x_start
y_extent = self.job.y_stop - self.job.y_start
z_extent = self.job.z_stop - self.job.z_start
t_extent = self.job.t_stop - self.job.t_start
num_tiles_in_x = math.ceil(x_extent/self.job.tile_size_x)
num_tiles_in_y = math.ceil(y_extent/self.job.tile_size_y)
num_tiles_in_z = math.ceil(z_extent/self.job.tile_size_z)
num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
self.job.save()
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError("Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import absolute_import
import sys
sys.path.append('..')
from ndingest.settings.settings import Settings
settings = Settings.load()
from ndingest.nddynamo.boss_tileindexdb import BossTileIndexDB
from ndingest.ndingestproj.bossingestproj import BossIngestProj
job_id = '123'
nd_proj = BossIngestProj('testCol', 'kasthuri11', 'image', 0, job_id)
import json
import six
import unittest
import warnings
import time
import botocore
class Test_BossTileIndexDB(unittest.TestCase):
"""
Note that the chunk keys used, for testing, do not have real hash keys.
The rest of the chunk key is valid.
"""
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
def populate_upload_queue(args):
"""Populate the ingest upload SQS Queue with tile information
Note: This activity will clear the upload queue of any existing
messages
Args:
args: {
'job_id': '',
'upload_queue': ARN,
'ingest_queue': ARN,
'collection_name': '',
'experiment_name': '',
'channel_name': '',
'resolution': 0,
'project_info': [col_id, exp_id, ch_id],
't_start': 0,
't_stop': 0,
't_tile_size': 0,
'x_start': 0,
'x_stop': 0,
'x_tile_size': 0,
'y_start': 0,
'y_stop': 0
'y_tile_size': 0,
'z_start': 0,
'z_stop': 0
'z_tile_size': 16,
}
Returns:
{'arn': Upload queue ARN,
'count': Number of messages put into the queue}
"""
log.debug("Starting to populate upload queue")
# DP NOTE: Could transform create_messages into a generator
# and yield after each sendBatchMessages
clear_queue(args['upload_queue'])
proj = BossIngestProj(args['collection_name'],
args['experiment_name'],
args['channel_name'],
args['resolution'],
args['job_id'])
queue = UploadQueue(proj)
msgs = create_messages(args)
sent = 0
# Implement a custom queue.sendBatchMessages so that more than 10
# messages can be sent at once (hard limit in sendBatchMessages)
while True:
batch = []
for i in range(SQS_BATCH_SIZE):
try:
batch.append({
'Id': str(i),
'MessageBody': next(msgs),
'DelaySeconds': 0
})
except StopIteration:
break
if len(batch) == 0:
break
retry = 3
while retry > 0:
resp = queue.queue.send_messages(Entries=batch)
sent += len(resp['Successful'])
if 'Failed' in resp and len(resp['Failed']) > 0:
log.debug("Batch failed to enqueue messages")
log.debug("Retries left: {}".format(retry))
log.debug("Boto3 send_messages response: {}".format(resp))
time.sleep(SQS_RETRY_TIMEOUT)
ids = [f['Id'] for f in resp['Failed']]
batch = [b for b in batch if b['Id'] in ids]
retry -= 1
if retry == 0:
log.debug("Exhausted retry count, stopping")
raise FailedToSendMessages(batch) # SFN will relaunch the activity
continue
else:
break
return {
'arn': args['upload_queue'],
'count': sent,
}
SETTINGS = BossSettings.load()
# Parse input args passed as a JSON string from the lambda loader
json_event = sys.argv[1]
event = json.loads(json_event)
print(event)
# extract bucket name and tile key from the event
bucket = event['Records'][0]['s3']['bucket']['name']
tile_key = urllib.parse.unquote_plus(
event['Records'][0]['s3']['object']['key'])
print("Bucket: {}".format(bucket))
print("Tile key: {}".format(tile_key))
# fetch metadata from the s3 object
proj_info = BossIngestProj.fromTileKey(tile_key)
tile_bucket = TileBucket(proj_info.project_name)
message_id, receipt_handle, metadata = tile_bucket.getMetadata(tile_key)
print("Metadata: {}".format(metadata))
# Currently this is what is sent from the client for the "metadata"
# metadata = {'chunk_key': 'chunk_key',
# 'ingest_job': self.ingest_job_id,
# 'parameters': {"upload_queue": XX
# "ingest_queue": XX,
# "ingest_lambda":XX,
# "KVIO_SETTINGS": XX,
# "STATEIO_CONFIG": XX,
# "OBJECTIO_CONFIG": XX
# },
# 'tile_size_x': "{}".format(self.config.config_data["ingest_job"]["tile_size"]["x"]),
def test_complete_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
assert (response.status_code == 201)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
assert(response.json()['ingest_job']['id'] == ingest_job['id'])
assert("credentials" in response.json())
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
# Can't complete until it is done
assert(response.status_code == 400)
# Wait for job to complete
for cnt in range(0, 30):
# Try joining to kick the status
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
self.client.get(url)
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
if response.json()["status"] == 1:
break
time.sleep(10)
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
assert(response.status_code == 204)
# Verify Queues are removed
with self.assertRaises(ClientError):
UploadQueue(nd_proj, endpoint_url=None)
with self.assertRaises(ClientError):
IngestQueue(nd_proj, endpoint_url=None)
# Verify status has changed
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()["status"], 2)
from io import BytesIO
from PIL import Image
import numpy as np
import math
import boto3
print("$$$ IN INGEST LAMBDA $$$")
# Load settings
SETTINGS = BossSettings.load()
# Parse input args passed as a JSON string from the lambda loader
json_event = sys.argv[1]
event = json.loads(json_event)
# Load the project info from the chunk key you are processing
proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"])
proj_info.job_id = event["ingest_job"]
# Handle up to 2 messages before quitting (helps deal with making sure all messages get processed)
run_cnt = 0
while run_cnt < 1: # Adjusted count down to 1 as lambda is crashing with full memory when pulling off more than 1.
# Get message from SQS flush queue, try for ~2 seconds
rx_cnt = 0
msg_data = None
msg_id = None
msg_rx_handle = None
while rx_cnt < 6:
ingest_queue = IngestQueue(proj_info)
msg = [x for x in ingest_queue.receiveMessage()]
if msg:
msg = msg[0]
def process(msg, context, region):
"""
Process a single message.
Args:
msg (dict): Contents described at the top of the file.
context (Context): Lambda context object.
region (str): Lambda execution region.
"""
job_id = int(msg['ingest_job'])
chunk_key = msg['chunk_key']
tile_key = msg['tile_key']
print("Tile key: {}".format(tile_key))
proj_info = BossIngestProj.fromTileKey(tile_key)
# Set the job id
proj_info.job_id = msg['ingest_job']
print("Data: {}".format(msg))
# update value in the dynamo table
tile_index_db = BossTileIndexDB(proj_info.project_name)
chunk = tile_index_db.getCuboid(chunk_key, job_id)
if chunk:
if tile_index_db.cuboidReady(chunk_key, chunk["tile_uploaded_map"]):
print("Chunk already has all its tiles: {}".format(chunk_key))
# Go ahead and setup to fire another ingest lambda so this tile
# entry will be deleted on successful execution of the ingest lambda.
chunk_ready = True
else:
print("Updating tile index for chunk_key: {}".format(chunk_key))
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
else:
# First tile in the chunk
print("Creating first entry for chunk_key: {}".format(chunk_key))
try:
tile_index_db.createCuboidEntry(chunk_key, job_id)
except ClientError as err:
# Under _exceptional_ circumstances, it's possible for another lambda
# to beat the current instance to creating the initial cuboid entry
# in the index.
error_code = err.response['Error'].get('Code', 'Unknown')
if error_code == 'ConditionalCheckFailedException':
print('Chunk key entry already created - proceeding.')
else:
raise
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
# ingest the chunk if we have all the tiles
if chunk_ready:
print("CHUNK READY SENDING MESSAGE: {}".format(chunk_key))
# insert a new job in the insert queue if we have all the tiles
ingest_queue = IngestQueue(proj_info)
ingest_queue.sendMessage(json.dumps(msg))
# Invoke Ingest lambda function
names = AWSNames.create_from_lambda_name(context.function_name)
lambda_client = boto3.client('lambda', region_name=region)
lambda_client.invoke(
FunctionName=names.tile_ingest_lambda,
InvocationType='Event',
Payload=json.dumps(msg).encode())
else:
print("Chunk not ready for ingest yet: {}".format(chunk_key))
print("DONE!")
def process(msg, context, region):
"""
Process a single message.
Args:
msg (dict): Contents described at the top of the file.
context (Context): Lambda context object.
region (str): Lambda execution region.
"""
job_id = int(msg['ingest_job'])
chunk_key = msg['chunk_key']
tile_key = msg['tile_key']
print("Tile key: {}".format(tile_key))
proj_info = BossIngestProj.fromTileKey(tile_key)
# Set the job id
proj_info.job_id = msg['ingest_job']
print("Data: {}".format(msg))
# update value in the dynamo table
tile_index_db = BossTileIndexDB(proj_info.project_name)
chunk = tile_index_db.getCuboid(chunk_key, job_id)
if chunk:
if tile_index_db.cuboidReady(chunk_key, chunk["tile_uploaded_map"]):
print("Chunk already has all its tiles: {}".format(chunk_key))
# Go ahead and setup to fire another ingest lambda so this tile
# entry will be deleted on successful execution of the ingest lambda.
chunk_ready = True
else:
print("Updating tile index for chunk_key: {}".format(chunk_key))
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
else:
# First tile in the chunk
print("Creating first entry for chunk_key: {}".format(chunk_key))
try:
tile_index_db.createCuboidEntry(chunk_key, job_id)
except ClientError as err:
# Under _exceptional_ circumstances, it's possible for another lambda
# to beat the current instance to creating the initial cuboid entry
# in the index.
error_code = err.response['Error'].get('Code', 'Unknown')
if error_code == 'ConditionalCheckFailedException':
print('Chunk key entry already created - proceeding.')
else:
raise
chunk_ready = tile_index_db.markTileAsUploaded(chunk_key, tile_key, job_id)
# ingest the chunk if we have all the tiles
if chunk_ready:
print("CHUNK READY SENDING MESSAGE: {}".format(chunk_key))
# insert a new job in the insert queue if we have all the tiles
ingest_queue = IngestQueue(proj_info)
ingest_queue.sendMessage(json.dumps(msg))
# Invoke Ingest lambda function
names = AWSNames.from_lambda(context.function_name)
lambda_client = boto3.client('lambda', region_name=region)
lambda_client.invoke(
FunctionName=names.tile_ingest.lambda_,
InvocationType='Event',
Payload=json.dumps(msg).encode())
else:
print("Chunk not ready for ingest yet: {}".format(chunk_key))
print("DONE!")
def handler(event, context):
# Load settings
SETTINGS = BossSettings.load()
# Used as a guard against trying to delete the SQS message when lambda is
# triggered by SQS.
sqs_triggered = 'Records' in event and len(event['Records']) > 0
if sqs_triggered :
# Lambda invoked by an SQS trigger.
msg_data = json.loads(event['Records'][0]['body'])
# Load the project info from the chunk key you are processing
chunk_key = msg_data['chunk_key']
proj_info = BossIngestProj.fromSupercuboidKey(chunk_key)
proj_info.job_id = msg_data['ingest_job']
else:
# Standard async invoke of this lambda.
# Load the project info from the chunk key you are processing
proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"])
proj_info.job_id = event["ingest_job"]
# Get message from SQS ingest queue, try for ~2 seconds
rx_cnt = 0
msg_data = None
msg_id = None
msg_rx_handle = None
while rx_cnt < 6:
ingest_queue = IngestQueue(proj_info)
try:
msg = [x for x in ingest_queue.receiveMessage()]
# StopIteration may be converted to a RunTimeError.
except (StopIteration, RuntimeError):
msg = None
if msg:
msg = msg[0]
print("MESSAGE: {}".format(msg))
print(len(msg))
msg_id = msg[0]
msg_rx_handle = msg[1]
msg_data = json.loads(msg[2])
print("MESSAGE DATA: {}".format(msg_data))
break
else:
rx_cnt += 1
print("No message found. Try {} of 6".format(rx_cnt))
time.sleep(1)
if not msg_id:
# No tiles ready to ingest.
print("No ingest message available")
return
# Get the chunk key of the tiles to ingest.
chunk_key = msg_data['chunk_key']
tile_error_queue = TileErrorQueue(proj_info)
print("Ingesting Chunk {}".format(chunk_key))
tiles_in_chunk = int(chunk_key.split('&')[1])
# Setup SPDB instance
sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"],
msg_data['parameters']["STATEIO_CONFIG"],
msg_data['parameters']["OBJECTIO_CONFIG"])
# Get tile list from Tile Index Table
tile_index_db = BossTileIndexDB(proj_info.project_name)
# tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk.
tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"]))
if tile_index_result is None:
# If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards.
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to chunk key missing from tile index table")
return
# Sort the tile keys
print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"]))
tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()]
if len(tile_key_list) < tiles_in_chunk:
print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list)))
if not sqs_triggered:
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
return
tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1]))
tile_key_list = ["&".join(x) for x in tile_key_list]
print("Sorted Tile Keys: {}".format(tile_key_list))
# Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits
resource_dict = msg_data['parameters']['resource']
_, exp_name, ch_name = resource_dict["boss_key"].split("&")
resource_dict["channel"]["name"] = ch_name
resource_dict["channel"]["description"] = ""
resource_dict["channel"]["sources"] = []
resource_dict["channel"]["related"] = []
resource_dict["channel"]["default_time_sample"] = 0
resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED"
resource_dict["experiment"]["name"] = exp_name
resource_dict["experiment"]["description"] = ""
resource_dict["experiment"]["num_time_samples"] = 1
resource_dict["experiment"]["time_step"] = None
resource_dict["experiment"]["time_step_unit"] = None
resource_dict["coord_frame"]["name"] = "cf"
resource_dict["coord_frame"]["name"] = ""
resource_dict["coord_frame"]["x_start"] = 0
resource_dict["coord_frame"]["x_stop"] = 100000
resource_dict["coord_frame"]["y_start"] = 0
resource_dict["coord_frame"]["y_stop"] = 100000
resource_dict["coord_frame"]["z_start"] = 0
resource_dict["coord_frame"]["z_stop"] = 100000
resource_dict["coord_frame"]["voxel_unit"] = "nanometers"
# Setup the resource
resource = BossResourceBasic()
resource.from_dict(resource_dict)
dtype = resource.get_numpy_data_type()
# read all tiles from bucket into a slab
tile_bucket = TileBucket(proj_info.project_name)
data = []
num_z_slices = 0
for tile_key in tile_key_list:
try:
image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key)
except KeyError:
print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format(
tile_key))
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to missing tile in bucket")
return
image_bytes = BytesIO(image_data)
image_size = image_bytes.getbuffer().nbytes
# Get tiles size from metadata, need to shape black tile if actual tile is corrupt.
if 'x_size' in metadata:
tile_size_x = metadata['x_size']
else:
print('MetadataMissing: x_size not in tile metadata: using 1024.')
tile_size_x = 1024
if 'y_size' in metadata:
tile_size_y = metadata['y_size']
else:
print('MetadataMissing: y_size not in tile metadata: using 1024.')
tile_size_y = 1024
if image_size == 0:
print('TileError: Zero length tile, using black instead: {}'.format(tile_key))
error_msg = 'Zero length tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
else:
try:
# DP NOTE: Issues when specifying dtype in the asarray function with Pillow ver 8.3.1.
# Fixed by separating array instantiation and dtype assignment.
tile_img = np.asarray(Image.open(image_bytes))
tile_img = tile_img.astype(dtype)
except TypeError as te:
print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}'
.format(image_size, tile_key))
error_msg = 'Incomplete tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
except OSError as oe:
print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}'
.format(image_size, tile_key, oe))
error_msg = 'OSError: {}'.format(oe)
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
data.append(tile_img)
num_z_slices += 1
# Make 3D array of image data. It should be in XYZ at this point
chunk_data = np.array(data)
del data
tile_dims = chunk_data.shape
# Break into Cube instances
print("Tile Dims: {}".format(tile_dims))
print("Num Z Slices: {}".format(num_z_slices))
num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0]))
num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1]))
print("Num X Cuboids: {}".format(num_x_cuboids))
print("Num Y Cuboids: {}".format(num_y_cuboids))
chunk_key_parts = BossUtil.decode_chunk_key(chunk_key)
t_index = chunk_key_parts['t_index']
for x_idx in range(0, num_x_cuboids):
for y_idx in range(0, num_y_cuboids):
# TODO: check time series support
cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution])
cube.zeros()
# Compute Morton ID
# TODO: verify Morton indices correct!
print(chunk_key_parts)
morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids)
morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids)
print("Morton X: {}".format(morton_x_ind))
print("Morton Y: {}".format(morton_y_ind))
morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])])
# Insert sub-region from chunk_data into cuboid
x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0]
x_end = x_start + CUBOIDSIZE[proj_info.resolution][0]
x_end = min(x_end, tile_dims[2])
y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1]
y_end = y_start + CUBOIDSIZE[proj_info.resolution][1]
y_end = min(y_end, tile_dims[1])
z_end = CUBOIDSIZE[proj_info.resolution][2]
# TODO: get sub-array w/o making a copy.
print("Yrange: {}".format(y_end - y_start))
print("Xrange: {}".format(x_end - x_start))
print("X start: {}".format(x_start))
print("X stop: {}".format(x_end))
cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices,
y_start:y_end, x_start:x_end]
# Create object key
object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index)
print("Object Key: {}".format(object_key))
# Put object in S3
sp.objectio.put_objects([object_key], [cube.to_blosc()])
# Add object to index
sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"]))
# Update id indices if this is an annotation channel
# We no longer index during ingest.
#if resource.data['channel']['type'] == 'annotation':
# try:
# sp.objectio.update_id_indices(
# resource, proj_info.resolution, [object_key], [cube.data])
# except SpdbError as ex:
# sns_client = boto3.client('sns')
# topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"]
# msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format(
# ex.message,
# resource.data['collection']['name'],
# resource.data['experiment']['name'],
# resource.data['channel']['name'])
# sns_client.publish(
# TopicArn=topic_arn,
# Subject='Object services misuse',
# Message=msg)
lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME)
names = AWSNames.from_lambda(context.function_name)
delete_tiles_data = {
'tile_key_list': tile_key_list,
'region': SETTINGS.REGION_NAME,
'bucket': tile_bucket.bucket.name
}
# Delete tiles from tile bucket.
lambda_client.invoke(
FunctionName=names.delete_tile_objs.lambda_,
InvocationType='Event',
Payload=json.dumps(delete_tiles_data).encode()
)
delete_tile_entry_data = {
'tile_index': tile_index_db.table.name,
'region': SETTINGS.REGION_NAME,
'chunk_key': chunk_key,
'task_id': msg_data['ingest_job']
}
# Delete entry from tile index.
lambda_client.invoke(
FunctionName=names.delete_tile_index_entry.lambda_,
InvocationType='Event',
Payload=json.dumps(delete_tile_entry_data).encode()
)
if not sqs_triggered:
# Delete message since it was processed successfully
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
def test_project_name(self):
"""Project name should be collection name & experiment name."""
prj = BossIngestProj('col1', 'exp1', 'chanA', '0', '123')
expected = 'col1&exp1'
self.assertEqual(expected, prj.project_name)
Note that these tests require a local DynamoDB to run against.
"""
from __future__ import print_function
from __future__ import absolute_import
import sys
sys.path.append("..")
from ndingest.settings.settings import Settings
settings = Settings.load()
from ndingest.nddynamo.boss_tileindexdb import BossTileIndexDB, TILE_UPLOADED_MAP_KEY
from ndingest.ndingestproj.bossingestproj import BossIngestProj
job_id = "123"
nd_proj = BossIngestProj("testCol", "kasthuri11", "image", 0, job_id)
import json
import six
import unittest
import warnings
import time
import botocore
class Test_BossTileIndexDB(unittest.TestCase):
"""
Note that the chunk keys used, for testing, do not have real hash keys.
The rest of the chunk key is valid.
"""
def setUp(self):
# Suppress ResourceWarning messages about unclosed connections.
def setup_ingest(self, creator, config_data):
"""
Setup the ingest job. This is the primary method for the ingest manager.
It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest
Args:
creator: The validated user from the request to create the ingest jon
config_data : Config data to create the ingest job
Returns:
IngestJob : data model containing the ingest job
Raises:
BossError : For all exceptions that happen
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name,
self.experiment.name,
self.channel.name, self.resolution,
self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
# Call the step function to populate the queue.
self.job.step_function_arn = self.populate_upload_queue()
# Compute # of tiles in the job
x_extent = self.job.x_stop - self.job.x_start
y_extent = self.job.y_stop - self.job.y_start
z_extent = self.job.z_stop - self.job.z_start
t_extent = self.job.t_stop - self.job.t_start
num_tiles_in_x = math.ceil(x_extent / self.job.tile_size_x)
num_tiles_in_y = math.ceil(y_extent / self.job.tile_size_y)
num_tiles_in_z = math.ceil(z_extent / self.job.tile_size_z)
num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
self.job.save()
# tile_bucket = TileBucket(self.job.collection + '&' + self.job.experiment)
# self.create_ingest_credentials(upload_queue, tile_bucket)
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError(
"Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
def test_project_name(self):
"""Project name should be collection name & experiment name."""
prj = BossIngestProj("col1", "exp1", "chanA", "0", "123")
expected = "col1&exp1"
self.assertEqual(expected, prj.project_name)
def handler(event, context):
# Load settings
SETTINGS = BossSettings.load()
# Used as a guard against trying to delete the SQS message when lambda is
# triggered by SQS.
sqs_triggered = 'Records' in event and len(event['Records']) > 0
if sqs_triggered :
# Lambda invoked by an SQS trigger.
msg_data = json.loads(event['Records'][0]['body'])
# Load the project info from the chunk key you are processing
chunk_key = msg_data['chunk_key']
proj_info = BossIngestProj.fromSupercuboidKey(chunk_key)
proj_info.job_id = msg_data['ingest_job']
else:
# Standard async invoke of this lambda.
# Load the project info from the chunk key you are processing
proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"])
proj_info.job_id = event["ingest_job"]
# Get message from SQS ingest queue, try for ~2 seconds
rx_cnt = 0
msg_data = None
msg_id = None
msg_rx_handle = None
while rx_cnt < 6:
ingest_queue = IngestQueue(proj_info)
msg = [x for x in ingest_queue.receiveMessage()]
if msg:
msg = msg[0]
print("MESSAGE: {}".format(msg))
print(len(msg))
msg_id = msg[0]
msg_rx_handle = msg[1]
msg_data = json.loads(msg[2])
print("MESSAGE DATA: {}".format(msg_data))
break
else:
rx_cnt += 1
print("No message found. Try {} of 6".format(rx_cnt))
time.sleep(1)
if not msg_id:
# No tiles ready to ingest.
print("No ingest message available")
return
# Get the chunk key of the tiles to ingest.
chunk_key = msg_data['chunk_key']
tile_error_queue = TileErrorQueue(proj_info)
print("Ingesting Chunk {}".format(chunk_key))
tiles_in_chunk = int(chunk_key.split('&')[1])
# Setup SPDB instance
sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"],
msg_data['parameters']["STATEIO_CONFIG"],
msg_data['parameters']["OBJECTIO_CONFIG"])
# Get tile list from Tile Index Table
tile_index_db = BossTileIndexDB(proj_info.project_name)
# tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk.
tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"]))
if tile_index_result is None:
# If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards.
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to chunk key missing from tile index table")
return
# Sort the tile keys
print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"]))
tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()]
if len(tile_key_list) < tiles_in_chunk:
print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list)))
if not sqs_triggered:
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
return
tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1]))
tile_key_list = ["&".join(x) for x in tile_key_list]
print("Sorted Tile Keys: {}".format(tile_key_list))
# Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits
resource_dict = msg_data['parameters']['resource']
_, exp_name, ch_name = resource_dict["boss_key"].split("&")
resource_dict["channel"]["name"] = ch_name
resource_dict["channel"]["description"] = ""
resource_dict["channel"]["sources"] = []
resource_dict["channel"]["related"] = []
resource_dict["channel"]["default_time_sample"] = 0
resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED"
resource_dict["experiment"]["name"] = exp_name
resource_dict["experiment"]["description"] = ""
resource_dict["experiment"]["num_time_samples"] = 1
resource_dict["experiment"]["time_step"] = None
resource_dict["experiment"]["time_step_unit"] = None
resource_dict["coord_frame"]["name"] = "cf"
resource_dict["coord_frame"]["name"] = ""
resource_dict["coord_frame"]["x_start"] = 0
resource_dict["coord_frame"]["x_stop"] = 100000
resource_dict["coord_frame"]["y_start"] = 0
resource_dict["coord_frame"]["y_stop"] = 100000
resource_dict["coord_frame"]["z_start"] = 0
resource_dict["coord_frame"]["z_stop"] = 100000
resource_dict["coord_frame"]["voxel_unit"] = "nanometers"
# Setup the resource
resource = BossResourceBasic()
resource.from_dict(resource_dict)
dtype = resource.get_numpy_data_type()
# read all tiles from bucket into a slab
tile_bucket = TileBucket(proj_info.project_name)
data = []
num_z_slices = 0
for tile_key in tile_key_list:
try:
image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key)
except KeyError:
print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format(
tile_key))
if not sqs_triggered:
# Remove message so it's not redelivered.
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
print("Aborting due to missing tile in bucket")
return
image_bytes = BytesIO(image_data)
image_size = image_bytes.getbuffer().nbytes
# Get tiles size from metadata, need to shape black tile if actual tile is corrupt.
if 'x_size' in metadata:
tile_size_x = metadata['x_size']
else:
print('MetadataMissing: x_size not in tile metadata: using 1024.')
tile_size_x = 1024
if 'y_size' in metadata:
tile_size_y = metadata['y_size']
else:
print('MetadataMissing: y_size not in tile metadata: using 1024.')
tile_size_y = 1024
if image_size == 0:
print('TileError: Zero length tile, using black instead: {}'.format(tile_key))
error_msg = 'Zero length tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
else:
try:
tile_img = np.asarray(Image.open(image_bytes), dtype=dtype)
except TypeError as te:
print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}'
.format(image_size, tile_key))
error_msg = 'Incomplete tile'
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
except OSError as oe:
print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}'
.format(image_size, tile_key, oe))
error_msg = 'OSError: {}'.format(oe)
enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg)
tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype)
data.append(tile_img)
num_z_slices += 1
# Make 3D array of image data. It should be in XYZ at this point
chunk_data = np.array(data)
del data
tile_dims = chunk_data.shape
# Break into Cube instances
print("Tile Dims: {}".format(tile_dims))
print("Num Z Slices: {}".format(num_z_slices))
num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0]))
num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1]))
print("Num X Cuboids: {}".format(num_x_cuboids))
print("Num Y Cuboids: {}".format(num_y_cuboids))
chunk_key_parts = BossUtil.decode_chunk_key(chunk_key)
t_index = chunk_key_parts['t_index']
for x_idx in range(0, num_x_cuboids):
for y_idx in range(0, num_y_cuboids):
# TODO: check time series support
cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution])
cube.zeros()
# Compute Morton ID
# TODO: verify Morton indices correct!
print(chunk_key_parts)
morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids)
morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids)
print("Morton X: {}".format(morton_x_ind))
print("Morton Y: {}".format(morton_y_ind))
morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])])
# Insert sub-region from chunk_data into cuboid
x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0]
x_end = x_start + CUBOIDSIZE[proj_info.resolution][0]
x_end = min(x_end, tile_dims[2])
y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1]
y_end = y_start + CUBOIDSIZE[proj_info.resolution][1]
y_end = min(y_end, tile_dims[1])
z_end = CUBOIDSIZE[proj_info.resolution][2]
# TODO: get sub-array w/o making a copy.
print("Yrange: {}".format(y_end - y_start))
print("Xrange: {}".format(x_end - x_start))
print("X start: {}".format(x_start))
print("X stop: {}".format(x_end))
cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices,
y_start:y_end, x_start:x_end]
# Create object key
object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index)
print("Object Key: {}".format(object_key))
# Put object in S3
sp.objectio.put_objects([object_key], [cube.to_blosc()])
# Add object to index
sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"]))
# Update id indices if this is an annotation channel
# We no longer index during ingest.
#if resource.data['channel']['type'] == 'annotation':
# try:
# sp.objectio.update_id_indices(
# resource, proj_info.resolution, [object_key], [cube.data])
# except SpdbError as ex:
# sns_client = boto3.client('sns')
# topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"]
# msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format(
# ex.message,
# resource.data['collection']['name'],
# resource.data['experiment']['name'],
# resource.data['channel']['name'])
# sns_client.publish(
# TopicArn=topic_arn,
# Subject='Object services misuse',
# Message=msg)
lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME)
names = AWSNames.create_from_lambda_name(context.function_name)
delete_tiles_data = {
'tile_key_list': tile_key_list,
'region': SETTINGS.REGION_NAME,
'bucket': tile_bucket.bucket.name
}
# Delete tiles from tile bucket.
lambda_client.invoke(
FunctionName=names.delete_tile_objs_lambda,
InvocationType='Event',
Payload=json.dumps(delete_tiles_data).encode()
)
delete_tile_entry_data = {
'tile_index': tile_index_db.table.name,
'region': SETTINGS.REGION_NAME,
'chunk_key': chunk_key,
'task_id': msg_data['ingest_job']
}
# Delete entry from tile index.
lambda_client.invoke(
FunctionName=names.delete_tile_index_entry_lambda,
InvocationType='Event',
Payload=json.dumps(delete_tile_entry_data).encode()
)
if not sqs_triggered:
# Delete message since it was processed successfully
ingest_queue.deleteMessage(msg_id, msg_rx_handle)
