def test_not_creator_admin(self):
"""Method to test only creators or admins can interact with ingest jobs"""
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(201, response.status_code)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Log in as the admin and create a job
self.client.force_login(self.superuser)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(200, response.status_code)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
self.assertEqual(204, response.status_code)
def test_generate_upload_tasks(self):
""""""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data)
ingest_mgmr.generate_upload_tasks(ingest_job.id)
assert (ingest_job.collection == 'my_col_1')
assert (ingest_job.experiment == 'my_exp_1')
assert (ingest_job.channel == 'my_ch_1')
# Pull the messages off the queue
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = UploadQueue(nd_proj, endpoint_url=None)
tmp = queue.receiveMessage(number_of_messages=10)
# receive message from the queue
for message_id, receipt_handle, message_body in tmp:
assert(message_body['job_id'] == ingest_job.id)
# delete message from the queue
response = queue.deleteMessage(message_id, receipt_handle)
assert ('Successful' in response)
ingest_mgmr.remove_ingest_credentials(ingest_job.id)
except:
raise
finally:
ingest_mgmr.delete_upload_queue()
ingest_mgmr.delete_ingest_queue()
def test_setup_ingest(self):
"""Method to test the setup_ingest method"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.user.id,
self.example_config_data)
assert (ingest_job is not None)
# Check if the queue's exist
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job.collection, ingest_job.experiment,
ingest_job.channel, ingest_job.resolution,
ingest_job.id)
ingest_mgmr.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
ingest_mgmr.remove_ingest_credentials(ingest_job.id)
except:
raise
finally:
ingest_mgmr.delete_upload_queue()
ingest_mgmr.delete_ingest_queue()
def test_not_creator_admin(self):
"""Method to test only creators or admins can interact with ingest jobs"""
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
assert (response.status_code == 201)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
# Log in as the admin and create a job
self.client.force_login(self.superuser)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
assert (response.status_code == 200)
assert (response.json()['ingest_job']['id'] == ingest_job['id'])
assert("credentials" in response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
assert (response.status_code == 204)
def test_post_new_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
assert (response.status_code == 201)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'],
ingest_job['experiment'],
ingest_job['channel_layer'], 0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
assert (response.status_code == 204)
def setup_ingest(self, creator, config_data):
"""
Setup the ingest job. This is the primary method for the ingest manager.
It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest
Args:
creator: The validated user from the request to create the ingest jon
config_data : Config data to create the ingest job
Returns:
IngestJob : data model containing the ingest job
Raises:
BossError : For all exceptions that happen
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name,
self.experiment.name,
self.channel.name, self.resolution,
self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
self.generate_upload_tasks()
tile_bucket = TileBucket(self.job.collection + '&' +
self.job.experiment)
self.create_ingest_credentials(upload_queue, tile_bucket)
# TODO create channel if needed
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError(
"Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
def setup_ingest(self, creator, config_data):
"""
Args:
Returns:
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name,
self.experiment.name,
self.channel_layer.name,
self.resolution, self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
self.generate_upload_tasks()
tile_bucket = TileBucket(self.job.collection + '&' +
self.job.experiment)
self.create_ingest_credentials(upload_queue, tile_bucket)
# Update status
self.job.status = 1
self.job.save()
# TODO create channel if needed
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError(
"Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
def test_complete_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(response.status_code, 201)
# Check if the queues exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'],
ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
# Can't complete until it is done
self.assertEqual(400, response.status_code)
# Wait for job to complete
print('trying to join job')
for cnt in range(0, 30):
# Try joining to kick the status
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
self.client.get(url)
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
if response.json()["status"] == IngestJob.UPLOADING:
break
time.sleep(10)
print('completing')
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
self.assertEqual(204, response.status_code)
def test_complete_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(response.status_code, 201)
# Check if the queues exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
# Can't complete until it is done
self.assertEqual(400, response.status_code)
# Wait for job to complete
print('trying to join job')
for cnt in range(0, 30):
# Try joining to kick the status
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
self.client.get(url)
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
if response.json()["status"] == IngestJob.UPLOADING:
break
time.sleep(10)
print('completing')
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
self.assertEqual(204, response.status_code)
def get_ingest_job_tile_error_queue(self, ingest_job):
"""
Return the tile index queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
ndingest.TileIndexQueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = TileErrorQueue(self.nd_proj, endpoint_url=None)
return queue
def get_ingest_job_ingest_queue(self, ingest_job):
"""
Return the ingest queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
Ndingest.ingestqueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = IngestQueue(self.nd_proj, endpoint_url=None)
return queue
def get_ingest_job_tile_error_queue(self, ingest_job):
"""
Return the tile index queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
ndingest.TileIndexQueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = TileErrorQueue(self.nd_proj, endpoint_url=None)
return queue
def get_ingest_job_upload_queue(self, ingest_job):
"""
Return the upload queue for an ingest job
Args:
ingest_job: Ingest job model
Returns:
Ndingest.uploadqueue
"""
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = UploadQueue(self.nd_proj, endpoint_url=None)
return queue
def cleanup_ingest_job(self, ingest_job, job_status):
"""
Delete or complete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
in the tile bucket for this job id. It does not delete the ingest job datamodel but changes its state.
Args:
ingest_job: Ingest job to cleanup
job_status(int): Status to update to
Returns:
(int): ingest job id for the job that was successfully deleted
Raises:
BossError : If the the job id is not valid or any exception happens in deletion process
"""
try:
# cleanup ingest job
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection,
ingest_job.experiment,
ingest_job.channel,
ingest_job.resolution, ingest_job.id)
# delete the ingest and upload_queue
self.delete_upload_queue()
if ingest_job.ingest_type != IngestJob.VOLUMETRIC_INGEST:
self.delete_ingest_queue()
# delete any pending entries in the tile index database and tile bucket
# Commented out due to removal of tile index's GSI.
# self.delete_tiles(ingest_job)
ingest_job.status = job_status
ingest_job.ingest_queue = None
ingest_job.upload_queue = None
ingest_job.end_date = timezone.now()
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job.id)
except Exception as e:
raise BossError("Unable to cleanup the upload queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(ingest_job.id),
ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job.id
def delete_ingest_job(self, ingest_job_id):
"""
Delete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
in the tile bucket for this job id. It does not delete the ingest job datamodel but marks it as deleted.
Args:
ingest_job_id: Ingest job id to delete
Returns:
Int : ingest job id for the job that was successfully deleted
Raises:
BossError : If the the job id is not valid or any exception happens in deletion process
"""
try:
# delete ingest job
ingest_job = IngestJob.objects.get(id=ingest_job_id)
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection,
ingest_job.experiment,
ingest_job.channel,
ingest_job.resolution, ingest_job.id)
# delete the ingest and upload_queue
self.delete_upload_queue()
self.delete_ingest_queue()
# delete any pending entries in the tile index database and tile bucket
self.delete_tiles(ingest_job)
ingest_job.status = 3
ingest_job.ingest_queue = None
ingest_job.upload_queue = None
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job_id)
except Exception as e:
raise BossError("Unable to delete the upload queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(ingest_job_id),
ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job_id
def cleanup_ingest_job(self, ingest_job, job_status):
"""
Delete or complete an ingest job with a specific id. Note this deletes the queues, credentials and all the remaining tiles
in the tile bucket for this job id. It does not delete the ingest job datamodel but changes its state.
Args:
ingest_job: Ingest job to cleanup
job_status(int): Status to update to
Returns:
(int): ingest job id for the job that was successfully deleted
Raises:
BossError : If the the job id is not valid or any exception happens in deletion process
"""
try:
# cleanup ingest job
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
# delete the queues
self.delete_upload_queue()
if ingest_job.ingest_type != IngestJob.VOLUMETRIC_INGEST:
self.delete_ingest_queue()
self.delete_tile_index_queue()
self.delete_tile_error_queue()
# delete any pending entries in the tile index database and tile bucket
# Commented out due to removal of tile index's GSI.
# self.delete_tiles(ingest_job)
ingest_job.status = job_status
ingest_job.ingest_queue = None
ingest_job.upload_queue = None
ingest_job.end_date = timezone.now()
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job.id)
except Exception as e:
raise BossError("Unable to cleanup the upload queue.{}".format(e), ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError("Ingest job with id {} does not exist".format(ingest_job.id), ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job.id
def test_post_new_volumetric_ingest_job(self):
""" Test view to create a new volumetric_ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict_volumetric(
)
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(201, response.status_code)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'],
ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
# There shouldn't be an ingest queue for a volumetric ingest
with self.assertRaises(ClientError):
IngestQueue(nd_proj, endpoint_url=None)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
self.assertEqual(204, response.status_code)
# Verify Queues are removed
with self.assertRaises(ClientError):
UploadQueue(nd_proj, endpoint_url=None)
# Verify the job is deleted
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
def test_post_new_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
self.assertEqual(201, response.status_code)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(upload_queue)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
self.assertIsNotNone(ingest_queue)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.json()['ingest_job']['id'], ingest_job['id'])
self.assertIn("credentials", response.json())
# # Delete the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.delete(url)
self.assertEqual(204, response.status_code)
# Verify Queues are removed
with self.assertRaises(ClientError):
UploadQueue(nd_proj, endpoint_url=None)
with self.assertRaises(ClientError):
IngestQueue(nd_proj, endpoint_url=None)
# Verify the job is deleted
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.status_code, 404)
def delete_ingest_job(self, ingest_job_id):
"""
Args:
ingest_job_id:
Returns:
"""
try:
# delete ingest job
ingest_job = IngestJob.objects.get(id=ingest_job_id)
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(ingest_job.collection,
ingest_job.experiment,
ingest_job.channel_layer,
ingest_job.resolution, ingest_job.id)
# delete the ingest and upload_queue
self.delete_upload_queue()
self.delete_ingest_queue()
# delete any pending entries in the tile index database and tile bucket
self.delete_tiles(ingest_job)
ingest_job.status = 3
ingest_job.save()
# Remove ingest credentials for a job
self.remove_ingest_credentials(ingest_job_id)
except Exception as e:
raise BossError("Unable to delete the upload queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(ingest_job_id),
ErrorCodes.OBJECT_NOT_FOUND)
return ingest_job_id
def __init__(self, status, db_host, job):
"""
Args:
status (str): Mark ingest job as 'deleted' or 'complete'.
db_host (str): Host of MySQL database.
job (dict):
collection (int): Collection id.
experiment (int): Experiment id.
channel (int): Channel id.
task_id (int): The ingest job's id.
resolution (int): Resolution of chunk.
ingest_type (int): Tile (0) or volumetric ingest (1).
"""
self._status_map = {
COMPLETE_STATUS: COMPLETE_DB,
DELETED_STATUS: DELETED_DB
}
self.db_host = db_host
self.job = job
# Validate job parameter.
for field in IngestCleaner.JOB_FIELDS:
if field not in job:
raise KeyError('Job must have {}'.format(field))
# Validate status parameter.
self.status = status
if status.lower() not in IngestCleaner.STATUS_VALUES:
raise BadStatusError('{} is not a valid status.'.format(status))
proj_class = BossIngestProj.load()
# coll/exp/chan are specified as strings, but not necessarily to give
# actual names when using ndingest's delete functionality.
self.nd_proj = proj_class(job['collection'], job['experiment'],
job['channel'], job['resolution'],
job['task_id'])
def setup_ingest(self, creator, config_data):
"""
Setup the ingest job. This is the primary method for the ingest manager.
It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest
Args:
creator: The validated user from the request to create the ingest jon
config_data : Config data to create the ingest job
Returns:
IngestJob : data model containing the ingest job
Raises:
BossError : For all exceptions that happen
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name, self.experiment.name, self.channel.name,
self.resolution, self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
if self.job.ingest_type == IngestJob.TILE_INGEST:
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
tile_index_queue = self.create_tile_index_queue()
self.add_trigger_tile_uploaded_lambda_from_queue(tile_index_queue.arn)
self.create_tile_error_queue()
elif self.job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# Will the management console be ok with ingest_queue being null?
pass
# Call the step function to populate the queue.
self.job.step_function_arn = self.populate_upload_queue(self.job)
# Compute # of tiles or chunks in the job
x_extent = self.job.x_stop - self.job.x_start
y_extent = self.job.y_stop - self.job.y_start
z_extent = self.job.z_stop - self.job.z_start
t_extent = self.job.t_stop - self.job.t_start
num_tiles_in_x = math.ceil(x_extent/self.job.tile_size_x)
num_tiles_in_y = math.ceil(y_extent/self.job.tile_size_y)
num_tiles_in_z = math.ceil(z_extent/self.job.tile_size_z)
num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
self.job.save()
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError("Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job
def test_complete_ingest_job(self):
""" Test view to create a new ingest job """
config_data = self.setup_helper.get_ingest_config_data_dict()
config_data = json.loads(json.dumps(config_data))
# # Post the data
url = '/' + version + '/ingest/'
response = self.client.post(url, data=config_data, format='json')
assert (response.status_code == 201)
# Check if the queue's exist
ingest_job = response.json()
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job['collection'], ingest_job['experiment'], ingest_job['channel'],
0, ingest_job['id'])
self.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
# Test joining the job
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
response = self.client.get(url)
assert(response.json()['ingest_job']['id'] == ingest_job['id'])
assert("credentials" in response.json())
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
# Can't complete until it is done
assert(response.status_code == 400)
# Wait for job to complete
for cnt in range(0, 30):
# Try joining to kick the status
url = '/' + version + '/ingest/{}/'.format(ingest_job['id'])
self.client.get(url)
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
if response.json()["status"] == 1:
break
time.sleep(10)
# Complete the job
url = '/' + version + '/ingest/{}/complete'.format(ingest_job['id'])
response = self.client.post(url)
assert(response.status_code == 204)
# Verify Queues are removed
with self.assertRaises(ClientError):
UploadQueue(nd_proj, endpoint_url=None)
with self.assertRaises(ClientError):
IngestQueue(nd_proj, endpoint_url=None)
# Verify status has changed
url = '/' + version + '/ingest/{}/status'.format(ingest_job['id'])
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()["status"], 2)
def setup_ingest(self, creator, config_data):
"""
Setup the ingest job. This is the primary method for the ingest manager.
It creates the ingest job and queues required for the ingest. It also uploads the messages for the ingest
Args:
creator: The validated user from the request to create the ingest jon
config_data : Config data to create the ingest job
Returns:
IngestJob : data model containing the ingest job
Raises:
BossError : For all exceptions that happen
"""
# Validate config data and schema
self.owner = creator
try:
valid_schema = self.validate_config_file(config_data)
valid_prop = self.validate_properties()
if valid_schema is True and valid_prop is True:
# create the django model for the job
self.job = self.create_ingest_job()
# create the additional resources needed for the ingest
# initialize the ndingest project for use with the library
proj_class = BossIngestProj.load()
self.nd_proj = proj_class(self.collection.name,
self.experiment.name,
self.channel.name, self.resolution,
self.job.id)
# Create the upload queue
upload_queue = self.create_upload_queue()
self.job.upload_queue = upload_queue.url
# Create the ingest queue
ingest_queue = self.create_ingest_queue()
self.job.ingest_queue = ingest_queue.url
# Call the step function to populate the queue.
self.job.step_function_arn = self.populate_upload_queue()
# Compute # of tiles in the job
x_extent = self.job.x_stop - self.job.x_start
y_extent = self.job.y_stop - self.job.y_start
z_extent = self.job.z_stop - self.job.z_start
t_extent = self.job.t_stop - self.job.t_start
num_tiles_in_x = math.ceil(x_extent / self.job.tile_size_x)
num_tiles_in_y = math.ceil(y_extent / self.job.tile_size_y)
num_tiles_in_z = math.ceil(z_extent / self.job.tile_size_z)
num_tiles_in_t = math.ceil(t_extent / self.job.tile_size_t)
self.job.tile_count = num_tiles_in_x * num_tiles_in_y * num_tiles_in_z * num_tiles_in_t
self.job.save()
# tile_bucket = TileBucket(self.job.collection + '&' + self.job.experiment)
# self.create_ingest_credentials(upload_queue, tile_bucket)
except BossError as err:
raise BossError(err.message, err.error_code)
except Exception as e:
raise BossError(
"Unable to create the upload and ingest queue.{}".format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
return self.job