def test_generate_upload_tasks(self):
""""""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data)
ingest_mgmr.generate_upload_tasks(ingest_job.id)
assert (ingest_job.collection == 'my_col_1')
assert (ingest_job.experiment == 'my_exp_1')
assert (ingest_job.channel == 'my_ch_1')
# Pull the messages off the queue
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel,
ingest_job.resolution, ingest_job.id)
queue = UploadQueue(nd_proj, endpoint_url=None)
tmp = queue.receiveMessage(number_of_messages=10)
# receive message from the queue
for message_id, receipt_handle, message_body in tmp:
assert(message_body['job_id'] == ingest_job.id)
# delete message from the queue
response = queue.deleteMessage(message_id, receipt_handle)
assert ('Successful' in response)
ingest_mgmr.remove_ingest_credentials(ingest_job.id)
except:
raise
finally:
ingest_mgmr.delete_upload_queue()
ingest_mgmr.delete_ingest_queue()
def test_setup_ingest(self):
"""Method to test the setup_ingest method"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.user.id,
self.example_config_data)
assert (ingest_job is not None)
# Check if the queue's exist
proj_class = BossIngestProj.load()
nd_proj = proj_class(ingest_job.collection, ingest_job.experiment,
ingest_job.channel, ingest_job.resolution,
ingest_job.id)
ingest_mgmr.nd_proj = nd_proj
upload_queue = UploadQueue(nd_proj, endpoint_url=None)
assert (upload_queue is not None)
ingest_queue = IngestQueue(nd_proj, endpoint_url=None)
assert (ingest_queue is not None)
ingest_mgmr.remove_ingest_credentials(ingest_job.id)
except:
raise
finally:
ingest_mgmr.delete_upload_queue()
ingest_mgmr.delete_ingest_queue()
def post(self, request):
"""
Post a new config job and create a new ingest job
Args:
request: Django Rest framework Request object
ingest_config_data: COnfiguration data for the ingest job
Returns:
"""
ingest_config_data = request.data
try:
self.track_usage_data(ingest_config_data, request)
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.request.user.id,
ingest_config_data)
serializer = IngestJobListSerializer(ingest_job)
return Response(serializer.data, status=status.HTTP_201_CREATED)
except BossError as err:
return err.to_http()
def post(self, request):
"""
Post a new config job and create a new ingest job
Args:
ingest_config_data:
Returns:
"""
ingest_config_data = request.data
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.request.user.id,
ingest_config_data)
serializer = IngestJobListSerializer(ingest_job)
return Response(serializer.data, status=status.HTTP_201_CREATED)
except BossError as err:
return err.to_http()
def post(self, request):
"""
Post a new config job and create a new ingest job
Args:
request: Django Rest framework Request object
ingest_config_data: COnfiguration data for the ingest job
Returns:
"""
ingest_config_data = request.data
# Add metrics to CloudWatch
extent = ingest_config_data['ingest_job']['extent']
tile_size = ingest_config_data['ingest_job']['tile_size']
database = ingest_config_data['database']
# Check that only permitted users are creating extra large ingests
try:
group = Group.objects.get(name=INGEST_GRP)
in_large_ingest_group = group.user_set.filter(
id=request.user.id).exists()
except Group.DoesNotExist:
# Just in case the group has not been created yet
in_large_ingest_group = False
if (not in_large_ingest_group) and \
((extent['x'][1] - extent['x'][0]) * \
(extent['y'][1] - extent['y'][0]) * \
(extent['z'][1] - extent['z'][0]) * \
(extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE):
return BossHTTPError(
"Large ingests require special permission to create. Contact system administrator.",
ErrorCodes.INVALID_STATE)
# Calculate the cost of the ingest
cost = (((extent['x'][1] - extent['x'][0]) / tile_size['x']) *
((extent['y'][1] - extent['y'][0]) / tile_size['y']) *
((extent['z'][1] - extent['z'][0]) / tile_size['z']) *
((extent['t'][1] - extent['t'][0]) / tile_size['t']) *
1.0625 # 1 lambda per tile + 1 lambda per 16 tiles (per cube)
* 1 # the cost per lambda
) # Calculating the cost of the lambda invocations
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username
},
{
'Name':
'Resource',
'Value':
'{}/{}/{}'.format(database['collection'],
database['experiment'], database['channel'])
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
client.put_metric_data(Namespace="BOSS/Ingest",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'ComputeCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Count'
}])
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.setup_ingest(self.request.user.id,
ingest_config_data)
serializer = IngestJobListSerializer(ingest_job)
return Response(serializer.data, status=status.HTTP_201_CREATED)
except BossError as err:
return err.to_http()