def test_generate_upload_tasks(self): """""" try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data) ingest_mgmr.generate_upload_tasks(ingest_job.id) assert (ingest_job.collection == 'my_col_1') assert (ingest_job.experiment == 'my_exp_1') assert (ingest_job.channel == 'my_ch_1') # Pull the messages off the queue proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel, ingest_job.resolution, ingest_job.id) queue = UploadQueue(nd_proj, endpoint_url=None) tmp = queue.receiveMessage(number_of_messages=10) # receive message from the queue for message_id, receipt_handle, message_body in tmp: assert(message_body['job_id'] == ingest_job.id) # delete message from the queue response = queue.deleteMessage(message_id, receipt_handle) assert ('Successful' in response) ingest_mgmr.remove_ingest_credentials(ingest_job.id) except: raise finally: ingest_mgmr.delete_upload_queue() ingest_mgmr.delete_ingest_queue()
def test_setup_ingest(self): """Method to test the setup_ingest method""" try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.user.id, self.example_config_data) assert (ingest_job is not None) # Check if the queue's exist proj_class = BossIngestProj.load() nd_proj = proj_class(ingest_job.collection, ingest_job.experiment, ingest_job.channel, ingest_job.resolution, ingest_job.id) ingest_mgmr.nd_proj = nd_proj upload_queue = UploadQueue(nd_proj, endpoint_url=None) assert (upload_queue is not None) ingest_queue = IngestQueue(nd_proj, endpoint_url=None) assert (ingest_queue is not None) ingest_mgmr.remove_ingest_credentials(ingest_job.id) except: raise finally: ingest_mgmr.delete_upload_queue() ingest_mgmr.delete_ingest_queue()
def post(self, request): """ Post a new config job and create a new ingest job Args: request: Django Rest framework Request object ingest_config_data: COnfiguration data for the ingest job Returns: """ ingest_config_data = request.data try: self.track_usage_data(ingest_config_data, request) ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.request.user.id, ingest_config_data) serializer = IngestJobListSerializer(ingest_job) return Response(serializer.data, status=status.HTTP_201_CREATED) except BossError as err: return err.to_http()
def post(self, request): """ Post a new config job and create a new ingest job Args: ingest_config_data: Returns: """ ingest_config_data = request.data try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.request.user.id, ingest_config_data) serializer = IngestJobListSerializer(ingest_job) return Response(serializer.data, status=status.HTTP_201_CREATED) except BossError as err: return err.to_http()
def post(self, request): """ Post a new config job and create a new ingest job Args: request: Django Rest framework Request object ingest_config_data: COnfiguration data for the ingest job Returns: """ ingest_config_data = request.data # Add metrics to CloudWatch extent = ingest_config_data['ingest_job']['extent'] tile_size = ingest_config_data['ingest_job']['tile_size'] database = ingest_config_data['database'] # Check that only permitted users are creating extra large ingests try: group = Group.objects.get(name=INGEST_GRP) in_large_ingest_group = group.user_set.filter( id=request.user.id).exists() except Group.DoesNotExist: # Just in case the group has not been created yet in_large_ingest_group = False if (not in_large_ingest_group) and \ ((extent['x'][1] - extent['x'][0]) * \ (extent['y'][1] - extent['y'][0]) * \ (extent['z'][1] - extent['z'][0]) * \ (extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE): return BossHTTPError( "Large ingests require special permission to create. Contact system administrator.", ErrorCodes.INVALID_STATE) # Calculate the cost of the ingest cost = (((extent['x'][1] - extent['x'][0]) / tile_size['x']) * ((extent['y'][1] - extent['y'][0]) / tile_size['y']) * ((extent['z'][1] - extent['z'][0]) / tile_size['z']) * ((extent['t'][1] - extent['t'][0]) / tile_size['t']) * 1.0625 # 1 lambda per tile + 1 lambda per 16 tiles (per cube) * 1 # the cost per lambda ) # Calculating the cost of the lambda invocations boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(database['collection'], database['experiment'], database['channel']) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') client.put_metric_data(Namespace="BOSS/Ingest", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'ComputeCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Count' }]) try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.setup_ingest(self.request.user.id, ingest_config_data) serializer = IngestJobListSerializer(ingest_job) return Response(serializer.data, status=status.HTTP_201_CREATED) except BossError as err: return err.to_http()