def delete(self, request, ingest_job_id): """ Args: request: ingest_job_id: Returns: """ try: ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check permissions if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can cancel an ingest job", ErrorCodes.INGEST_NOT_CREATOR) ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.DELETED) blog = bossLogger() blog.info("Deleted Ingest Job {}".format(ingest_job_id)) return Response(status=status.HTTP_204_NO_CONTENT) except BossError as err: return err.to_http()
def __init__(self, message, code=ErrorCodes.KEYCLOAK_EXCEPTION): """ Custom HTTP error class for converting a KeyCloakError exception into a JsonResponse Args: message (str): Error message to send back to user Note: If called in an exception handler, expects the exception to be a KeyCloakError """ ex = sys.exc_info()[1] self.status_code = ex.status if ex else RESP_CODES[code] data = { 'status': self.status_code, 'code': ErrorCodes.KEYCLOAK_EXCEPTION, 'message': message } if ex: if isinstance(ex.data, str): val = json.loads(ex.data) else: val = ex.data data.update(val) msg = "BossKeycloakError" for k in data: msg += " - {}: {}".format(k.capitalize(), data[k]) log = bossLogger() log.info(msg) super(BossKeycloakError, self).__init__(data)
def get_session(self): """ Method to get a boto3.session.Session object. If session credentials have expired a new session is created. If no sessions are available (if a previous consumer of a session didn't return it for some reason, e.g. an exception occurred), a new session is created :return: boto3.session.Session """ temp_session = None while not temp_session: try: temp_session = self.__sessions.get(block=False) except queue.Empty: # No session was available so generate one blog = bossLogger() blog.info( "AWSManager - No session was available while trying to execute get_session. Dynamically creating a new session." ) self.__create_session() return temp_session
def lambda_connect_sqs(self, queue, lambda_name, num_msgs=1): """ Adds an SQS event trigger to the given lambda. Args: queue (SQS.Queue): SQS queue that will be the trigger source. lambda_name (str): Lambda function name. num_msgs (optional[int]): Number of messages to send to the lambda. Defaults to 1, max 10. Raises: (ValueError): if num_msgs is greater than the SQS max batch size. """ if num_msgs < 1 or num_msgs > MAX_SQS_BATCH_SIZE: raise ValueError('lambda_connect_sqs(): Bad num_msgs: {}'.format(num_msgs)) queue_arn = queue.attributes['QueueArn'] timeout = self.get_ingest_lambda_timeout(lambda_name) # AWS recommends that an SQS queue used as a lambda event source should # have a visibility timeout that's 6 times the lambda's timeout. queue.set_attributes(Attributes={'VisibilityTimeout': str(timeout * 6)}) client = boto3.client('lambda', region_name=bossutils.aws.get_region()) try: client.create_event_source_mapping( EventSourceArn=queue_arn, FunctionName=lambda_name, BatchSize=num_msgs) except client.exceptions.ResourceConflictException: log = bossLogger() log.warning(f'ResourceConflictException caught trying to connect {queue_arn} to {lambda_name}. This should be harmless because this happens when the queue has already been connected.')
def wrapper(self, *args, **kwargs): try: return function(self, *args, **kwargs) except hvac.exceptions.Forbidden as e: blog = bossLogger() blog.info(str(e)) msg = "Your token had expired. Dynamically creating a new one." blog.info(msg) Vault.login(self) return function(self, *args, **kwargs)
def __create_session(self): """ Method to create a new boto3.session.Session object and add it to the pool :return: None """ temp_session = get_session() blog = bossLogger() blog.info( "AWSManager - Created new boto3 session and added to the pool") self.__sessions.put(temp_session)
def get_ingest_lambda_timeout(self, name): """ Get the current timeout of the tile ingest lambda. Args: name (str): Name of lambda. Returns: (int): Number of seconds of the timeout. """ client = boto3.client('lambda', region_name=bossutils.aws.get_region()) try: resp = client.get_function(FunctionName=name) return resp['Configuration']['Timeout'] except Exception as ex: log = bossLogger() log.error(f"Couldn't get lambda: {name} data from AWS: {ex}") raise
def __init__(self, message, code): """ Custom HTTP error class :param status: HTTP Status code :type status: int :param code: An optional, arbitrary, and unique code to identify where the error was generated :type code: int :param message: Message to provide feedback to the user :return: """ # Set status code self.status_code = RESP_CODES[code] # Log blog = bossLogger() blog.info( "BossHTTPError - Status: {0} - Code: {1} - Message: {2}".format( self.status_code, code, message)) # Return data = {'status': self.status_code, 'code': code, 'message': message} super(BossHTTPError, self).__init__(data)
def __init__(self, pid_file_name, pid_dir="/var/run"): super().__init__(pid_file_name, pid_dir) self.log = logger.bossLogger() self.config = configuration.BossConfig() # kvio settings kvio_config = { "cache_host": self.config['aws']['cache'], "cache_db": self.config['aws']['cache-db'], "read_timeout": 86400 } # state settings state_config = { "cache_state_host": self.config['aws']['cache-state'], "cache_state_db": self.config['aws']['cache-state-db'] } # object store settings object_store_config = { "s3_flush_queue": self.config["aws"]["s3-flush-queue"], "cuboid_bucket": self.config['aws']['cuboid_bucket'], "page_in_lambda_function": self.config['lambda']['page_in_function'], "page_out_lambda_function": self.config['lambda']['flush_function'], "s3_index_table": self.config['aws']['s3-index-table'], "id_index_table": self.config['aws']['id-index-table'], "id_count_table": self.config['aws']['id-count-table'] } config_data = { "kv_config": kvio_config, "state_config": state_config, "object_store_config": object_store_config } self.lambda_data = {"config": config_data, "lambda-name": "s3_flush"} self.sqs_watcher = SqsWatcher(self.lambda_data)
def remove_sqs_event_source_from_lambda(self, queue_arn, lambda_name): """ Removes an SQS event triggger from the given lambda. Args: queue_arn (str): Arn of SQS queue that will be the trigger source. lambda_name (str): Lambda function name. """ log = bossLogger() client = boto3.client('lambda', region_name=bossutils.aws.get_region()) try: resp = client.list_event_source_mappings( EventSourceArn=queue_arn, FunctionName=lambda_name) except Exception as ex: log.error(f"Couldn't list event source mappings for {lambda_name}: {ex}") return for evt in resp['EventSourceMappings']: try: client.delete_event_source_mapping(UUID=evt['UUID']) except client.exceptions.ResourceNotFoundException: pass except Exception as ex: log.error(f"Couldn't remove event source mapping {queue_arn} from {lambda_name}: {ex}")
def __init__(self, lambda_data): self.lambda_data = lambda_data self.log = logger.bossLogger() self.old_message_num = 0 self.message_num = 0
def post(self, request, ingest_job_id): """ Signal an ingest job is complete and should be cleaned up by POSTing to this view Args: request: Django Rest framework Request object ingest_job_id: Ingest job id Returns: """ try: blog = bossLogger() ingest_mgmr = IngestManager() ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) if ingest_job.status == IngestJob.PREPARING: # If status is Preparing. Deny return BossHTTPError( "You cannot complete a job that is still preparing. You must cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.UPLOADING: try: data = ingest_mgmr.try_enter_wait_on_queue_state( ingest_job) return Response(data=data, status=status.HTTP_202_ACCEPTED) except BossError as be: if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG): # If there are messages in the tile error queue, this # will have to be handled manually. Non-empty ingest # or tile index queues should resolve on their own. return Response(data={ 'wait_secs': WAIT_FOR_QUEUES_SECS, 'info': 'Internal queues not empty yet' }, status=status.HTTP_400_BAD_REQUEST) raise elif ingest_job.status == IngestJob.WAIT_ON_QUEUES: pass # Continue below. elif ingest_job.status == IngestJob.COMPLETE: # If status is already Complete, just return another 204 return Response(data={'job_status': ingest_job.status}, status=status.HTTP_204_NO_CONTENT) elif ingest_job.status == IngestJob.DELETED: # Job had already been cancelled return BossHTTPError("Ingest job has already been cancelled.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.FAILED: # Job had failed return BossHTTPError( "Ingest job has failed during creation. You must Cancel instead.", ErrorCodes.BAD_REQUEST) elif ingest_job.status == IngestJob.COMPLETING: return Response(data={'job_status': ingest_job.status}, status=status.HTTP_202_ACCEPTED) # Check if user is the ingest job creator or the sys admin if not self.is_user_or_admin(request, ingest_job): return BossHTTPError( "Only the creator or admin can complete an ingest job", ErrorCodes.INGEST_NOT_CREATOR) # Try to start completing. try: data = ingest_mgmr.try_start_completing(ingest_job) if data['job_status'] == IngestJob.WAIT_ON_QUEUES: # Refuse complete requests until wait period expires. return Response(data=data, status=status.HTTP_400_BAD_REQUEST) except BossError as be: if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG): return Response(data={ 'wait_secs': WAIT_FOR_QUEUES_SECS, 'info': 'Internal queues not empty yet' }, status=status.HTTP_400_BAD_REQUEST) raise blog.info("Completion process started for ingest Job {}".format( ingest_job_id)) return Response(data=data, status=status.HTTP_202_ACCEPTED) # TODO SH This is a quick fix to make sure the ingest-client does not run close option. # the clean up code commented out below, because it is not working correctly. # return Response(status=status.HTTP_204_NO_CONTENT) # if ingest_job.ingest_type == IngestJob.TILE_INGEST: # # Check if any messages remain in the ingest queue # ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job) # num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages']) # # # Kick off extra lambdas just in case # if num_messages_in_queue: # blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue)) # ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue) # # # Give lambda a few seconds to fire things off # time.sleep(30) # # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST: # ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # # ToDo: call cleanup method for volumetric ingests. Don't want # # to cleanup until after testing with real data. # #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE) # # blog.info("Complete successful") # return Response(status=status.HTTP_204_NO_CONTENT) except BossError as err: return err.to_http() except Exception as err: blog.error('Caught general exception: {}'.format(err)) return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def track_usage_data(self, ingest_config_data, request): """ Set up usage tracking of this ingest. Args: ingest_config_data (dict): Ingest job config. Raises: (BossError): if user doesn't have permission for a large ingest. """ # need to get bytes per pixel to caculate ingest in total bytes try: ingest_job = ingest_config_data['ingest_job'] theCollection = Collection.objects.get( name=ingest_config_data['database']['collection']) theExperiment = Experiment.objects.get( name=ingest_config_data['database']['experiment']) theChannel = Channel.objects.get( name=ingest_config_data['database']['channel']) bytesPerPixel = int(theChannel.datatype.replace("uint", "")) / 8 # Add metrics to CloudWatch extent = ingest_job['extent'] database = ingest_config_data['database'] # Check that only permitted users are creating extra large ingests try: group = Group.objects.get(name=INGEST_GRP) in_large_ingest_group = group.user_set.filter( id=request.user.id).exists() except Group.DoesNotExist: # Just in case the group has not been created yet in_large_ingest_group = False if (not in_large_ingest_group) and \ ((extent['x'][1] - extent['x'][0]) * \ (extent['y'][1] - extent['y'][0]) * \ (extent['z'][1] - extent['z'][0]) * \ (extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE): raise BossError( "Large ingests require special permission to create. Contact system administrator.", ErrorCodes.INVALID_STATE) # Calculate the cost of the ingest in pixels costInPixels = ((extent['x'][1] - extent['x'][0]) * (extent['y'][1] - extent['y'][0]) * (extent['z'][1] - extent['z'][0]) * (extent['t'][1] - extent['t'][0])) cost = costInPixels * bytesPerPixel BossThrottle().check('ingest', ThrottleMetric.METRIC_TYPE_INGRESS, request.user, cost, ThrottleMetric.METRIC_UNITS_BYTES) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username or "public" }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(database['collection'], database['experiment'], database['channel']) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') try: client.put_metric_data(Namespace="BOSS/Ingest", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'IngressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) except Exception as e: log = bossLogger() log.exception('Error during put_metric_data: {}'.format(e)) log.exception('Allowing bossDB to continue after logging') except BossError as err: return err.to_http() except Exception as err: return BossError("{}".format(err), ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def try_start_completing(self, ingest_job): """ Tries to start completion process. It is assumed that the ingest job status is currently WAIT_ON_QUEUES. If ingest_job status can be set to COMPLETING, then this process "wins" and starts the completion process. Args: ingest_job: Ingest job model Returns: (dict): { status: (job status str), wait_secs: (int) - # seconds client should wait } Raises: (BossError): If completion process cannot be started or is already in process. """ completing_success = { 'job_status': IngestJob.COMPLETING, 'wait_secs': 0 } if ingest_job.status == IngestJob.COMPLETING: return completing_success try: self.ensure_queues_empty(ingest_job) except BossError as be: # Ensure state goes back to UPLOADING if the upload queue isn't # empty. if be.message == UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG: ingest_job.status = IngestJob.UPLOADING ingest_job.save() raise if ingest_job.status != IngestJob.WAIT_ON_QUEUES: raise BossError(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, ErrorCodes.BAD_REQUEST) wait_remaining = self.calculate_remaining_queue_wait(ingest_job) if wait_remaining > 0: return { 'job_status': IngestJob.WAIT_ON_QUEUES, 'wait_secs': wait_remaining } rows_updated = (IngestJob.objects .exclude(status=IngestJob.COMPLETING) .filter(id=ingest_job.id) .update(status=IngestJob.COMPLETING) ) # If successfully set status to COMPLETING, kick off the completion # process. Otherwise, completion already started. if rows_updated > 0: self._start_completion_activity(ingest_job) log = bossLogger() log.info(f"Started completion step function for job: {ingest_job.id}") return completing_success
from bosscore.serializers import CollectionSerializer, ExperimentSerializer, ChannelSerializer, \ CoordinateFrameSerializer, CoordinateFrameUpdateSerializer, ExperimentReadSerializer, ChannelReadSerializer, \ ExperimentUpdateSerializer, ChannelUpdateSerializer, CoordinateFrameDeleteSerializer from bosscore.models import Collection, Experiment, Channel, CoordinateFrame from bosscore.constants import ADMIN_GRP from bossutils.configuration import BossConfig from bossutils.logger import bossLogger boss_config = BossConfig() try: DEFAULT_CUBOID_BUCKET_NAME = 'cuboids.' + boss_config['system'][ 'fqdn'].split('.', 1)[1] except Exception as ex: DEFAULT_CUBOID_BUCKET_NAME = '' bossLogger().error(f'Failed getting system.fqdn from boss.config: {ex}') class CollectionDetail(APIView): """ View to access a collection object """ def get(self, request, collection): """ Get a single instance of a collection Args: request: DRF Request object collection: Collection name specifying the collection you want Returns:
from functools import wraps from django.conf import settings as django_settings from rest_framework.views import APIView from rest_framework.response import Response from bosscore.error import BossKeycloakError, BossHTTPError, ErrorCodes from bosscore.models import BossRole from bosscore.serializers import UserSerializer, BossRoleSerializer from bosscore.privileges import check_role, BossPrivilegeManager from bossutils.keycloak import KeyCloakClient, KeyCloakError from bossutils.logger import bossLogger LOG = bossLogger() LOCAL_KEYCLOAK_TESTING = getattr(django_settings, 'LOCAL_KEYCLOAK_TESTING', False) KEYCLOAK_ADMIN_USER = getattr(django_settings, 'KEYCLOAK_ADMIN_USER', '') KEYCLOAK_ADMIN_PASSWORD = getattr(django_settings, 'KEYCLOAK_ADMIN_PASSWORD', '') #### ## Should there be a hard coded list of valid roles, or shoulda all methods defer ## to Keycloak to make the check that the role is valid. Basically, do we expect ## for different applications to have their own roles? #### VALID_ROLES = ('admin', 'user-manager', 'resource-manager')
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if "filter" in request.query_params: ids = request.query_params["filter"] else: ids = None if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Define access mode. access_mode = utils.get_access_mode(request) if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, "ids": ids } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, self.bit_depth): return BossHTTPError( "Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * self.bit_depth / 8) # Calculating the number of bytes # seprate direction of movement from api and provide units of metric BossThrottle().check('cutout', ThrottleMetric.METRIC_TYPE_EGRESS, request.user, cost, ThrottleMetric.METRIC_UNITS_BYTES) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username or "public" }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') try: client.put_metric_data(Namespace="BOSS/Cutout", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'EgressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) except Exception as e: log = bossLogger() log.exception('Error during put_metric_data: {}'.format(e)) log.exception('Allowing bossDB to continue after logging') # Get interface to SPDB or CVDB cache if resource.get_channel().is_cloudvolume(): cache = CloudVolumeDB() else: cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], filter_ids=req.get_filter_ids(), iso=iso, access_mode=access_mode) to_renderer = {"time_request": req.time_request, "data": data} # Send data to renderer return Response(to_renderer)
def __init__(self): self.blog = bossLogger() self.metricdb = MetricDatabase() boss_config = bossutils.configuration.BossConfig() self.topic = boss_config['aws']['prod_mailing_list'] self.fqdn = boss_config['system']['fqdn']
def get(self, request, collection, experiment, channel, orientation, resolution, x_args, y_args, z_args, t_args=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param orientation: Image plane requested. Vaid options include xy,xz or yz :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Process request and validate try: request_args = { "service": "image", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "orientation": orientation, "resolution": resolution, "x_args": x_args, "y_args": y_args, "z_args": z_args, "time_args": t_args } req = BossRequest(request, request_args) except BossError as err: return err.to_http() #Define access mode access_mode = utils.get_access_mode(request) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * self.bit_depth / 8) # Calculating the number of bytes BossThrottle().check('image', ThrottleMetric.METRIC_TYPE_EGRESS, request.user, cost, ThrottleMetric.METRIC_UNITS_BYTES) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username or "public" }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') try: client.put_metric_data(Namespace="BOSS/Image", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'EgressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) except Exception as e: log = bossLogger() log.exception('Error during put_metric_data: {}'.format(e)) log.exception('Allowing bossDB to continue after logging') # Get interface to SPDB cache cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Do a cutout as specified data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], access_mode=access_mode) # Covert the cutout back to an image and return it if orientation == 'xy': img = data.xy_image() elif orientation == 'yz': img = data.yz_image() elif orientation == 'xz': img = data.xz_image() else: return BossHTTPError("Invalid orientation: {}".format(orientation), ErrorCodes.INVALID_CUTOUT_ARGS) return Response(img)
import boto3 import json import pymysql import pymysql.cursors import time from boss_db import get_db_connection from bossutils import logger from ndingest.nddynamo.boss_tileindexdb import TASK_INDEX, MAX_TASK_ID_SUFFIX, TILE_UPLOADED_MAP_KEY from ingestclient.core.backend import BossBackend """ Scans the tile index in DynamoDB for any chunks that have missing tiles. If tiles are missing, they are placed back in the upload queue for that ingest job. If there are missing tiles, the ingest job's state is reset to UPLOADING. """ log = logger.bossLogger() # Tile index attributes defined in ndingest.git/nddynamo/schemas/boss_tile_index.json. APPENDED_TASK_ID = 'appended_task_id' CHUNK_KEY = 'chunk_key' SQS_BATCH_SIZE = 10 SQS_RETRY_TIMEOUT = 15 # These values are defined in boss.git/django/bossingest/models.py. UPLOADING_STATUS = 1 WAIT_ON_QUEUES = 6 TILE_INGEST = 0 def activity_entry_point(args):
def __init__(self): self.blog = bossLogger() self.blog.info("MetricDatabase object created")
def post(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle POST requests for a cuboid of data while providing all datamodel params Due to parser implementation, request.data should be a numpy array already. :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which dataset or annotation project you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if isinstance(request.data, BossParserError): return request.data.to_http() # Check for optional iso flag if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Get BossRequest and BossResource from parser req = request.data[0] resource = request.data[1] # Get bit depth try: expected_data_type = resource.get_numpy_data_type() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure datatype is valid if expected_data_type != request.data[2].dtype: return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure the dimensions of the data match the dimensions of the post URL if len(request.data[2].shape) == 4: expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()) else: expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span()) if expected_shape != request.data[2].shape: return BossHTTPError( "Data dimensions in URL do not match POSTed data.", ErrorCodes.DATA_DIMENSION_MISMATCH) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * resource.get_bit_depth() / 8 ) # Calculating the number of bytes BossThrottle().check('cutout', ThrottleMetric.METRIC_TYPE_INGRESS, request.user, cost, ThrottleMetric.METRIC_UNITS_BYTES) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username or "public" }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') try: client.put_metric_data(Namespace="BOSS/Cutout", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'IngressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) except Exception as e: log = bossLogger() log.exception('Error during put_metric_data: {}'.format(e)) log.exception('Allowing bossDB to continue after logging') # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Write block to cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) try: if len(request.data[2].shape) == 4: cache.write_cuboid(resource, corner, req.get_resolution(), request.data[2], req.get_time()[0], iso=iso) else: cache.write_cuboid(resource, corner, req.get_resolution(), np.expand_dims(request.data[2], axis=0), req.get_time()[0], iso=iso) except Exception as e: # TODO: Eventually remove as this level of detail should not be sent to the user return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BOSS_SYSTEM_ERROR) # If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data channel = resource.get_channel() if channel.downsample_status.upper() == "DOWNSAMPLED": # Get Channel object and update status lookup_key = resource.get_lookup_key() _, exp_id, _ = lookup_key.split("&") channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "NOT_DOWNSAMPLED" channel_obj.downsample_arn = "" channel_obj.save() # Send data to renderer return HttpResponse(status=201)