def delete(self, request, ingest_job_id):
"""
Args:
request:
ingest_job_id:
Returns:
"""
try:
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check permissions
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can cancel an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.DELETED)
blog = bossLogger()
blog.info("Deleted Ingest Job {}".format(ingest_job_id))
return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
def __init__(self, message, code=ErrorCodes.KEYCLOAK_EXCEPTION):
"""
Custom HTTP error class for converting a KeyCloakError exception into a JsonResponse
Args:
message (str): Error message to send back to user
Note: If called in an exception handler, expects the exception to be a KeyCloakError
"""
ex = sys.exc_info()[1]
self.status_code = ex.status if ex else RESP_CODES[code]
data = {
'status': self.status_code,
'code': ErrorCodes.KEYCLOAK_EXCEPTION,
'message': message
}
if ex:
if isinstance(ex.data, str):
val = json.loads(ex.data)
else:
val = ex.data
data.update(val)
msg = "BossKeycloakError"
for k in data:
msg += " - {}: {}".format(k.capitalize(), data[k])
log = bossLogger()
log.info(msg)
super(BossKeycloakError, self).__init__(data)
def get_session(self):
"""
Method to get a boto3.session.Session object.
If session credentials have expired a new session is created.
If no sessions are available (if a previous consumer of a session didn't return it for some reason, e.g. an
exception occurred), a new session is created
:return: boto3.session.Session
"""
temp_session = None
while not temp_session:
try:
temp_session = self.__sessions.get(block=False)
except queue.Empty:
# No session was available so generate one
blog = bossLogger()
blog.info(
"AWSManager - No session was available while trying to execute get_session. Dynamically creating a new session."
)
self.__create_session()
return temp_session
def lambda_connect_sqs(self, queue, lambda_name, num_msgs=1):
"""
Adds an SQS event trigger to the given lambda.
Args:
queue (SQS.Queue): SQS queue that will be the trigger source.
lambda_name (str): Lambda function name.
num_msgs (optional[int]): Number of messages to send to the lambda. Defaults to 1, max 10.
Raises:
(ValueError): if num_msgs is greater than the SQS max batch size.
"""
if num_msgs < 1 or num_msgs > MAX_SQS_BATCH_SIZE:
raise ValueError('lambda_connect_sqs(): Bad num_msgs: {}'.format(num_msgs))
queue_arn = queue.attributes['QueueArn']
timeout = self.get_ingest_lambda_timeout(lambda_name)
# AWS recommends that an SQS queue used as a lambda event source should
# have a visibility timeout that's 6 times the lambda's timeout.
queue.set_attributes(Attributes={'VisibilityTimeout': str(timeout * 6)})
client = boto3.client('lambda', region_name=bossutils.aws.get_region())
try:
client.create_event_source_mapping(
EventSourceArn=queue_arn,
FunctionName=lambda_name,
BatchSize=num_msgs)
except client.exceptions.ResourceConflictException:
log = bossLogger()
log.warning(f'ResourceConflictException caught trying to connect {queue_arn} to {lambda_name}. This should be harmless because this happens when the queue has already been connected.')
def wrapper(self, *args, **kwargs):
try:
return function(self, *args, **kwargs)
except hvac.exceptions.Forbidden as e:
blog = bossLogger()
blog.info(str(e))
msg = "Your token had expired. Dynamically creating a new one."
blog.info(msg)
Vault.login(self)
return function(self, *args, **kwargs)
def __create_session(self):
"""
Method to create a new boto3.session.Session object and add it to the pool
:return: None
"""
temp_session = get_session()
blog = bossLogger()
blog.info(
"AWSManager - Created new boto3 session and added to the pool")
self.__sessions.put(temp_session)
def get_ingest_lambda_timeout(self, name):
"""
Get the current timeout of the tile ingest lambda.
Args:
name (str): Name of lambda.
Returns:
(int): Number of seconds of the timeout.
"""
client = boto3.client('lambda', region_name=bossutils.aws.get_region())
try:
resp = client.get_function(FunctionName=name)
return resp['Configuration']['Timeout']
except Exception as ex:
log = bossLogger()
log.error(f"Couldn't get lambda: {name} data from AWS: {ex}")
raise
def __init__(self, message, code):
"""
Custom HTTP error class
:param status: HTTP Status code
:type status: int
:param code: An optional, arbitrary, and unique code to identify where the error was generated
:type code: int
:param message: Message to provide feedback to the user
:return:
"""
# Set status code
self.status_code = RESP_CODES[code]
# Log
blog = bossLogger()
blog.info(
"BossHTTPError - Status: {0} - Code: {1} - Message: {2}".format(
self.status_code, code, message))
# Return
data = {'status': self.status_code, 'code': code, 'message': message}
super(BossHTTPError, self).__init__(data)
def __init__(self, pid_file_name, pid_dir="/var/run"):
super().__init__(pid_file_name, pid_dir)
self.log = logger.bossLogger()
self.config = configuration.BossConfig()
# kvio settings
kvio_config = {
"cache_host": self.config['aws']['cache'],
"cache_db": self.config['aws']['cache-db'],
"read_timeout": 86400
}
# state settings
state_config = {
"cache_state_host": self.config['aws']['cache-state'],
"cache_state_db": self.config['aws']['cache-state-db']
}
# object store settings
object_store_config = {
"s3_flush_queue": self.config["aws"]["s3-flush-queue"],
"cuboid_bucket": self.config['aws']['cuboid_bucket'],
"page_in_lambda_function":
self.config['lambda']['page_in_function'],
"page_out_lambda_function":
self.config['lambda']['flush_function'],
"s3_index_table": self.config['aws']['s3-index-table'],
"id_index_table": self.config['aws']['id-index-table'],
"id_count_table": self.config['aws']['id-count-table']
}
config_data = {
"kv_config": kvio_config,
"state_config": state_config,
"object_store_config": object_store_config
}
self.lambda_data = {"config": config_data, "lambda-name": "s3_flush"}
self.sqs_watcher = SqsWatcher(self.lambda_data)
def remove_sqs_event_source_from_lambda(self, queue_arn, lambda_name):
"""
Removes an SQS event triggger from the given lambda.
Args:
queue_arn (str): Arn of SQS queue that will be the trigger source.
lambda_name (str): Lambda function name.
"""
log = bossLogger()
client = boto3.client('lambda', region_name=bossutils.aws.get_region())
try:
resp = client.list_event_source_mappings(
EventSourceArn=queue_arn,
FunctionName=lambda_name)
except Exception as ex:
log.error(f"Couldn't list event source mappings for {lambda_name}: {ex}")
return
for evt in resp['EventSourceMappings']:
try:
client.delete_event_source_mapping(UUID=evt['UUID'])
except client.exceptions.ResourceNotFoundException:
pass
except Exception as ex:
log.error(f"Couldn't remove event source mapping {queue_arn} from {lambda_name}: {ex}")
def __init__(self, lambda_data):
self.lambda_data = lambda_data
self.log = logger.bossLogger()
self.old_message_num = 0
self.message_num = 0
def post(self, request, ingest_job_id):
"""
Signal an ingest job is complete and should be cleaned up by POSTing to this view
Args:
request: Django Rest framework Request object
ingest_job_id: Ingest job id
Returns:
"""
try:
blog = bossLogger()
ingest_mgmr = IngestManager()
ingest_job = ingest_mgmr.get_ingest_job(ingest_job_id)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
if ingest_job.status == IngestJob.PREPARING:
# If status is Preparing. Deny
return BossHTTPError(
"You cannot complete a job that is still preparing. You must cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.UPLOADING:
try:
data = ingest_mgmr.try_enter_wait_on_queue_state(
ingest_job)
return Response(data=data, status=status.HTTP_202_ACCEPTED)
except BossError as be:
if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG or
be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG):
# If there are messages in the tile error queue, this
# will have to be handled manually. Non-empty ingest
# or tile index queues should resolve on their own.
return Response(data={
'wait_secs': WAIT_FOR_QUEUES_SECS,
'info': 'Internal queues not empty yet'
},
status=status.HTTP_400_BAD_REQUEST)
raise
elif ingest_job.status == IngestJob.WAIT_ON_QUEUES:
pass
# Continue below.
elif ingest_job.status == IngestJob.COMPLETE:
# If status is already Complete, just return another 204
return Response(data={'job_status': ingest_job.status},
status=status.HTTP_204_NO_CONTENT)
elif ingest_job.status == IngestJob.DELETED:
# Job had already been cancelled
return BossHTTPError("Ingest job has already been cancelled.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.FAILED:
# Job had failed
return BossHTTPError(
"Ingest job has failed during creation. You must Cancel instead.",
ErrorCodes.BAD_REQUEST)
elif ingest_job.status == IngestJob.COMPLETING:
return Response(data={'job_status': ingest_job.status},
status=status.HTTP_202_ACCEPTED)
# Check if user is the ingest job creator or the sys admin
if not self.is_user_or_admin(request, ingest_job):
return BossHTTPError(
"Only the creator or admin can complete an ingest job",
ErrorCodes.INGEST_NOT_CREATOR)
# Try to start completing.
try:
data = ingest_mgmr.try_start_completing(ingest_job)
if data['job_status'] == IngestJob.WAIT_ON_QUEUES:
# Refuse complete requests until wait period expires.
return Response(data=data,
status=status.HTTP_400_BAD_REQUEST)
except BossError as be:
if (be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG
or be.message == TILE_INDEX_QUEUE_NOT_EMPTY_ERR_MSG
or be.message == INGEST_QUEUE_NOT_EMPTY_ERR_MSG):
return Response(data={
'wait_secs': WAIT_FOR_QUEUES_SECS,
'info': 'Internal queues not empty yet'
},
status=status.HTTP_400_BAD_REQUEST)
raise
blog.info("Completion process started for ingest Job {}".format(
ingest_job_id))
return Response(data=data, status=status.HTTP_202_ACCEPTED)
# TODO SH This is a quick fix to make sure the ingest-client does not run close option.
# the clean up code commented out below, because it is not working correctly.
# return Response(status=status.HTTP_204_NO_CONTENT)
# if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# # Check if any messages remain in the ingest queue
# ingest_queue = ingest_mgmr.get_ingest_job_ingest_queue(ingest_job)
# num_messages_in_queue = int(ingest_queue.queue.attributes['ApproximateNumberOfMessages'])
#
# # Kick off extra lambdas just in case
# if num_messages_in_queue:
# blog.info("{} messages remaining in Ingest Queue".format(num_messages_in_queue))
# ingest_mgmr.invoke_ingest_lambda(ingest_job, num_messages_in_queue)
#
# # Give lambda a few seconds to fire things off
# time.sleep(30)
#
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# # ToDo: call cleanup method for volumetric ingests. Don't want
# # to cleanup until after testing with real data.
# #ingest_mgmr.cleanup_ingest_job(ingest_job, IngestJob.COMPLETE)
#
# blog.info("Complete successful")
# return Response(status=status.HTTP_204_NO_CONTENT)
except BossError as err:
return err.to_http()
except Exception as err:
blog.error('Caught general exception: {}'.format(err))
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def track_usage_data(self, ingest_config_data, request):
"""
Set up usage tracking of this ingest.
Args:
ingest_config_data (dict): Ingest job config.
Raises:
(BossError): if user doesn't have permission for a large ingest.
"""
# need to get bytes per pixel to caculate ingest in total bytes
try:
ingest_job = ingest_config_data['ingest_job']
theCollection = Collection.objects.get(
name=ingest_config_data['database']['collection'])
theExperiment = Experiment.objects.get(
name=ingest_config_data['database']['experiment'])
theChannel = Channel.objects.get(
name=ingest_config_data['database']['channel'])
bytesPerPixel = int(theChannel.datatype.replace("uint", "")) / 8
# Add metrics to CloudWatch
extent = ingest_job['extent']
database = ingest_config_data['database']
# Check that only permitted users are creating extra large ingests
try:
group = Group.objects.get(name=INGEST_GRP)
in_large_ingest_group = group.user_set.filter(
id=request.user.id).exists()
except Group.DoesNotExist:
# Just in case the group has not been created yet
in_large_ingest_group = False
if (not in_large_ingest_group) and \
((extent['x'][1] - extent['x'][0]) * \
(extent['y'][1] - extent['y'][0]) * \
(extent['z'][1] - extent['z'][0]) * \
(extent['t'][1] - extent['t'][0]) > settings.INGEST_MAX_SIZE):
raise BossError(
"Large ingests require special permission to create. Contact system administrator.",
ErrorCodes.INVALID_STATE)
# Calculate the cost of the ingest in pixels
costInPixels = ((extent['x'][1] - extent['x'][0]) *
(extent['y'][1] - extent['y'][0]) *
(extent['z'][1] - extent['z'][0]) *
(extent['t'][1] - extent['t'][0]))
cost = costInPixels * bytesPerPixel
BossThrottle().check('ingest', ThrottleMetric.METRIC_TYPE_INGRESS,
request.user, cost,
ThrottleMetric.METRIC_UNITS_BYTES)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username or "public"
},
{
'Name':
'Resource',
'Value':
'{}/{}/{}'.format(database['collection'],
database['experiment'],
database['channel'])
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
try:
client.put_metric_data(Namespace="BOSS/Ingest",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'IngressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
except Exception as e:
log = bossLogger()
log.exception('Error during put_metric_data: {}'.format(e))
log.exception('Allowing bossDB to continue after logging')
except BossError as err:
return err.to_http()
except Exception as err:
return BossError("{}".format(err),
ErrorCodes.BOSS_SYSTEM_ERROR).to_http()
def try_start_completing(self, ingest_job):
"""
Tries to start completion process.
It is assumed that the ingest job status is currently WAIT_ON_QUEUES.
If ingest_job status can be set to COMPLETING, then this process "wins"
and starts the completion process.
Args:
ingest_job: Ingest job model
Returns:
(dict): { status: (job status str), wait_secs: (int) - # seconds client should wait }
Raises:
(BossError): If completion process cannot be started or is already
in process.
"""
completing_success = {
'job_status': IngestJob.COMPLETING,
'wait_secs': 0
}
if ingest_job.status == IngestJob.COMPLETING:
return completing_success
try:
self.ensure_queues_empty(ingest_job)
except BossError as be:
# Ensure state goes back to UPLOADING if the upload queue isn't
# empty.
if be.message == UPLOAD_QUEUE_NOT_EMPTY_ERR_MSG:
ingest_job.status = IngestJob.UPLOADING
ingest_job.save()
raise
if ingest_job.status != IngestJob.WAIT_ON_QUEUES:
raise BossError(NOT_IN_WAIT_ON_QUEUES_STATE_ERR_MSG, ErrorCodes.BAD_REQUEST)
wait_remaining = self.calculate_remaining_queue_wait(ingest_job)
if wait_remaining > 0:
return {
'job_status': IngestJob.WAIT_ON_QUEUES,
'wait_secs': wait_remaining
}
rows_updated = (IngestJob.objects
.exclude(status=IngestJob.COMPLETING)
.filter(id=ingest_job.id)
.update(status=IngestJob.COMPLETING)
)
# If successfully set status to COMPLETING, kick off the completion
# process. Otherwise, completion already started.
if rows_updated > 0:
self._start_completion_activity(ingest_job)
log = bossLogger()
log.info(f"Started completion step function for job: {ingest_job.id}")
return completing_success
from bosscore.serializers import CollectionSerializer, ExperimentSerializer, ChannelSerializer, \
CoordinateFrameSerializer, CoordinateFrameUpdateSerializer, ExperimentReadSerializer, ChannelReadSerializer, \
ExperimentUpdateSerializer, ChannelUpdateSerializer, CoordinateFrameDeleteSerializer
from bosscore.models import Collection, Experiment, Channel, CoordinateFrame
from bosscore.constants import ADMIN_GRP
from bossutils.configuration import BossConfig
from bossutils.logger import bossLogger
boss_config = BossConfig()
try:
DEFAULT_CUBOID_BUCKET_NAME = 'cuboids.' + boss_config['system'][
'fqdn'].split('.', 1)[1]
except Exception as ex:
DEFAULT_CUBOID_BUCKET_NAME = ''
bossLogger().error(f'Failed getting system.fqdn from boss.config: {ex}')
class CollectionDetail(APIView):
"""
View to access a collection object
"""
def get(self, request, collection):
"""
Get a single instance of a collection
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
Returns:
from functools import wraps
from django.conf import settings as django_settings
from rest_framework.views import APIView
from rest_framework.response import Response
from bosscore.error import BossKeycloakError, BossHTTPError, ErrorCodes
from bosscore.models import BossRole
from bosscore.serializers import UserSerializer, BossRoleSerializer
from bosscore.privileges import check_role, BossPrivilegeManager
from bossutils.keycloak import KeyCloakClient, KeyCloakError
from bossutils.logger import bossLogger
LOG = bossLogger()
LOCAL_KEYCLOAK_TESTING = getattr(django_settings, 'LOCAL_KEYCLOAK_TESTING',
False)
KEYCLOAK_ADMIN_USER = getattr(django_settings, 'KEYCLOAK_ADMIN_USER', '')
KEYCLOAK_ADMIN_PASSWORD = getattr(django_settings, 'KEYCLOAK_ADMIN_PASSWORD',
'')
####
## Should there be a hard coded list of valid roles, or shoulda all methods defer
## to Keycloak to make the check that the role is valid. Basically, do we expect
## for different applications to have their own roles?
####
VALID_ROLES = ('admin', 'user-manager', 'resource-manager')
def get(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Define access mode.
access_mode = utils.get_access_mode(request)
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError(
"Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) * self.bit_depth /
8) # Calculating the number of bytes
# seprate direction of movement from api and provide units of metric
BossThrottle().check('cutout', ThrottleMetric.METRIC_TYPE_EGRESS,
request.user, cost,
ThrottleMetric.METRIC_UNITS_BYTES)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username or "public"
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
try:
client.put_metric_data(Namespace="BOSS/Cutout",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'EgressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
except Exception as e:
log = bossLogger()
log.exception('Error during put_metric_data: {}'.format(e))
log.exception('Allowing bossDB to continue after logging')
# Get interface to SPDB or CVDB cache
if resource.get_channel().is_cloudvolume():
cache = CloudVolumeDB()
else:
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(),
iso=iso,
access_mode=access_mode)
to_renderer = {"time_request": req.time_request, "data": data}
# Send data to renderer
return Response(to_renderer)
def __init__(self):
self.blog = bossLogger()
self.metricdb = MetricDatabase()
boss_config = bossutils.configuration.BossConfig()
self.topic = boss_config['aws']['prod_mailing_list']
self.fqdn = boss_config['system']['fqdn']
def get(self,
request,
collection,
experiment,
channel,
orientation,
resolution,
x_args,
y_args,
z_args,
t_args=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param orientation: Image plane requested. Vaid options include xy,xz or yz
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Process request and validate
try:
request_args = {
"service": "image",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"orientation": orientation,
"resolution": resolution,
"x_args": x_args,
"y_args": y_args,
"z_args": z_args,
"time_args": t_args
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
#Define access mode
access_mode = utils.get_access_mode(request)
# Convert to Resource
resource = spdb.project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Datatype does not match channel",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) * self.bit_depth /
8) # Calculating the number of bytes
BossThrottle().check('image', ThrottleMetric.METRIC_TYPE_EGRESS,
request.user, cost,
ThrottleMetric.METRIC_UNITS_BYTES)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username or "public"
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
try:
client.put_metric_data(Namespace="BOSS/Image",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'EgressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
except Exception as e:
log = bossLogger()
log.exception('Error during put_metric_data: {}'.format(e))
log.exception('Allowing bossDB to continue after logging')
# Get interface to SPDB cache
cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Do a cutout as specified
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
access_mode=access_mode)
# Covert the cutout back to an image and return it
if orientation == 'xy':
img = data.xy_image()
elif orientation == 'yz':
img = data.yz_image()
elif orientation == 'xz':
img = data.xz_image()
else:
return BossHTTPError("Invalid orientation: {}".format(orientation),
ErrorCodes.INVALID_CUTOUT_ARGS)
return Response(img)
import boto3 import json import pymysql import pymysql.cursors import time from boss_db import get_db_connection from bossutils import logger from ndingest.nddynamo.boss_tileindexdb import TASK_INDEX, MAX_TASK_ID_SUFFIX, TILE_UPLOADED_MAP_KEY from ingestclient.core.backend import BossBackend """ Scans the tile index in DynamoDB for any chunks that have missing tiles. If tiles are missing, they are placed back in the upload queue for that ingest job. If there are missing tiles, the ingest job's state is reset to UPLOADING. """ log = logger.bossLogger() # Tile index attributes defined in ndingest.git/nddynamo/schemas/boss_tile_index.json. APPENDED_TASK_ID = 'appended_task_id' CHUNK_KEY = 'chunk_key' SQS_BATCH_SIZE = 10 SQS_RETRY_TIMEOUT = 15 # These values are defined in boss.git/django/bossingest/models.py. UPLOADING_STATUS = 1 WAIT_ON_QUEUES = 6 TILE_INGEST = 0 def activity_entry_point(args):
def __init__(self):
self.blog = bossLogger()
self.blog.info("MetricDatabase object created")
def post(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle POST requests for a cuboid of data while providing all datamodel params
Due to parser implementation, request.data should be a numpy array already.
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Check for optional iso flag
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Get BossRequest and BossResource from parser
req = request.data[0]
resource = request.data[1]
# Get bit depth
try:
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure datatype is valid
if expected_data_type != request.data[2].dtype:
return BossHTTPError("Datatype does not match channel",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure the dimensions of the data match the dimensions of the post URL
if len(request.data[2].shape) == 4:
expected_shape = (len(req.get_time()), req.get_z_span(),
req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(),
req.get_x_span())
if expected_shape != request.data[2].shape:
return BossHTTPError(
"Data dimensions in URL do not match POSTed data.",
ErrorCodes.DATA_DIMENSION_MISMATCH)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) *
resource.get_bit_depth() / 8
) # Calculating the number of bytes
BossThrottle().check('cutout', ThrottleMetric.METRIC_TYPE_INGRESS,
request.user, cost,
ThrottleMetric.METRIC_UNITS_BYTES)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username or "public"
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
try:
client.put_metric_data(Namespace="BOSS/Cutout",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'IngressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
except Exception as e:
log = bossLogger()
log.exception('Error during put_metric_data: {}'.format(e))
log.exception('Allowing bossDB to continue after logging')
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(request.data[2].shape) == 4:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
request.data[2],
req.get_time()[0],
iso=iso)
else:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
np.expand_dims(request.data[2], axis=0),
req.get_time()[0],
iso=iso)
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
return BossHTTPError('Error during write_cuboid: {}'.format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
# If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data
channel = resource.get_channel()
if channel.downsample_status.upper() == "DOWNSAMPLED":
# Get Channel object and update status
lookup_key = resource.get_lookup_key()
_, exp_id, _ = lookup_key.split("&")
channel_obj = Channel.objects.get(name=channel.name,
experiment=int(exp_id))
channel_obj.downsample_status = "NOT_DOWNSAMPLED"
channel_obj.downsample_arn = ""
channel_obj.save()
# Send data to renderer
return HttpResponse(status=201)
