def get(self, request, collection, experiment, dataset, resolution,
x_range, y_range, z_range):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param dataset: Dataset identifier, indicating which channel or layer you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
req = BossRequest(request)
except BossError as err:
return BossHTTPError(err.args[0], err.args[1], err.args[2])
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(
resource, corner, extent, req.get_resolution(),
[req.get_time().start, req.get_time().stop])
# Send data to renderer
return Response(data)
def get(self, request, collection, experiment, channel, resolution, id):
"""
Return the bounding box containing the object
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
resolution: Data resolution
id: The id of the object
Returns:
JSON dict with the bounding box of the object
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# validate that id is an int
# Check if this is annotation channel
if 'type' in request.query_params:
bb_type = request.query_params['type']
if bb_type != 'loose' and bb_type != 'tight':
return BossHTTPError("Invalid option for bounding box type {}. The valid options are : loose or tight"
.format(bb_type), ErrorCodes.INVALID_ARGUMENT)
else:
bb_type = 'loose'
try:
request_args = {
"service": "boundingbox",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"id": id
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
try:
# Get interface to SPDB cache
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG)
data = spdb.get_bounding_box(resource, int(resolution), int(id), bb_type=bb_type)
if data is None:
return BossHTTPError("The id does not exist. {}".format(id), ErrorCodes.OBJECT_NOT_FOUND)
return Response(data, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError("Type error in the boundingbox view. {}".format(e), ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment,channel, resolution, x_range, y_range, z_range, t_range=None):
"""
Return a list of ids in the spatial region.
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
num_ids: Number of id you want to reserve
Returns:
JSON dict with start_id and count of ids reserved
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# Check if this is annotation channel
# Process request and validate
try:
request_args = {
"service": "ids",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
try:
# Reserve ids
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG)
ids = spdb.get_ids_in_region(resource, int(resolution), corner, extent)
return Response(ids, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError("Type error in the ids view. {}".format(e), ErrorCodes.TYPE_ERROR)
def run(self):
# Setup SPDB instance
config = configuration.BossConfig()
kvio_config = {"cache_host": config['aws']['cache'],
"cache_db": config['aws']['cache-state-db'],
"read_timeout": 86400}
state_config = {"cache_state_host": config['aws']['cache-state'],
"cache_state_db": config['aws']['cache-db']}
object_store_config = {"s3_flush_queue": config['aws']["s3-flush-queue"],
"cuboid_bucket": config['aws']['cuboid_bucket'],
"page_in_lambda_function": config['lambda']['page_in_function'],
"page_out_lambda_function": config['lambda']['flush_function'],
"s3_index_table": config['aws']['s3-index-table'],
"id_index_table": config['aws']['id-index-table'],
"id_count_table": config['aws']['id-count-table']}
sp = SpatialDB(kvio_config,
state_config,
object_store_config)
while True:
self.log.info("Checking for delayed write operations.")
try:
self.process(sp)
time.sleep(5)
except Exception as err:
self.log.error("An error occurred running the process() method! \n {}".format(err))
def get(self, request, collection, experiment, channel, num_ids):
"""
Reserve a unique, sequential list of annotation ids for the provided channel to use as
object ids for annotations.
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
num_ids: Number of id you want to reserve
Returns:
JSON dict with start_id and count of ids reserved
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# validate that num_ids is an int
# Check if this is annotation channel
try:
request_args = {
"service": "reserve",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
try:
# Reserve ids
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
start_id = spdb.reserve_ids(resource, int(num_ids))
data = {'start_id': start_id[0], 'count': num_ids}
return Response(data, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError(
"Type error in the reserve id view. {}".format(e),
ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment, channel, num_ids):
"""
Reserve a unique, sequential list of annotation ids for the provided channel to use as
object ids for annotations.
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
num_ids: Number of id you want to reserve
Returns:
JSON dict with start_id and count of ids reserved
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# validate that num_ids is an int
# Check if this is annotation channel
try:
request_args = {
"service": "reserve",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
try:
# Reserve ids
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG)
start_id = spdb.reserve_ids(resource, int(num_ids))
data = {'start_id': start_id[0], 'count': num_ids}
return Response(data, status=200)
except (TypeError, ValueError)as e:
return BossHTTPError("Type error in the reserve id view. {}".format(e), ErrorCodes.TYPE_ERROR)
def get(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Define access mode.
access_mode = utils.get_access_mode(request)
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError(
"Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) * self.bit_depth /
8) # Calculating the number of bytes
BossThrottle().check('cutout_egress', request.user, cost)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
client.put_metric_data(Namespace="BOSS/Cutout",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'EgressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(),
iso=iso,
access_mode=access_mode)
to_renderer = {"time_request": req.time_request, "data": data}
# Send data to renderer
return Response(to_renderer)
def post(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle POST requests for a cuboid of data while providing all datamodel params
Due to parser implementation, request.data should be a numpy array already.
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Check for optional iso flag
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Get BossRequest and BossResource from parser
req = request.data[0]
resource = request.data[1]
# Get bit depth
try:
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure datatype is valid
if expected_data_type != request.data[2].dtype:
return BossHTTPError("Datatype does not match channel",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure the dimensions of the data match the dimensions of the post URL
if len(request.data[2].shape) == 4:
expected_shape = (len(req.get_time()), req.get_z_span(),
req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(),
req.get_x_span())
if expected_shape != request.data[2].shape:
return BossHTTPError(
"Data dimensions in URL do not match POSTed data.",
ErrorCodes.DATA_DIMENSION_MISMATCH)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) *
resource.get_bit_depth() / 8
) # Calculating the number of bytes
BossThrottle().check('cutout_ingress', request.user, cost)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
client.put_metric_data(Namespace="BOSS/Cutout",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'IngressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(request.data[2].shape) == 4:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
request.data[2],
req.get_time()[0],
iso=iso)
else:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
np.expand_dims(request.data[2], axis=0),
req.get_time()[0],
iso=iso)
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
return BossHTTPError('Error during write_cuboid: {}'.format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
# If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data
channel = resource.get_channel()
if channel.downsample_status.upper() == "DOWNSAMPLED":
# Get Channel object and update status
lookup_key = resource.get_lookup_key()
_, exp_id, _ = lookup_key.split("&")
channel_obj = Channel.objects.get(name=channel.name,
experiment=int(exp_id))
channel_obj.downsample_status = "NOT_DOWNSAMPLED"
channel_obj.downsample_arn = ""
channel_obj.save()
# Send data to renderer
return HttpResponse(status=201)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
if "no-cache" in request.query_params:
if request.query_params["no-cache"].lower() == "true":
no_cache = True
else:
no_cache = False
else:
no_cache = False
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(), iso=iso, no_cache=no_cache)
to_renderer = {"time_request": req.time_request,
"data": data}
# Send data to renderer
return Response(to_renderer)
def post(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None):
"""
View to handle POST requests for a cuboid of data while providing all datamodel params
Due to parser implementation, request.data should be a numpy array already.
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Get BossRequest and BossResource from parser
req = request.data[0]
resource = request.data[1]
# Get bit depth
try:
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR)
# Make sure datatype is valid
if expected_data_type != request.data[2].dtype:
return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure the dimensions of the data match the dimensions of the post URL
if len(request.data[2].shape) == 4:
expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span())
if expected_shape != request.data[2].shape:
return BossHTTPError("Data dimensions in URL do not match POSTed data.",
ErrorCodes.DATA_DIMENSION_MISMATCH)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(request.data[2].shape) == 4:
cache.write_cuboid(resource, corner, req.get_resolution(), request.data[2], req.get_time()[0])
else:
cache.write_cuboid(resource, corner, req.get_resolution(),
np.expand_dims(request.data[2], axis=0), req.get_time()[0])
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BOSS_SYSTEM_ERROR)
# Send data to renderer
return HttpResponse(status=201)
def get(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
Return a list of ids in the spatial region.
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
num_ids: Number of id you want to reserve
Returns:
JSON dict with start_id and count of ids reserved
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# Check if this is annotation channel
# Process request and validate
try:
request_args = {
"service": "ids",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
try:
# Reserve ids
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
ids = spdb.get_ids_in_region(resource, int(resolution), corner,
extent)
return Response(ids, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError("Type error in the ids view. {}".format(e),
ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment, channel, resolution, id):
"""
Return the bounding box containing the object
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
resolution: Data resolution
id: The id of the object
Returns:
JSON dict with the bounding box of the object
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# validate that id is an int
# Check if this is annotation channel
if 'type' in request.query_params:
bb_type = request.query_params['type']
if bb_type != 'loose' and bb_type != 'tight':
return BossHTTPError(
"Invalid option for bounding box type {}. The valid options are : loose or tight"
.format(bb_type), ErrorCodes.INVALID_ARGUMENT)
else:
bb_type = 'loose'
try:
request_args = {
"service": "boundingbox",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"id": id
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
try:
# Get interface to SPDB cache
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
data = spdb.get_bounding_box(resource,
int(resolution),
int(id),
bb_type=bb_type)
if data is None:
return BossHTTPError("The id does not exist. {}".format(id),
ErrorCodes.OBJECT_NOT_FOUND)
return Response(data, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError(
"Type error in the boundingbox view. {}".format(e),
ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Define access mode.
access_mode = utils.get_access_mode(request)
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(), iso=iso, access_mode=access_mode)
to_renderer = {"time_request": req.time_request,
"data": data}
# Send data to renderer
return Response(to_renderer)
def post(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None):
"""
View to handle POST requests for a cuboid of data while providing all datamodel params
Due to parser implementation, request.data should be a numpy array already.
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Check for optional iso flag
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Get BossRequest and BossResource from parser
req = request.data[0]
resource = request.data[1]
# Get bit depth
try:
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR)
# Make sure datatype is valid
if expected_data_type != request.data[2].dtype:
return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure the dimensions of the data match the dimensions of the post URL
if len(request.data[2].shape) == 4:
expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span())
if expected_shape != request.data[2].shape:
return BossHTTPError("Data dimensions in URL do not match POSTed data.",
ErrorCodes.DATA_DIMENSION_MISMATCH)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(request.data[2].shape) == 4:
cache.write_cuboid(resource, corner, req.get_resolution(), request.data[2], req.get_time()[0], iso=iso)
else:
cache.write_cuboid(resource, corner, req.get_resolution(),
np.expand_dims(request.data[2], axis=0), req.get_time()[0], iso=iso)
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BOSS_SYSTEM_ERROR)
# If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data
channel = resource.get_channel()
if channel.downsample_status.upper() == "DOWNSAMPLED":
# Get Channel object and update status
lookup_key = resource.get_lookup_key()
_, exp_id, _ = lookup_key.split("&")
channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id))
channel_obj.downsample_status = "NOT_DOWNSAMPLED"
channel_obj.downsample_arn = ""
channel_obj.save()
# Send data to renderer
return HttpResponse(status=201)
def put(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle PUT requests for a overwriting a cuboid to 0s
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check for optional iso flag
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get data type
try:
self.bit_depth = resource.get_bit_depth()
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Set a limit of CUTOUT_MAX_SIZE
if is_too_large(req, self.bit_depth):
return BossHTTPError(
"Cutout overwrite is over 500MB when uncompressed. Reduce overwrite dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get the shape of the requested data clear
if len(req.get_time()) > 2:
expected_shape = (len(req.get_time()), req.get_z_span(),
req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(),
req.get_x_span())
# Create a binary numpy array for overwrite with specified shape and dtype
black_cuboid = np.ones(expected_shape, dtype=expected_data_type)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(black_cuboid.shape) == 4:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
black_cuboid,
req.get_time()[0],
iso=iso,
to_black=True)
else:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
np.expand_dims(black_cuboid, axis=0),
req.get_time()[0],
iso=iso,
to_black=True)
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
log = BossLogger().logger
log.exception('Error during write_cuboid: {}'.format(e))
return BossHTTPError('Error during write_cuboid: {}'.format(e),
ErrorCodes.BAD_REQUEST)
# If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data
channel = resource.get_channel()
if channel.downsample_status.upper() == "DOWNSAMPLED":
# Get Channel object and update status
lookup_key = resource.get_lookup_key()
_, exp_id, _ = lookup_key.split("&")
channel_obj = Channel.objects.get(name=channel.name,
experiment=int(exp_id))
channel_obj.downsample_status = "NOT_DOWNSAMPLED"
channel_obj.downsample_arn = ""
channel_obj.save()
# Send data to renderer
return HttpResponse(status=200)