def test_request_tile_init_tileargs_channel(self):
"""
Test initialization of tile arguments for a tile request for a channel
:return:
"""
url = '/' + version + '/tile/col1/exp1/channel1/xy/512/2/0/0/1'
res = 2
(x_start, x_stop) = (0, 512)
(y_start, y_stop) = (0, 512)
(z_start, z_stop) = (1, 2)
# Create the request
req = HttpRequest()
req.META = {'PATH_INFO': url}
drfrequest = Request(req)
drfrequest.version = version
ret = BossRequest(drfrequest)
self.assertEqual(ret.get_resolution(), res)
self.assertEqual(ret.get_x_start(), x_start)
self.assertEqual(ret.get_x_stop(), x_stop)
self.assertEqual(ret.get_x_span(), x_stop - x_start)
self.assertEqual(ret.get_y_start(), y_start)
self.assertEqual(ret.get_y_stop(), y_stop)
self.assertEqual(ret.get_y_span(), y_stop - y_start)
self.assertEqual(ret.get_z_start(), z_start)
self.assertEqual(ret.get_z_stop(), z_stop)
self.assertEqual(ret.get_z_span(), z_stop - z_start)
def get(self, request, collection, experiment, dataset, resolution,
x_range, y_range, z_range):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param dataset: Dataset identifier, indicating which channel or layer you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
req = BossRequest(request)
except BossError as err:
return BossHTTPError(err.args[0], err.args[1], err.args[2])
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(
resource, corner, extent, req.get_resolution(),
[req.get_time().start, req.get_time().stop])
# Send data to renderer
return Response(data)
def get(self, request, collection, experiment,channel, resolution, x_range, y_range, z_range, t_range=None):
"""
Return a list of ids in the spatial region.
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
num_ids: Number of id you want to reserve
Returns:
JSON dict with start_id and count of ids reserved
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# Check if this is annotation channel
# Process request and validate
try:
request_args = {
"service": "ids",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
try:
# Reserve ids
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG)
ids = spdb.get_ids_in_region(resource, int(resolution), corner, extent)
return Response(ids, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError("Type error in the ids view. {}".format(e), ErrorCodes.TYPE_ERROR)
def test_request_tile_init_tileargs_channel(self):
"""
Test initialization of tile arguments for a tile request for a channel
:return:
"""
url = '/' + version + '/tile/col1/exp1/channel1/xy/512/2/0/0/1'
res = 2
(x_start, x_stop) = (0, 512)
(y_start, y_stop) = (0, 512)
(z_start, z_stop) = (1, 2)
# Create the request dict
request_args = {
"service": "tile",
"collection_name": "col1",
"experiment_name": "exp1",
"channel_name": "channel1",
"orientation": "xy",
"tile_size": 512,
"resolution": 2,
"x_args": "0",
"y_args": "0",
"z_args": "1",
"time_args": None
}
# Create the request
request = self.rf.get(url)
force_authenticate(request, user=self.user)
drfrequest = Tile().initialize_request(request)
drfrequest.version = version
ret = BossRequest(drfrequest, request_args)
self.assertEqual(ret.get_resolution(), res)
self.assertEqual(ret.get_x_start(), x_start)
self.assertEqual(ret.get_x_stop(), x_stop)
self.assertEqual(ret.get_x_span(), x_stop - x_start)
self.assertEqual(ret.get_y_start(), y_start)
self.assertEqual(ret.get_y_stop(), y_stop)
self.assertEqual(ret.get_y_span(), y_stop - y_start)
self.assertEqual(ret.get_z_start(), z_start)
self.assertEqual(ret.get_z_stop(), z_stop)
self.assertEqual(ret.get_z_span(), z_stop - z_start)
def test_request_tile_init_tileargs_channel(self):
"""
Test initialization of tile arguments for a tile request for a channel
:return:
"""
url = '/' + version + '/tile/col1/exp1/channel1/xy/512/2/0/0/1'
res = 2
(x_start, x_stop) = (0, 512)
(y_start, y_stop) = (0, 512)
(z_start, z_stop) = (1, 2)
# Create the request dict
request_args = {
"service": "tile",
"collection_name": "col1",
"experiment_name": "exp1",
"channel_name": "channel1",
"orientation": "xy",
"tile_size": 512,
"resolution": 2,
"x_args": "0",
"y_args": "0",
"z_args": "1",
"time_args": None
}
# Create the request
request = self.rf.get(url)
force_authenticate(request, user=self.user)
drfrequest = Tile().initialize_request(request)
drfrequest.version = version
ret = BossRequest(drfrequest, request_args)
self.assertEqual(ret.get_resolution(), res)
self.assertEqual(ret.get_x_start(), x_start)
self.assertEqual(ret.get_x_stop(), x_stop)
self.assertEqual(ret.get_x_span(), x_stop - x_start)
self.assertEqual(ret.get_y_start(), y_start)
self.assertEqual(ret.get_y_stop(), y_stop)
self.assertEqual(ret.get_y_span(), y_stop - y_start)
self.assertEqual(ret.get_z_start(), z_start)
self.assertEqual(ret.get_z_stop(), z_stop)
self.assertEqual(ret.get_z_span(), z_stop - z_start)
def get(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Define access mode.
access_mode = utils.get_access_mode(request)
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError(
"Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) * self.bit_depth /
8) # Calculating the number of bytes
BossThrottle().check('cutout_egress', request.user, cost)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
client.put_metric_data(Namespace="BOSS/Cutout",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'EgressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(),
iso=iso,
access_mode=access_mode)
to_renderer = {"time_request": req.time_request, "data": data}
# Send data to renderer
return Response(to_renderer)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
if "no-cache" in request.query_params:
if request.query_params["no-cache"].lower() == "true":
no_cache = True
else:
no_cache = False
else:
no_cache = False
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(), iso=iso, no_cache=no_cache)
to_renderer = {"time_request": req.time_request,
"data": data}
# Send data to renderer
return Response(to_renderer)
def get(self,
request,
collection,
experiment,
channel,
orientation,
resolution,
x_args,
y_args,
z_args,
t_args=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param orientation: Image plane requested. Vaid options include xy,xz or yz
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Process request and validate
try:
request_args = {
"service": "image",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"orientation": orientation,
"resolution": resolution,
"x_args": x_args,
"y_args": y_args,
"z_args": z_args,
"time_args": t_args
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
#Define access mode
access_mode = utils.get_access_mode(request)
# Convert to Resource
resource = spdb.project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Datatype does not match channel",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) * self.bit_depth /
8) # Calculating the number of bytes
BossThrottle().check('image', ThrottleMetric.METRIC_TYPE_EGRESS,
request.user, cost,
ThrottleMetric.METRIC_UNITS_BYTES)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username or "public"
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
try:
client.put_metric_data(Namespace="BOSS/Image",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'EgressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
except Exception as e:
log = bossLogger()
log.exception('Error during put_metric_data: {}'.format(e))
log.exception('Allowing bossDB to continue after logging')
# Get interface to SPDB cache
cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Do a cutout as specified
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
access_mode=access_mode)
# Covert the cutout back to an image and return it
if orientation == 'xy':
img = data.xy_image()
elif orientation == 'yz':
img = data.yz_image()
elif orientation == 'xz':
img = data.xz_image()
else:
return BossHTTPError("Invalid orientation: {}".format(orientation),
ErrorCodes.INVALID_CUTOUT_ARGS)
return Response(img)
def get(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
Return a list of ids in the spatial region.
Args:
request: DRF Request object
collection: Collection name specifying the collection you want
experiment: Experiment name specifying the experiment
channel: Channel_name
num_ids: Number of id you want to reserve
Returns:
JSON dict with start_id and count of ids reserved
Raises:
BossHTTPError for an invalid request
"""
# validate resource
# permissions?
# Check if this is annotation channel
# Process request and validate
try:
request_args = {
"service": "ids",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# create a resource
resource = project.BossResourceDjango(req)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
try:
# Reserve ids
spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
ids = spdb.get_ids_in_region(resource, int(resolution), corner,
extent)
return Response(ids, status=200)
except (TypeError, ValueError) as e:
return BossHTTPError("Type error in the ids view. {}".format(e),
ErrorCodes.TYPE_ERROR)
def get(self,
request,
collection,
experiment,
channel,
orientation,
resolution,
x_args,
y_args,
z_args,
t_args=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param orientation: Image plane requested. Vaid options include xy,xz or yz
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Process request and validate
try:
request_args = {
"service": "image",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"orientation": orientation,
"resolution": resolution,
"x_args": x_args,
"y_args": y_args,
"z_args": z_args,
"time_args": t_args
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
if "no-cache" in request.query_params:
if request.query_params["no-cache"].lower() == "true":
no_cache = True
else:
no_cache = False
else:
no_cache = False
# Convert to Resource
resource = spdb.project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Datatype does not match channel",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Do a cutout as specified
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
no_cache=no_cache)
# Covert the cutout back to an image and return it
if orientation == 'xy':
img = data.xy_image()
elif orientation == 'yz':
img = data.yz_image()
elif orientation == 'xz':
img = data.xz_image()
else:
return BossHTTPError("Invalid orientation: {}".format(orientation),
ErrorCodes.INVALID_CUTOUT_ARGS)
return Response(img)
def get(self,
request,
collection,
experiment,
channel,
orientation,
tile_size,
resolution,
x_idx,
y_idx,
z_idx,
t_idx=None):
"""
View to handle GET requests for a tile when providing indices. Currently only supports XY plane
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_idx: the tile index in the X dimension
:param y_idx: the tile index in the Y dimension
:param z_idx: the tile index in the Z dimension
:param t_idx: the tile index in the T dimension
:return:
"""
# TODO: DMK Merge Tile and Image view once updated request validation is sorted out
# Process request and validate
try:
request_args = {
"service": "tile",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"orientation": orientation,
"tile_size": tile_size,
"resolution": resolution,
"x_args": x_idx,
"y_args": y_idx,
"z_args": z_idx,
"time_args": t_idx
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
#Define access_mode
access_mode = utils.get_access_mode(request)
# Convert to Resource
resource = spdb.project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Datatype does not match channel",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Add metrics to CloudWatch
cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() *
(req.get_time().stop - req.get_time().start) * self.bit_depth /
8) # Calculating the number of bytes
BossThrottle().check('tile_egress', request.user, cost)
boss_config = bossutils.configuration.BossConfig()
dimensions = [
{
'Name': 'User',
'Value': request.user.username
},
{
'Name': 'Resource',
'Value': '{}/{}/{}'.format(collection, experiment, channel)
},
{
'Name': 'Stack',
'Value': boss_config['system']['fqdn']
},
]
session = bossutils.aws.get_session()
client = session.client('cloudwatch')
client.put_metric_data(Namespace="BOSS/Tile",
MetricData=[{
'MetricName': 'InvokeCount',
'Dimensions': dimensions,
'Value': 1.0,
'Unit': 'Count'
}, {
'MetricName': 'EgressCost',
'Dimensions': dimensions,
'Value': cost,
'Unit': 'Bytes'
}])
# Get interface to SPDB cache
cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Do a cutout as specified
data = cache.cutout(
resource,
corner,
extent,
req.get_resolution(),
[req.get_time().start, req.get_time().stop],
access_mode=access_mode)
# Covert the cutout back to an image and return it
if orientation == 'xy':
img = data.xy_image()
elif orientation == 'yz':
img = data.yz_image()
elif orientation == 'xz':
img = data.xz_image()
else:
return BossHTTPError("Invalid orientation: {}".format(orientation),
ErrorCodes.INVALID_CUTOUT_ARGS)
return Response(img)
def post(self, request, collection, experiment, dataset, resolution,
x_range, y_range, z_range):
"""
View to handle POST requests for a cuboid of data while providing all datamodel params
Due to parser implementation, request.data should be a numpy array already.
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param dataset: Dataset identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
req = BossRequest(request)
except BossError as err:
return BossHTTPError(err.args[0], err.args[1], err.args[2])
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Make sure datatype is valid
if expected_data_type != request.data.dtype:
return BossHTTPError("Datatype does not match channel/layer",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure the dimensions of the data match the dimensions of the post URL
if len(request.data.shape) == 4:
expected_shape = (len(req.get_time()), req.get_z_span(),
req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(),
req.get_x_span())
if expected_shape != request.data.shape:
return BossHTTPError(
"Data dimensions in URL do not match POSTed data.",
ErrorCodes.DATA_DIMENSION_MISMATCH)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(request.data.shape) == 4:
cache.write_cuboid(resource, corner, req.get_resolution(),
request.data,
req.get_time()[0])
else:
cache.write_cuboid(resource, corner, req.get_resolution(),
np.expand_dims(request.data, axis=0),
req.get_time()[0])
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
return BossHTTPError('Error during write_cuboid: {}'.format(e),
ErrorCodes.BOSS_SYSTEM_ERROR)
# Send data to renderer
return HttpResponse(status=201)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None):
"""
View to handle GET requests for a cuboid of data while providing all params
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which channel you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check if parsing completed without error. If an error did occur, return to user.
if "filter" in request.query_params:
ids = request.query_params["filter"]
else:
ids = None
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
# Define access mode.
access_mode = utils.get_access_mode(request)
if isinstance(request.data, BossParserError):
return request.data.to_http()
# Process request and validate
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
"ids": ids
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR)
# Make sure cutout request is under 500MB UNCOMPRESSED
if is_too_large(req, self.bit_depth):
return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Get a Cube instance with all time samples
data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop],
filter_ids=req.get_filter_ids(), iso=iso, access_mode=access_mode)
to_renderer = {"time_request": req.time_request,
"data": data}
# Send data to renderer
return Response(to_renderer)
def get(self,
request,
collection,
experiment,
dataset,
orientation,
tile_size,
resolution,
x_idx,
y_idx,
z_idx,
t_idx=None):
"""
View to handle GET requests for a tile when providing indices. Currently only supports XY plane
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param dataset: Dataset identifier, indicating which channel or layer you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_idx: the tile index in the X dimension
:param y_idx: the tile index in the Y dimension
:param z_idx: the tile index in the Z dimension
:param t_idx: the tile index in the T dimension
:return:
"""
# TODO: DMK Merge Tile and Image view once updated request validation is sorted out
# Process request and validate
try:
req = BossRequest(request)
except BossError as err:
return BossError.to_http()
# Convert to Resource
resource = spdb.project.BossResourceDjango(req)
# Get bit depth
try:
self.bit_depth = resource.get_bit_depth()
except ValueError:
return BossHTTPError("Datatype does not match channel/layer",
ErrorCodes.DATATYPE_DOES_NOT_MATCH)
# Make sure cutout request is under 1GB UNCOMPRESSED
total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span(
) * len(req.get_time()) * (self.bit_depth / 8)
if total_bytes > settings.CUTOUT_MAX_SIZE:
return BossHTTPError(
"Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get interface to SPDB cache
cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS,
settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Get the params to pull data out of the cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
extent = (req.get_x_span(), req.get_y_span(), req.get_z_span())
# Do a cutout as specified
data = cache.cutout(
resource, corner, extent, req.get_resolution(),
[req.get_time().start, req.get_time().stop])
# Covert the cutout back to an image and return it
if orientation == 'xy':
img = data.xy_image()
elif orientation == 'yz':
img = data.yz_image()
elif orientation == 'xz':
img = data.xz_image()
else:
return BossHTTPError("Invalid orientation: {}".format(orientation),
ErrorCodes.INVALID_CUTOUT_ARGS)
return Response(img)
def put(self,
request,
collection,
experiment,
channel,
resolution,
x_range,
y_range,
z_range,
t_range=None):
"""
View to handle PUT requests for a overwriting a cuboid to 0s
:param request: DRF Request object
:type request: rest_framework.request.Request
:param collection: Unique Collection identifier, indicating which collection you want to access
:param experiment: Experiment identifier, indicating which experiment you want to access
:param channel: Channel identifier, indicating which dataset or annotation project you want to access
:param resolution: Integer indicating the level in the resolution hierarchy (0 = native)
:param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200)
:param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200)
:param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200)
:return:
"""
# Check for optional iso flag
if "iso" in request.query_params:
if request.query_params["iso"].lower() == "true":
iso = True
else:
iso = False
else:
iso = False
try:
request_args = {
"service": "cutout",
"collection_name": collection,
"experiment_name": experiment,
"channel_name": channel,
"resolution": resolution,
"x_args": x_range,
"y_args": y_range,
"z_args": z_range,
"time_args": t_range,
}
req = BossRequest(request, request_args)
except BossError as err:
return err.to_http()
# Convert to Resource
resource = project.BossResourceDjango(req)
# Get data type
try:
self.bit_depth = resource.get_bit_depth()
expected_data_type = resource.get_numpy_data_type()
except ValueError:
return BossHTTPError(
"Unsupported data type: {}".format(resource.get_data_type()),
ErrorCodes.TYPE_ERROR)
# Set a limit of CUTOUT_MAX_SIZE
if is_too_large(req, self.bit_depth):
return BossHTTPError(
"Cutout overwrite is over 500MB when uncompressed. Reduce overwrite dimensions.",
ErrorCodes.REQUEST_TOO_LARGE)
# Get the shape of the requested data clear
if len(req.get_time()) > 2:
expected_shape = (len(req.get_time()), req.get_z_span(),
req.get_y_span(), req.get_x_span())
else:
expected_shape = (req.get_z_span(), req.get_y_span(),
req.get_x_span())
# Create a binary numpy array for overwrite with specified shape and dtype
black_cuboid = np.ones(expected_shape, dtype=expected_data_type)
# Get interface to SPDB cache
cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG,
settings.OBJECTIO_CONFIG)
# Write block to cache
corner = (req.get_x_start(), req.get_y_start(), req.get_z_start())
try:
if len(black_cuboid.shape) == 4:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
black_cuboid,
req.get_time()[0],
iso=iso,
to_black=True)
else:
cache.write_cuboid(resource,
corner,
req.get_resolution(),
np.expand_dims(black_cuboid, axis=0),
req.get_time()[0],
iso=iso,
to_black=True)
except Exception as e:
# TODO: Eventually remove as this level of detail should not be sent to the user
log = BossLogger().logger
log.exception('Error during write_cuboid: {}'.format(e))
return BossHTTPError('Error during write_cuboid: {}'.format(e),
ErrorCodes.BAD_REQUEST)
# If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data
channel = resource.get_channel()
if channel.downsample_status.upper() == "DOWNSAMPLED":
# Get Channel object and update status
lookup_key = resource.get_lookup_key()
_, exp_id, _ = lookup_key.split("&")
channel_obj = Channel.objects.get(name=channel.name,
experiment=int(exp_id))
channel_obj.downsample_status = "NOT_DOWNSAMPLED"
channel_obj.downsample_arn = ""
channel_obj.save()
# Send data to renderer
return HttpResponse(status=200)
