def get(self, request, collection, experiment, dataset, resolution, x_range, y_range, z_range): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param dataset: Dataset identifier, indicating which channel or layer you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: req = BossRequest(request) except BossError as err: return BossHTTPError(err.args[0], err.args[1], err.args[2]) # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop]) # Send data to renderer return Response(data)
def test_request_tile_init_tileargs_channel(self): """ Test initialization of tile arguments for a tile request for a channel :return: """ url = '/' + version + '/tile/col1/exp1/channel1/xy/512/2/0/0/1' res = 2 (x_start, x_stop) = (0, 512) (y_start, y_stop) = (0, 512) (z_start, z_stop) = (1, 2) # Create the request req = HttpRequest() req.META = {'PATH_INFO': url} drfrequest = Request(req) drfrequest.version = version ret = BossRequest(drfrequest) self.assertEqual(ret.get_resolution(), res) self.assertEqual(ret.get_x_start(), x_start) self.assertEqual(ret.get_x_stop(), x_stop) self.assertEqual(ret.get_x_span(), x_stop - x_start) self.assertEqual(ret.get_y_start(), y_start) self.assertEqual(ret.get_y_stop(), y_stop) self.assertEqual(ret.get_y_span(), y_stop - y_start) self.assertEqual(ret.get_z_start(), z_start) self.assertEqual(ret.get_z_stop(), z_stop) self.assertEqual(ret.get_z_span(), z_stop - z_start)
def get(self, request, collection, experiment,channel, resolution, x_range, y_range, z_range, t_range=None): """ Return a list of ids in the spatial region. Args: request: DRF Request object collection: Collection name specifying the collection you want experiment: Experiment name specifying the experiment channel: Channel_name num_ids: Number of id you want to reserve Returns: JSON dict with start_id and count of ids reserved Raises: BossHTTPError for an invalid request """ # validate resource # permissions? # Check if this is annotation channel # Process request and validate try: request_args = { "service": "ids", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # create a resource resource = project.BossResourceDjango(req) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) try: # Reserve ids spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) ids = spdb.get_ids_in_region(resource, int(resolution), corner, extent) return Response(ids, status=200) except (TypeError, ValueError) as e: return BossHTTPError("Type error in the ids view. {}".format(e), ErrorCodes.TYPE_ERROR)
def test_request_tile_init_tileargs_channel(self): """ Test initialization of tile arguments for a tile request for a channel :return: """ url = '/' + version + '/tile/col1/exp1/channel1/xy/512/2/0/0/1' res = 2 (x_start, x_stop) = (0, 512) (y_start, y_stop) = (0, 512) (z_start, z_stop) = (1, 2) # Create the request dict request_args = { "service": "tile", "collection_name": "col1", "experiment_name": "exp1", "channel_name": "channel1", "orientation": "xy", "tile_size": 512, "resolution": 2, "x_args": "0", "y_args": "0", "z_args": "1", "time_args": None } # Create the request request = self.rf.get(url) force_authenticate(request, user=self.user) drfrequest = Tile().initialize_request(request) drfrequest.version = version ret = BossRequest(drfrequest, request_args) self.assertEqual(ret.get_resolution(), res) self.assertEqual(ret.get_x_start(), x_start) self.assertEqual(ret.get_x_stop(), x_stop) self.assertEqual(ret.get_x_span(), x_stop - x_start) self.assertEqual(ret.get_y_start(), y_start) self.assertEqual(ret.get_y_stop(), y_stop) self.assertEqual(ret.get_y_span(), y_stop - y_start) self.assertEqual(ret.get_z_start(), z_start) self.assertEqual(ret.get_z_stop(), z_stop) self.assertEqual(ret.get_z_span(), z_stop - z_start)
def parse(self, stream, media_type=None, parser_context=None): """Method to decompress bytes from a POST that contains blosc compressed numpy ndarray Only should be used if data sent was compressed using blosc.pack_array() :param stream: Request stream stream type: django.core.handlers.wsgi.WSGIRequest :param media_type: :param parser_context: :return: """ try: req = BossRequest(parser_context['request']) except BossError as err: return BossParserError(err.args[0], err.args[1], err.args[2]) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: bit_depth = resource.get_bit_depth() except ValueError: return BossParserError( "Unsupported data type provided to parser: {}".format( resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * bit_depth / 8 if total_bytes > settings.CUTOUT_MAX_SIZE: return BossParserError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Decompress and return try: return blosc.unpack_array(stream.read()) except EOFError: return BossParserError( "Failed to unpack data. Verify the datatype of your POSTed data and " "xyz dimensions used in the POST URL.", ErrorCodes.DATA_DIMENSION_MISMATCH)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ Return a list of ids in the spatial region. Args: request: DRF Request object collection: Collection name specifying the collection you want experiment: Experiment name specifying the experiment channel: Channel_name num_ids: Number of id you want to reserve Returns: JSON dict with start_id and count of ids reserved Raises: BossHTTPError for an invalid request """ # validate resource # permissions? # Check if this is annotation channel # Process request and validate try: request_args = { "service": "ids", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # create a resource resource = project.BossResourceDjango(req) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) try: # Reserve ids spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) ids = spdb.get_ids_in_region(resource, int(resolution), corner, extent) return Response(ids, status=200) except (TypeError, ValueError) as e: return BossHTTPError("Type error in the ids view. {}".format(e), ErrorCodes.TYPE_ERROR)
def parse(self, stream, media_type=None, parser_context=None): """Method to decompress bytes from a POST that contains a gzipped npy saved numpy ndarray :param stream: Request stream stream type: django.core.handlers.wsgi.WSGIRequest :param media_type: :param parser_context: :return: """ try: request_args = { "service": "cutout", "collection_name": parser_context['kwargs']['collection'], "experiment_name": parser_context['kwargs']['experiment'], "channel_name": parser_context['kwargs']['channel'], "resolution": parser_context['kwargs']['resolution'], "x_args": parser_context['kwargs']['x_range'], "y_args": parser_context['kwargs']['y_range'], "z_args": parser_context['kwargs']['z_range'], } if 't_range' in parser_context['kwargs']: request_args["time_args"] = parser_context['kwargs']['t_range'] else: request_args["time_args"] = None req = BossRequest(parser_context['request'], request_args) except BossError as err: return BossParserError(err.message, err.status_code) except Exception as err: return BossParserError(str(err), ErrorCodes.UNHANDLED_EXCEPTION) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: bit_depth = resource.get_bit_depth() except ValueError: return BossParserError( "Unsupported data type provided to parser: {}".format( resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * bit_depth / 8 if total_bytes > settings.CUTOUT_MAX_SIZE: return BossParserError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Decompress and return try: data_bytes = zlib.decompress(stream.read()) # Open data_obj = io.BytesIO(data_bytes) parsed_data = np.load(data_obj) except EOFError: return BossParserError( "Failed to unpack data. Verify the datatype of your POSTed data and " "xyz dimensions used in the POST URL.", ErrorCodes.DATA_DIMENSION_MISMATCH) return req, resource, parsed_data
def parse(self, stream, media_type=None, parser_context=None): """Method to decompress bytes from a POST that contains blosc compressed matrix data **Bytes object decompressed should be C-ordered** :param stream: Request stream stream type: django.core.handlers.wsgi.WSGIRequest :param media_type: :param parser_context: :return: """ # Process request and validate try: request_args = { "service": "cutout", "collection_name": parser_context['kwargs']['collection'], "experiment_name": parser_context['kwargs']['experiment'], "channel_name": parser_context['kwargs']['channel'], "resolution": parser_context['kwargs']['resolution'], "x_args": parser_context['kwargs']['x_range'], "y_args": parser_context['kwargs']['y_range'], "z_args": parser_context['kwargs']['z_range'], } if 't_range' in parser_context['kwargs']: request_args["time_args"] = parser_context['kwargs']['t_range'] else: request_args["time_args"] = None req = BossRequest(parser_context['request'], request_args) except BossError as err: return BossParserError(err.message, err.error_code) except Exception as err: return BossParserError(str(err), ErrorCodes.UNHANDLED_EXCEPTION) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: bit_depth = resource.get_bit_depth() except ValueError: return BossParserError( "Unsupported data type provided to parser: {}".format( resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * bit_depth / 8 if total_bytes > settings.CUTOUT_MAX_SIZE: return BossParserError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) try: # Decompress raw_data = blosc.decompress(stream.read()) data_mat = np.fromstring(raw_data, dtype=resource.get_numpy_data_type()) except: return BossParserError( "Failed to decompress data. Verify the datatype/bitdepth of your data " "matches the channel.", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Reshape and return try: if req.time_request: # Time series request (even if single time point) - Get 4D matrix parsed_data = np.reshape( data_mat, (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()), order='C') else: # Not a time series request (time range [0,1] auto-populated) - Get 3D matrix parsed_data = np.reshape( data_mat, (req.get_z_span(), req.get_y_span(), req.get_x_span()), order='C') except ValueError: return BossParserError( "Failed to unpack data. Verify the datatype of your POSTed data and " "xyz dimensions used in the POST URL.", ErrorCodes.DATA_DIMENSION_MISMATCH) return req, resource, parsed_data
def get(self, request, collection, experiment, dataset, orientation, tile_size, resolution, x_idx, y_idx, z_idx, t_idx=None): """ View to handle GET requests for a tile when providing indices. Currently only supports XY plane :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param dataset: Dataset identifier, indicating which channel or layer you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_idx: the tile index in the X dimension :param y_idx: the tile index in the Y dimension :param z_idx: the tile index in the Z dimension :param t_idx: the tile index in the T dimension :return: """ # TODO: DMK Merge Tile and Image view once updated request validation is sorted out # Process request and validate try: req = BossRequest(request) except BossError as err: return BossError.to_http() # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Datatype does not match channel/layer", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Do a cutout as specified data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop]) # Covert the cutout back to an image and return it if orientation == 'xy': img = data.xy_image() elif orientation == 'yz': img = data.yz_image() elif orientation == 'xz': img = data.xz_image() else: return BossHTTPError("Invalid orientation: {}".format(orientation), ErrorCodes.INVALID_CUTOUT_ARGS) return Response(img)
def get(self, request, collection, experiment, channel, orientation, resolution, x_args, y_args, z_args, t_args=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param orientation: Image plane requested. Vaid options include xy,xz or yz :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Process request and validate try: request_args = { "service": "image", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "orientation": orientation, "resolution": resolution, "x_args": x_args, "y_args": y_args, "z_args": z_args, "time_args": t_args } req = BossRequest(request, request_args) except BossError as err: return err.to_http() #Define access mode access_mode = utils.get_access_mode(request) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * self.bit_depth / 8) # Calculating the number of bytes BossThrottle().check('image', ThrottleMetric.METRIC_TYPE_EGRESS, request.user, cost, ThrottleMetric.METRIC_UNITS_BYTES) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username or "public" }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') try: client.put_metric_data(Namespace="BOSS/Image", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'EgressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) except Exception as e: log = bossLogger() log.exception('Error during put_metric_data: {}'.format(e)) log.exception('Allowing bossDB to continue after logging') # Get interface to SPDB cache cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Do a cutout as specified data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], access_mode=access_mode) # Covert the cutout back to an image and return it if orientation == 'xy': img = data.xy_image() elif orientation == 'yz': img = data.yz_image() elif orientation == 'xz': img = data.xz_image() else: return BossHTTPError("Invalid orientation: {}".format(orientation), ErrorCodes.INVALID_CUTOUT_ARGS) return Response(img)
def parse(self, stream, media_type=None, parser_context=None): """Method to decompress bytes from a POST that contains blosc compressed matrix data **Bytes object decompressed should be C-ordered** :param stream: Request stream stream type: django.core.handlers.wsgi.WSGIRequest :param media_type: :param parser_context: :return: """ # Process request and validate try: req = BossRequest(parser_context['request']) except BossError as err: return BossParserError(err.args[0], err.args[1], err.args[2]) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: bit_depth = resource.get_bit_depth() except ValueError: return BossParserError( "Unsupported data type provided to parser: {}".format( resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * bit_depth / 8 if total_bytes > settings.CUTOUT_MAX_SIZE: return BossParserError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) try: # Decompress raw_data = blosc.decompress(stream.read()) data_mat = np.fromstring(raw_data, dtype=resource.get_numpy_data_type()) except: return BossParserError( "Failed to decompress data. Verify the datatype/bitdepth of your data " "matches the channel/layer.", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Reshape and return try: if len(req.get_time()) > 1: # Time series data return np.reshape(data_mat, (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()), order='C') else: return np.reshape( data_mat, (req.get_z_span(), req.get_y_span(), req.get_x_span()), order='C') except ValueError: return BossParserError( "Failed to unpack data. Verify the datatype of your POSTed data and " "xyz dimensions used in the POST URL.", ErrorCodes.DATA_DIMENSION_MISMATCH)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if "filter" in request.query_params: ids = request.query_params["filter"] else: ids = None if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Define access mode. access_mode = utils.get_access_mode(request) if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, "ids": ids } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, self.bit_depth): return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], filter_ids=req.get_filter_ids(), iso=iso, access_mode=access_mode) to_renderer = {"time_request": req.time_request, "data": data} # Send data to renderer return Response(to_renderer)
def parse(self, stream, media_type=None, parser_context=None): """Method to decompress bytes from a POST that contains blosc compressed matrix data **Bytes object decompressed should be C-ordered** :param stream: Request stream stream type: django.core.handlers.wsgi.WSGIRequest :param media_type: :param parser_context: :return: """ # Process request and validate try: request_args = { "service": "cutout", "collection_name": parser_context['kwargs']['collection'], "experiment_name": parser_context['kwargs']['experiment'], "channel_name": parser_context['kwargs']['channel'], "resolution": parser_context['kwargs']['resolution'], "x_args": parser_context['kwargs']['x_range'], "y_args": parser_context['kwargs']['y_range'], "z_args": parser_context['kwargs']['z_range'], } if 't_range' in parser_context['kwargs']: request_args["time_args"] = parser_context['kwargs']['t_range'] else: request_args["time_args"] = None req = BossRequest(parser_context['request'], request_args) except BossError as err: self.consume_request(stream) return BossParserError(err.message, err.error_code) except Exception as err: self.consume_request(stream) return BossParserError(str(err), ErrorCodes.UNHANDLED_EXCEPTION) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: bit_depth = resource.get_bit_depth() except ValueError: return BossParserError("Unsupported data type provided to parser: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, bit_depth): return BossParserError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) try: # Decompress raw_data = blosc.decompress(stream.read()) data_mat = np.fromstring(raw_data, dtype=resource.get_numpy_data_type()) except MemoryError: return BossParserError("Ran out of memory decompressing data.", ErrorCodes.BOSS_SYSTEM_ERROR) except: return BossParserError("Failed to decompress data. Verify the datatype/bitdepth of your data " "matches the channel.", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Reshape and return try: if req.time_request: # Time series request (even if single time point) - Get 4D matrix parsed_data = np.reshape(data_mat, (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()), order='C') else: # Not a time series request (time range [0,1] auto-populated) - Get 3D matrix parsed_data = np.reshape(data_mat, (req.get_z_span(), req.get_y_span(), req.get_x_span()), order='C') except ValueError: return BossParserError("Failed to unpack data. Verify the datatype of your POSTed data and " "xyz dimensions used in the POST URL.", ErrorCodes.DATA_DIMENSION_MISMATCH) return req, resource, parsed_data
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if "filter" in request.query_params: ids = request.query_params["filter"] else: ids = None if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False if "no-cache" in request.query_params: if request.query_params["no-cache"].lower() == "true": no_cache = True else: no_cache = False else: no_cache = False if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, "ids": ids } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, self.bit_depth): return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], filter_ids=req.get_filter_ids(), iso=iso, no_cache=no_cache) to_renderer = {"time_request": req.time_request, "data": data} # Send data to renderer return Response(to_renderer)
def get(self, request, collection, experiment, channel, orientation, resolution, x_args, y_args, z_args, t_args=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param orientation: Image plane requested. Vaid options include xy,xz or yz :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_args: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_args: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_args: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Process request and validate try: request_args = { "service": "image", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "orientation": orientation, "resolution": resolution, "x_args": x_args, "y_args": y_args, "z_args": z_args, "time_args": t_args } req = BossRequest(request, request_args) except BossError as err: return err.to_http() if "no-cache" in request.query_params: if request.query_params["no-cache"].lower() == "true": no_cache = True else: no_cache = False else: no_cache = False # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Do a cutout as specified data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], no_cache=no_cache) # Covert the cutout back to an image and return it if orientation == 'xy': img = data.xy_image() elif orientation == 'yz': img = data.yz_image() elif orientation == 'xz': img = data.xz_image() else: return BossHTTPError("Invalid orientation: {}".format(orientation), ErrorCodes.INVALID_CUTOUT_ARGS) return Response(img)
def get(self, request, collection, experiment, channel, orientation, tile_size, resolution, x_idx, y_idx, z_idx, t_idx=None): """ View to handle GET requests for a tile when providing indices. Currently only supports XY plane :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_idx: the tile index in the X dimension :param y_idx: the tile index in the Y dimension :param z_idx: the tile index in the Z dimension :param t_idx: the tile index in the T dimension :return: """ # TODO: DMK Merge Tile and Image view once updated request validation is sorted out # Process request and validate try: request_args = { "service": "tile", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "orientation": orientation, "tile_size": tile_size, "resolution": resolution, "x_args": x_idx, "y_args": y_idx, "z_args": z_idx, "time_args": t_idx } req = BossRequest(request, request_args) except BossError as err: return err.to_http() #Define access_mode access_mode = utils.get_access_mode(request) # Convert to Resource resource = spdb.project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Datatype does not match channel", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * self.bit_depth / 8) # Calculating the number of bytes BossThrottle().check('tile_egress', request.user, cost) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') client.put_metric_data(Namespace="BOSS/Tile", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'EgressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) # Get interface to SPDB cache cache = spdb.spatialdb.SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Do a cutout as specified data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], access_mode=access_mode) # Covert the cutout back to an image and return it if orientation == 'xy': img = data.xy_image() elif orientation == 'yz': img = data.yz_image() elif orientation == 'xz': img = data.xz_image() else: return BossHTTPError("Invalid orientation: {}".format(orientation), ErrorCodes.INVALID_CUTOUT_ARGS) return Response(img)
def post(self, request, collection, experiment, dataset, resolution, x_range, y_range, z_range): """ View to handle POST requests for a cuboid of data while providing all datamodel params Due to parser implementation, request.data should be a numpy array already. :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param dataset: Dataset identifier, indicating which dataset or annotation project you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: req = BossRequest(request) except BossError as err: return BossHTTPError(err.args[0], err.args[1], err.args[2]) # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: expected_data_type = resource.get_numpy_data_type() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure datatype is valid if expected_data_type != request.data.dtype: return BossHTTPError("Datatype does not match channel/layer", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure the dimensions of the data match the dimensions of the post URL if len(request.data.shape) == 4: expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()) else: expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span()) if expected_shape != request.data.shape: return BossHTTPError( "Data dimensions in URL do not match POSTed data.", ErrorCodes.DATA_DIMENSION_MISMATCH) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Write block to cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) try: if len(request.data.shape) == 4: cache.write_cuboid(resource, corner, req.get_resolution(), request.data, req.get_time()[0]) else: cache.write_cuboid(resource, corner, req.get_resolution(), np.expand_dims(request.data, axis=0), req.get_time()[0]) except Exception as e: # TODO: Eventually remove as this level of detail should not be sent to the user return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BOSS_SYSTEM_ERROR) # Send data to renderer return HttpResponse(status=201)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if "filter" in request.query_params: ids = request.query_params["filter"] else: ids = None if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Define access mode. access_mode = utils.get_access_mode(request) if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, "ids": ids } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, self.bit_depth): return BossHTTPError( "Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * self.bit_depth / 8) # Calculating the number of bytes BossThrottle().check('cutout_egress', request.user, cost) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') client.put_metric_data(Namespace="BOSS/Cutout", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'EgressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], filter_ids=req.get_filter_ids(), iso=iso, access_mode=access_mode) to_renderer = {"time_request": req.time_request, "data": data} # Send data to renderer return Response(to_renderer)
def put(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle PUT requests for a overwriting a cuboid to 0s :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which dataset or annotation project you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check for optional iso flag if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get data type try: self.bit_depth = resource.get_bit_depth() expected_data_type = resource.get_numpy_data_type() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Set a limit of CUTOUT_MAX_SIZE if is_too_large(req, self.bit_depth): return BossHTTPError( "Cutout overwrite is over 500MB when uncompressed. Reduce overwrite dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get the shape of the requested data clear if len(req.get_time()) > 2: expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()) else: expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span()) # Create a binary numpy array for overwrite with specified shape and dtype black_cuboid = np.ones(expected_shape, dtype=expected_data_type) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Write block to cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) try: if len(black_cuboid.shape) == 4: cache.write_cuboid(resource, corner, req.get_resolution(), black_cuboid, req.get_time()[0], iso=iso, to_black=True) else: cache.write_cuboid(resource, corner, req.get_resolution(), np.expand_dims(black_cuboid, axis=0), req.get_time()[0], iso=iso, to_black=True) except Exception as e: # TODO: Eventually remove as this level of detail should not be sent to the user log = BossLogger().logger log.exception('Error during write_cuboid: {}'.format(e)) return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BAD_REQUEST) # If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data channel = resource.get_channel() if channel.downsample_status.upper() == "DOWNSAMPLED": # Get Channel object and update status lookup_key = resource.get_lookup_key() _, exp_id, _ = lookup_key.split("&") channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "NOT_DOWNSAMPLED" channel_obj.downsample_arn = "" channel_obj.save() # Send data to renderer return HttpResponse(status=200)