def get(self, request, collection, experiment, dataset, resolution, x_range, y_range, z_range): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param dataset: Dataset identifier, indicating which channel or layer you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: req = BossRequest(request) except BossError as err: return BossHTTPError(err.args[0], err.args[1], err.args[2]) # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 1GB UNCOMPRESSED total_bytes = req.get_x_span() * req.get_y_span() * req.get_z_span( ) * len(req.get_time()) * (self.bit_depth / 8) if total_bytes > settings.CUTOUT_MAX_SIZE: return BossHTTPError( "Cutout request is over 1GB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop]) # Send data to renderer return Response(data)
def post(self, request, collection, experiment, channel): """View to kick off a channel's downsample process Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) return start(request, resource)
def delete(self, request, collection, experiment, channel): """View to cancel an in-progress downsample operation Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) channel = resource.get_channel() if channel.downsample_status.upper() != "IN_PROGRESS": return BossHTTPError( "You can only cancel and in-progress downsample operation.", ErrorCodes.INVALID_STATE) # Get Channel object lookup_key = resource.get_lookup_key() _, exp_id, _ = lookup_key.split("&") channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) # Call cancel on the Step Function session = bossutils.aws.get_session() bossutils.aws.sfn_cancel( session, channel_obj.downsample_arn, error="User Cancel", cause="User has requested the downsample operation to stop.") # Clear ARN channel_obj.downsample_arn = "" # Change Status channel_obj.downsample_status = "NOT_DOWNSAMPLED" channel_obj.save() return HttpResponse(status=204)
def get(self, request, collection, experiment, channel, num_ids): """ Reserve a unique, sequential list of annotation ids for the provided channel to use as object ids for annotations. Args: request: DRF Request object collection: Collection name specifying the collection you want experiment: Experiment name specifying the experiment channel: Channel_name num_ids: Number of id you want to reserve Returns: JSON dict with start_id and count of ids reserved Raises: BossHTTPError for an invalid request """ # validate resource # permissions? # validate that num_ids is an int # Check if this is annotation channel try: request_args = { "service": "reserve", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # create a resource resource = project.BossResourceDjango(req) try: # Reserve ids spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) start_id = spdb.reserve_ids(resource, int(num_ids)) data = {'start_id': start_id[0], 'count': num_ids} return Response(data, status=200) except (TypeError, ValueError) as e: return BossHTTPError( "Type error in the reserve id view. {}".format(e), ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if "filter" in request.query_params: ids = request.query_params["filter"] else: ids = None if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Define access mode. access_mode = utils.get_access_mode(request) if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, "ids": ids } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, self.bit_depth): return BossHTTPError( "Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Add metrics to CloudWatch cost = (req.get_x_span() * req.get_y_span() * req.get_z_span() * (req.get_time().stop - req.get_time().start) * self.bit_depth / 8) # Calculating the number of bytes BossThrottle().check('cutout_egress', request.user, cost) boss_config = bossutils.configuration.BossConfig() dimensions = [ { 'Name': 'User', 'Value': request.user.username }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment, channel) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] session = bossutils.aws.get_session() client = session.client('cloudwatch') client.put_metric_data(Namespace="BOSS/Cutout", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'EgressCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Bytes' }]) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout( resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], filter_ids=req.get_filter_ids(), iso=iso, access_mode=access_mode) to_renderer = {"time_request": req.time_request, "data": data} # Send data to renderer return Response(to_renderer)
def post(self, request, collection, experiment, channel): """View to kick off a channel's downsample process Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) channel = resource.get_channel() if channel.downsample_status.upper() == "IN_PROGRESS": return BossHTTPError( "Channel is currently being downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) elif channel.downsample_status.upper() == "DOWNSAMPLED" and \ not request.user.is_staff: return BossHTTPError( "Channel is already downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) session = bossutils.aws.get_session() # Make sure only one Channel is downsampled at a time channel_objs = Channel.objects.filter(downsample_status='IN_PROGRESS') for channel_obj in channel_objs: # Verify that the channel is still being downsampled status = bossutils.aws.sfn_status(session, channel_obj.downsample_arn) if status == 'RUNNING': return BossHTTPError( "Another Channel is currently being downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) if request.user.is_staff: # DP HACK: allow admin users to override the coordinate frame frame = request.data else: frame = {} # Call Step Function boss_config = bossutils.configuration.BossConfig() experiment = resource.get_experiment() coord_frame = resource.get_coord_frame() lookup_key = resource.get_lookup_key() col_id, exp_id, ch_id = lookup_key.split("&") def get_frame(idx): return int(frame.get(idx, getattr(coord_frame, idx))) args = { 'collection_id': int(col_id), 'experiment_id': int(exp_id), 'channel_id': int(ch_id), 'annotation_channel': not channel.is_image(), 'data_type': resource.get_data_type(), 's3_bucket': boss_config["aws"]["cuboid_bucket"], 's3_index': boss_config["aws"]["s3-index-table"], 'x_start': get_frame('x_start'), 'y_start': get_frame('y_start'), 'z_start': get_frame('z_start'), 'x_stop': get_frame('x_stop'), 'y_stop': get_frame('y_stop'), 'z_stop': get_frame('z_stop'), 'resolution': int(channel.base_resolution), 'resolution_max': int(experiment.num_hierarchy_levels), 'res_lt_max': int(channel.base_resolution) + 1 < int( experiment.num_hierarchy_levels), 'type': experiment.hierarchy_method, 'iso_resolution': int(resource.get_isotropic_level()), # This step function executes: boss-tools/activities/resolution_hierarchy.py 'downsample_volume_lambda': boss_config['lambda']['downsample_volume'], 'aws_region': get_region(), } # Check that only administrators are triggering extra large downsamples if (not request.user.is_staff) and \ ((args['x_stop'] - args['x_start']) * \ (args['y_stop'] - args['y_start']) * \ (args['z_stop'] - args['z_start']) > settings.DOWNSAMPLE_MAX_SIZE): return BossHTTPError( "Large downsamples require admin permissions to trigger. Invalid Request.", ErrorCodes.INVALID_STATE) # Add metrics to CloudWatch def get_cubes(axis, dim): extent = args['{}_stop'.format(axis)] - args['{}_start'.format( axis)] return -(-extent // dim) ## ceil div cost = ( get_cubes('x', 512) * get_cubes('y', 512) * get_cubes('z', 16) / 4 # Number of cubes for a downsampled volume * 0.75 # Assume the frame is only 75% filled * 2 # 1 for invoking a lambda # 1 for time it takes lambda to run * 1.33 # Add 33% overhead for all other non-base resolution downsamples ) dimensions = [ { 'Name': 'User', 'Value': request.user.username }, { 'Name': 'Resource', 'Value': '{}/{}/{}'.format(collection, experiment.name, channel.name) }, { 'Name': 'Stack', 'Value': boss_config['system']['fqdn'] }, ] client = session.client('cloudwatch') client.put_metric_data(Namespace="BOSS/Downsample", MetricData=[{ 'MetricName': 'InvokeCount', 'Dimensions': dimensions, 'Value': 1.0, 'Unit': 'Count' }, { 'MetricName': 'ComputeCost', 'Dimensions': dimensions, 'Value': cost, 'Unit': 'Count' }]) # Start downsample downsample_sfn = boss_config['sfn']['downsample_sfn'] arn = bossutils.aws.sfn_execute(session, downsample_sfn, dict(args)) # Change Status and Save ARN channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "IN_PROGRESS" channel_obj.downsample_arn = arn channel_obj.save() return HttpResponse(status=201)
def get(self, request, collection, experiment, channel): """View to provide a channel's downsample status and properties Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get Status channel = resource.get_channel() experiment = resource.get_experiment() to_renderer = {"status": channel.downsample_status} # Check Step Function if status is in-progress and update if channel.downsample_status == "IN_PROGRESS": lookup_key = resource.get_lookup_key() _, exp_id, _ = lookup_key.split("&") # Get channel object channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) # Update the status from the step function session = bossutils.aws.get_session() status = bossutils.aws.sfn_status(session, channel_obj.downsample_arn) if status == "SUCCEEDED": # Change to DOWNSAMPLED channel_obj.downsample_status = "DOWNSAMPLED" channel_obj.save() to_renderer["status"] = "DOWNSAMPLED" # DP NOTE: This code should be moved to spdb when change # tracking is added to automatically calculate # frame extents for the user # DP NOTE: Clear the cache of any cubes for the channel # This is to prevent serving stale data after # (re)downsampling log = BossLogger().logger for pattern in ("CACHED-CUBOID&" + lookup_key + "&*", "CACHED-CUBOID&ISO&" + lookup_key + "&*"): log.debug("Clearing cache of {} cubes".format(pattern)) try: cache = RedisKVIO(settings.KVIO_SETTINGS) pipe = cache.cache_client.pipeline() for key in cache.cache_client.scan_iter(match=pattern): pipe.delete(key) pipe.execute() except Exception as ex: log.exception( "Problem clearing cache after downsample finished") elif status == "FAILED" or status == "TIMED_OUT": # Change status to FAILED channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "FAILED" channel_obj.save() to_renderer["status"] = "FAILED" # Get hierarchy levels to_renderer["num_hierarchy_levels"] = experiment.num_hierarchy_levels # Gen Voxel dims voxel_size = {} voxel_dims = resource.get_downsampled_voxel_dims(iso=iso) for res, dims in enumerate(voxel_dims): voxel_size["{}".format(res)] = dims to_renderer["voxel_size"] = voxel_size # Gen Extent dims extent = {} extent_dims = resource.get_downsampled_extent_dims(iso=iso) for res, dims in enumerate(extent_dims): extent["{}".format(res)] = dims to_renderer["extent"] = extent # Get Cuboid dims cuboid_size = {} for res in range(0, experiment.num_hierarchy_levels): cuboid_size["{}".format(res)] = CUBOIDSIZE[res] to_renderer["cuboid_size"] = cuboid_size # Send data to renderer return Response(to_renderer)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle GET requests for a cuboid of data while providing all params :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which channel you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if "filter" in request.query_params: ids = request.query_params["filter"] else: ids = None if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False if "no-cache" in request.query_params: if request.query_params["no-cache"].lower() == "true": no_cache = True else: no_cache = False else: no_cache = False if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, "ids": ids } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: self.bit_depth = resource.get_bit_depth() except ValueError: return BossHTTPError("Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure cutout request is under 500MB UNCOMPRESSED if is_too_large(req, self.bit_depth): return BossHTTPError("Cutout request is over 500MB when uncompressed. Reduce cutout dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) # Get a Cube instance with all time samples data = cache.cutout(resource, corner, extent, req.get_resolution(), [req.get_time().start, req.get_time().stop], filter_ids=req.get_filter_ids(), iso=iso, no_cache=no_cache) to_renderer = {"time_request": req.time_request, "data": data} # Send data to renderer return Response(to_renderer)
def post(self, request, collection, experiment, channel): """View to kick off a channel's downsample process Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) channel = resource.get_channel() if channel.downsample_status.upper() == "IN_PROGRESS": return BossHTTPError("Channel is currently being downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) elif channel.downsample_status.upper() == "DOWNSAMPLED" and \ not request.user.is_staff: return BossHTTPError("Channel is already downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) if request.user.is_staff: # DP HACK: allow admin users to override the coordinate frame frame = request.data else: frame = {} # Call Step Function boss_config = bossutils.configuration.BossConfig() experiment = resource.get_experiment() coord_frame = resource.get_coord_frame() lookup_key = resource.get_lookup_key() col_id, exp_id, ch_id = lookup_key.split("&") def get_frame(idx): return int(frame.get(idx, getattr(coord_frame, idx))) args = { 'collection_id': int(col_id), 'experiment_id': int(exp_id), 'channel_id': int(ch_id), 'annotation_channel': not channel.is_image(), 'data_type': resource.get_data_type(), 's3_bucket': boss_config["aws"]["cuboid_bucket"], 's3_index': boss_config["aws"]["s3-index-table"], 'x_start': get_frame('x_start'), 'y_start': get_frame('y_start'), 'z_start': get_frame('z_start'), 'x_stop': get_frame('x_stop'), 'y_stop': get_frame('y_stop'), 'z_stop': get_frame('z_stop'), 'resolution': int(channel.base_resolution), 'resolution_max': int(experiment.num_hierarchy_levels), 'res_lt_max': int(channel.base_resolution) + 1 < int(experiment.num_hierarchy_levels), 'type': experiment.hierarchy_method, 'iso_resolution': int(resource.get_isotropic_level()), # This step function executes: boss-tools/activities/resolution_hierarchy.py 'downsample_volume_lambda': boss_config['lambda']['downsample_volume'], 'aws_region': get_region(), } session = bossutils.aws.get_session() downsample_sfn = boss_config['sfn']['downsample_sfn'] arn = bossutils.aws.sfn_execute(session, downsample_sfn, dict(args)) # Change Status and Save ARN channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "IN_PROGRESS" channel_obj.downsample_arn = arn channel_obj.save() return HttpResponse(status=201)
def get(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ Return a list of ids in the spatial region. Args: request: DRF Request object collection: Collection name specifying the collection you want experiment: Experiment name specifying the experiment channel: Channel_name num_ids: Number of id you want to reserve Returns: JSON dict with start_id and count of ids reserved Raises: BossHTTPError for an invalid request """ # validate resource # permissions? # Check if this is annotation channel # Process request and validate try: request_args = { "service": "ids", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # create a resource resource = project.BossResourceDjango(req) # Get the params to pull data out of the cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) extent = (req.get_x_span(), req.get_y_span(), req.get_z_span()) try: # Reserve ids spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) ids = spdb.get_ids_in_region(resource, int(resolution), corner, extent) return Response(ids, status=200) except (TypeError, ValueError) as e: return BossHTTPError("Type error in the ids view. {}".format(e), ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment, channel, resolution, id): """ Return the bounding box containing the object Args: request: DRF Request object collection: Collection name specifying the collection you want experiment: Experiment name specifying the experiment channel: Channel_name resolution: Data resolution id: The id of the object Returns: JSON dict with the bounding box of the object Raises: BossHTTPError for an invalid request """ # validate resource # permissions? # validate that id is an int # Check if this is annotation channel if 'type' in request.query_params: bb_type = request.query_params['type'] if bb_type != 'loose' and bb_type != 'tight': return BossHTTPError( "Invalid option for bounding box type {}. The valid options are : loose or tight" .format(bb_type), ErrorCodes.INVALID_ARGUMENT) else: bb_type = 'loose' try: request_args = { "service": "boundingbox", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "id": id } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # create a resource resource = project.BossResourceDjango(req) try: # Get interface to SPDB cache spdb = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) data = spdb.get_bounding_box(resource, int(resolution), int(id), bb_type=bb_type) if data is None: return BossHTTPError("The id does not exist. {}".format(id), ErrorCodes.OBJECT_NOT_FOUND) return Response(data, status=200) except (TypeError, ValueError) as e: return BossHTTPError( "Type error in the boundingbox view. {}".format(e), ErrorCodes.TYPE_ERROR)
def get(self, request, collection, experiment, channel): """View to provide a channel's downsample status and properties Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get Status channel = resource.get_channel() experiment = resource.get_experiment() to_renderer = {"status": channel.downsample_status} # Check Step Function if status is in-progress and update if channel.downsample_status == "IN_PROGRESS": lookup_key = resource.get_lookup_key() _, exp_id, _ = lookup_key.split("&") # Get channel object channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) # Update the status from the step function session = bossutils.aws.get_session() status = bossutils.aws.sfn_status(session, channel_obj.downsample_arn) if status == "SUCCEEDED": # Change to DOWNSAMPLED channel_obj.downsample_status = "DOWNSAMPLED" channel_obj.save() to_renderer["status"] = "DOWNSAMPLED" elif status == "FAILED" or status == "TIMED_OUT": # Change status to FAILED channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "FAILED" channel_obj.save() to_renderer["status"] = "FAILED" # Get hierarchy levels to_renderer["num_hierarchy_levels"] = experiment.num_hierarchy_levels # Gen Voxel dims voxel_size = {} voxel_dims = resource.get_downsampled_voxel_dims(iso=iso) for res, dims in enumerate(voxel_dims): voxel_size["{}".format(res)] = dims to_renderer["voxel_size"] = voxel_size # Gen Extent dims extent = {} extent_dims = resource.get_downsampled_extent_dims(iso=iso) for res, dims in enumerate(extent_dims): extent["{}".format(res)] = dims to_renderer["extent"] = extent # Get Cuboid dims cuboid_size = {} for res in range(0, experiment.num_hierarchy_levels): cuboid_size["{}".format(res)] = CUBOIDSIZE[res] to_renderer["cuboid_size"] = cuboid_size # Send data to renderer return Response(to_renderer)
def post(self, request, collection, experiment, dataset, resolution, x_range, y_range, z_range): """ View to handle POST requests for a cuboid of data while providing all datamodel params Due to parser implementation, request.data should be a numpy array already. :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param dataset: Dataset identifier, indicating which dataset or annotation project you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check if parsing completed without error. If an error did occur, return to user. if isinstance(request.data, BossParserError): return request.data.to_http() # Process request and validate try: req = BossRequest(request) except BossError as err: return BossHTTPError(err.args[0], err.args[1], err.args[2]) # Convert to Resource resource = project.BossResourceDjango(req) # Get bit depth try: expected_data_type = resource.get_numpy_data_type() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Make sure datatype is valid if expected_data_type != request.data.dtype: return BossHTTPError("Datatype does not match channel/layer", ErrorCodes.DATATYPE_DOES_NOT_MATCH) # Make sure the dimensions of the data match the dimensions of the post URL if len(request.data.shape) == 4: expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()) else: expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span()) if expected_shape != request.data.shape: return BossHTTPError( "Data dimensions in URL do not match POSTed data.", ErrorCodes.DATA_DIMENSION_MISMATCH) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Write block to cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) try: if len(request.data.shape) == 4: cache.write_cuboid(resource, corner, req.get_resolution(), request.data, req.get_time()[0]) else: cache.write_cuboid(resource, corner, req.get_resolution(), np.expand_dims(request.data, axis=0), req.get_time()[0]) except Exception as e: # TODO: Eventually remove as this level of detail should not be sent to the user return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BOSS_SYSTEM_ERROR) # Send data to renderer return HttpResponse(status=201)
def put(self, request, collection, experiment, channel, resolution, x_range, y_range, z_range, t_range=None): """ View to handle PUT requests for a overwriting a cuboid to 0s :param request: DRF Request object :type request: rest_framework.request.Request :param collection: Unique Collection identifier, indicating which collection you want to access :param experiment: Experiment identifier, indicating which experiment you want to access :param channel: Channel identifier, indicating which dataset or annotation project you want to access :param resolution: Integer indicating the level in the resolution hierarchy (0 = native) :param x_range: Python style range indicating the X coordinates of where to post the cuboid (eg. 100:200) :param y_range: Python style range indicating the Y coordinates of where to post the cuboid (eg. 100:200) :param z_range: Python style range indicating the Z coordinates of where to post the cuboid (eg. 100:200) :return: """ # Check for optional iso flag if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False try: request_args = { "service": "cutout", "collection_name": collection, "experiment_name": experiment, "channel_name": channel, "resolution": resolution, "x_args": x_range, "y_args": y_range, "z_args": z_range, "time_args": t_range, } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get data type try: self.bit_depth = resource.get_bit_depth() expected_data_type = resource.get_numpy_data_type() except ValueError: return BossHTTPError( "Unsupported data type: {}".format(resource.get_data_type()), ErrorCodes.TYPE_ERROR) # Set a limit of CUTOUT_MAX_SIZE if is_too_large(req, self.bit_depth): return BossHTTPError( "Cutout overwrite is over 500MB when uncompressed. Reduce overwrite dimensions.", ErrorCodes.REQUEST_TOO_LARGE) # Get the shape of the requested data clear if len(req.get_time()) > 2: expected_shape = (len(req.get_time()), req.get_z_span(), req.get_y_span(), req.get_x_span()) else: expected_shape = (req.get_z_span(), req.get_y_span(), req.get_x_span()) # Create a binary numpy array for overwrite with specified shape and dtype black_cuboid = np.ones(expected_shape, dtype=expected_data_type) # Get interface to SPDB cache cache = SpatialDB(settings.KVIO_SETTINGS, settings.STATEIO_CONFIG, settings.OBJECTIO_CONFIG) # Write block to cache corner = (req.get_x_start(), req.get_y_start(), req.get_z_start()) try: if len(black_cuboid.shape) == 4: cache.write_cuboid(resource, corner, req.get_resolution(), black_cuboid, req.get_time()[0], iso=iso, to_black=True) else: cache.write_cuboid(resource, corner, req.get_resolution(), np.expand_dims(black_cuboid, axis=0), req.get_time()[0], iso=iso, to_black=True) except Exception as e: # TODO: Eventually remove as this level of detail should not be sent to the user log = BossLogger().logger log.exception('Error during write_cuboid: {}'.format(e)) return BossHTTPError('Error during write_cuboid: {}'.format(e), ErrorCodes.BAD_REQUEST) # If the channel status is DOWNSAMPLED change status to NOT_DOWNSAMPLED since you just wrote data channel = resource.get_channel() if channel.downsample_status.upper() == "DOWNSAMPLED": # Get Channel object and update status lookup_key = resource.get_lookup_key() _, exp_id, _ = lookup_key.split("&") channel_obj = Channel.objects.get(name=channel.name, experiment=int(exp_id)) channel_obj.downsample_status = "NOT_DOWNSAMPLED" channel_obj.downsample_arn = "" channel_obj.save() # Send data to renderer return HttpResponse(status=200)
def get(self, request, collection, experiment, channel): """View to provide a channel's downsample status and properties Args: request: DRF Request object collection (str): Unique Collection identifier, indicating which collection you want to access experiment (str): Experiment identifier, indicating which experiment you want to access channel (str): Channel identifier, indicating which channel you want to access Returns: """ if "iso" in request.query_params: if request.query_params["iso"].lower() == "true": iso = True else: iso = False else: iso = False # Process request and validate try: request_args = { "service": "downsample", "collection_name": collection, "experiment_name": experiment, "channel_name": channel } req = BossRequest(request, request_args) except BossError as err: return err.to_http() # Convert to Resource resource = project.BossResourceDjango(req) # Get Status channel = resource.get_channel() experiment = resource.get_experiment() to_renderer = {"status": channel.downsample_status} # Get hierarchy levels to_renderer["num_hierarchy_levels"] = experiment.num_hierarchy_levels # Gen Voxel dims voxel_size = {} voxel_dims = resource.get_downsampled_voxel_dims(iso=iso) for res, dims in enumerate(voxel_dims): voxel_size["{}".format(res)] = dims to_renderer["voxel_size"] = voxel_size # Gen Extent dims extent = {} extent_dims = resource.get_downsampled_extent_dims(iso=iso) for res, dims in enumerate(extent_dims): extent["{}".format(res)] = dims to_renderer["extent"] = extent # Get Cuboid dims cuboid_size = {} for res in range(0, experiment.num_hierarchy_levels): cuboid_size["{}".format(res)] = CUBOIDSIZE[res] to_renderer["cuboid_size"] = cuboid_size # Send data to renderer return Response(to_renderer)