def post(self, request, collection):
"""Create a new collection
View to create a new collection and an associated bosskey for that collection
Args:
request: DRF Request object
collection : Collection name
Returns:
Collection
"""
col_data = request.data.copy()
col_data['name'] = collection
# Save the object
serializer = CollectionSerializer(data=col_data)
if serializer.is_valid():
serializer.save(creator=self.request.user)
collection_obj = Collection.objects.get(name=col_data['name'])
# Assign permissions to the users primary group and admin group
BossPermissionManager.add_permissions_primary_group(self.request.user, collection_obj)
BossPermissionManager.add_permissions_admin_group(collection_obj)
lookup_key = str(collection_obj.pk)
boss_key = collection_obj.name
LookUpKey.add_lookup(lookup_key, boss_key, collection_obj.name)
return Response(serializer.data, status=status.HTTP_201_CREATED)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
def delete(self, request, collection, experiment, channel_layer):
"""
Delete a Channel or a Layer
Args:
request: DRF Request object
collection: Collection name
experiment: Experiment name
channel_layer: Channel or Layer name
Returns :
Http status
"""
try:
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment, collection=collection_obj)
channel_layer_obj = ChannelLayer.objects.get(name=channel_layer, experiment=experiment_obj)
if request.user.has_perm("delete", channel_layer_obj):
channel_layer_obj.delete()
# delete the lookup key for this object
LookUpKey.delete_lookup_key(collection, experiment, channel_layer)
return HttpResponse(status=204)
else:
return BossPermissionError('delete', channel_layer)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except ChannelLayer.DoesNotExist:
return BossResourceNotFoundError(channel_layer)
except ProtectedError:
return BossHTTPError("Cannot delete {}. It has layers that reference it.".format(channel_layer),
ErrorCodes.INTEGRITY_ERROR)
def delete(self, request, collection, experiment):
"""
Delete a experiment
Args:
request: DRF Request object
collection: Name of collection
experiment: Experiment name to delete
Returns:
Http status
"""
try:
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment, collection=collection_obj)
if request.user.has_perm("delete", experiment_obj):
experiment_obj.delete()
# # get the lookup key and delete all the meta data for this object
# bosskey = collection + '&' + experiment
# lkey = LookUpKey.get_lookup_key(bosskey)
# mdb = MetaDB()
# mdb.delete_meta_keys(lkey)
# delete the lookup key for this object
LookUpKey.delete_lookup_key(collection, experiment, None)
return HttpResponse(status=204)
else:
return BossPermissionError('delete', experiment)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except ProtectedError:
return BossHTTPError("Cannot delete {}. It has channels or layers that reference "
"it.".format(experiment), ErrorCodes.INTEGRITY_ERROR)
def test_update_lookup_experiment(self):
"""
On an experiment rename, make sure only resources that are children of
the experiment are changed.
This is a regression test.
"""
new_exp_name = 'new_exp'
collection = 'col2'
orig_exp_name = 'exp1'
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=orig_exp_name, collection=collection_obj)
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk)
boss_key = collection_obj.name + '&' + new_exp_name
# Method under test.
LookUpKey.update_lookup_experiment(
lookup_key, boss_key, collection_obj.name, new_exp_name)
# There should be still 5 rows in the lookup table with orig_exp_name
# as the experiment name under col1. There should be the experiment
# and 4 channels.
all_lookup_objs = BossLookup.objects.filter(experiment_name=orig_exp_name)
self.assertEqual(5, len(all_lookup_objs))
def put(self, request, collection):
"""
Update a collection using django rest framework
Args:
request: DRF Request object
collection: Collection name
Returns:
Collection
"""
try:
# Check if the object exists
collection_obj = Collection.objects.get(name=collection)
# Check for permissions
if request.user.has_perm("update", collection_obj):
serializer = CollectionSerializer(collection_obj, data=request.data, partial=True)
if serializer.is_valid():
serializer.save()
# update the lookup key if you update the name
if 'name' in request.data and request.data['name'] != collection:
lookup_key = str(collection_obj.pk)
boss_key = request.data['name']
LookUpKey.update_lookup(lookup_key, boss_key, request.data['name'])
return Response(serializer.data)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('update', collection)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
def test_update_lookup_experiment(self):
"""
On an experiment rename, make sure only resources that are children of
the experiment are changed.
This is a regression test.
"""
new_exp_name = 'new_exp'
collection = 'col2'
orig_exp_name = 'exp1'
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=orig_exp_name,
collection=collection_obj)
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk)
boss_key = collection_obj.name + '&' + new_exp_name
# Method under test.
LookUpKey.update_lookup_experiment(lookup_key, boss_key,
collection_obj.name, new_exp_name)
# There should be still 5 rows in the lookup table with orig_exp_name
# as the experiment name under col1. There should be the experiment
# and 4 channels.
all_lookup_objs = BossLookup.objects.filter(
experiment_name=orig_exp_name)
self.assertEqual(5, len(all_lookup_objs))
def post(self, request, collection, experiment):
"""Create a new experiment
View to create a new experiment and an associated bosskey for that experiment
Args:
request: DRF Request object
collection : Collection name
experiment : Experiment name
Returns:
Experiment
"""
experiment_data = request.data.copy()
experiment_data['name'] = experiment
try:
# Get the collection information
collection_obj = Collection.objects.get(name=collection)
if request.user.has_perm("add", collection_obj):
experiment_data['collection'] = collection_obj.pk
# Update the coordinate frame
if 'coord_frame' not in experiment_data:
return BossHTTPError("This request requires a valid coordinate frame",
ErrorCodes.INVALID_POST_ARGUMENT)
coord_frame_obj = CoordinateFrame.objects.get(name=experiment_data['coord_frame'])
experiment_data['coord_frame'] = coord_frame_obj.pk
serializer = ExperimentSerializer(data=experiment_data)
if serializer.is_valid():
serializer.save(creator=self.request.user)
experiment_obj = Experiment.objects.get(name=experiment_data['name'], collection=collection_obj)
# Assign permissions to the users primary group and admin group
BossPermissionManager.add_permissions_primary_group(self.request.user, experiment_obj)
BossPermissionManager.add_permissions_admin_group(experiment_obj)
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk)
boss_key = collection_obj.name + '&' + experiment_obj.name
LookUpKey.add_lookup(lookup_key, boss_key, collection_obj.name, experiment_obj.name)
serializer = ExperimentReadSerializer(experiment_obj)
return Response(serializer.data, status=status.HTTP_201_CREATED)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('add', collection)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except CoordinateFrame.DoesNotExist:
return BossResourceNotFoundError(experiment_data['coord_frame'])
except ValueError:
return BossHTTPError("Value Error.Collection id {} in post data needs to be an integer"
.format(experiment_data['collection']), ErrorCodes.TYPE_ERROR)
def put(self, request, collection, experiment):
"""
Update a experiment using django rest framework
Args:
request: DRF Request object
collection: Collection name
experiment : Experiment name for the new experiment
Returns:
Experiment
"""
try:
# Check if the object exists
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment,
collection=collection_obj)
if request.user.has_perm("update", experiment_obj):
serializer = ExperimentUpdateSerializer(experiment_obj,
data=request.data,
partial=True)
if serializer.is_valid():
serializer.save()
# update the lookup key if you update the name
if 'name' in request.data and request.data[
'name'] != experiment:
lookup_key = str(collection_obj.pk) + '&' + str(
experiment_obj.pk)
boss_key = collection_obj.name + '&' + request.data[
'name']
LookUpKey.update_lookup_experiment(
lookup_key, boss_key, collection_obj.name,
request.data['name'])
# return the object back to the user
experiment = serializer.data['name']
experiment_obj = Experiment.objects.get(
name=experiment, collection=collection_obj)
serializer = ExperimentReadSerializer(experiment_obj)
return Response(serializer.data)
else:
return BossHTTPError("{}".format(serializer.errors),
ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('update', experiment)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except BossError as err:
return err.to_http()
def invoke_ingest_lambda(self, ingest_job, num_invokes=1):
"""Method to trigger extra lambda functions to make sure all the ingest jobs that are actually fully populated
kick through
Args:
ingest_job: Ingest job object
num_invokes(int): number of invocations to fire
Returns:
"""
bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
fake_chunk_key = (BossBackend(self.config)).encode_chunk_key(
16, project_info, ingest_job.resolution, 0, 0, 0, 0)
event = {
"ingest_job": ingest_job.id,
"chunk_key": fake_chunk_key,
"lambda-name": "ingest"
}
# Invoke Ingest lambda functions
lambda_client = boto3.client('lambda',
region_name=bossutils.aws.get_region())
for _ in range(0, num_invokes):
lambda_client.invoke(FunctionName=INGEST_LAMBDA,
InvocationType='Event',
Payload=json.dumps(event).encode())
def invoke_ingest_lambda(self, ingest_job, num_invokes=1):
"""Method to trigger extra lambda functions to make sure all the ingest jobs that are actually fully populated
kick through
Args:
ingest_job: Ingest job object
num_invokes(int): number of invocations to fire
Returns:
"""
bosskey = ingest_job.collection + CONNECTOR + ingest_job.experiment + CONNECTOR + ingest_job.channel
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
fake_chunk_key = (BossBackend(self.config)).encode_chunk_key(16, project_info,
ingest_job.resolution,
0, 0, 0, 0)
event = {"ingest_job": ingest_job.id,
"chunk_key": fake_chunk_key,
"function-name": INGEST_LAMBDA,
"lambda-name": "ingest"}
# Invoke Ingest lambda functions
lambda_client = boto3.client('lambda', region_name=bossutils.aws.get_region())
for _ in range(0, num_invokes):
lambda_client.invoke(FunctionName=INGEST_LAMBDA,
InvocationType='Event',
Payload=json.dumps(event).encode())
def put(self, request, collection, experiment, channel_layer):
"""
Update new Channel or Layer
Args:
request: DRF Request object
collection: Collection name
experiment: Experiment name
channel_layer: Channel or Layer name
Returns :
ChannelLayer
"""
channel_layer_data = request.data.copy()
if 'is_channel' in channel_layer_data:
channel_layer_data['is_channel'] = self.get_bool(channel_layer_data['is_channel'])
try:
# Check if the object exists
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment, collection=collection_obj)
channel_layer_obj = ChannelLayer.objects.get(name=channel_layer, experiment=experiment_obj)
if request.user.has_perm("update", channel_layer_obj):
serializer = ChannelLayerSerializer(channel_layer_obj, data=request.data, partial=True)
if serializer.is_valid():
serializer.save()
# update the lookup key if you update the name
if 'name' in request.data and request.data['name'] != channel_layer:
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk) + '&' \
+ str(channel_layer_obj.pk)
boss_key = collection_obj.name + '&' + experiment_obj.name + '&' + request.data['name']
LookUpKey.update_lookup(lookup_key, boss_key, collection_obj.name, experiment_obj.name,
request.data['name'])
return Response(serializer.data)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('update', channel_layer)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except ChannelLayer.DoesNotExist:
return BossResourceNotFoundError(channel_layer)
def _generate_upload_queue_args(self, ingest_job):
"""
Generate dictionary to include in messages placed in the tile upload queue.
Args:
ingest_job (IngestJob):
Returns:
(dict)
Raises:
(BossError): If ingest_job.ingest_type invalid.
"""
bosskey = ingest_job.collection + CONNECTOR + ingest_job.experiment + CONNECTOR + ingest_job.channel
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
args = {
'job_id': ingest_job.id,
'upload_queue': ingest_job.upload_queue,
'ingest_queue': ingest_job.ingest_queue,
'resolution': ingest_job.resolution,
'project_info': lookup_key.split(CONNECTOR),
'ingest_type': ingest_job.ingest_type,
't_start': ingest_job.t_start,
't_stop': ingest_job.t_stop,
't_tile_size': 1,
'x_start': ingest_job.x_start,
'x_stop': ingest_job.x_stop,
'x_tile_size': ingest_job.tile_size_x,
'y_start': ingest_job.y_start,
'y_stop': ingest_job.y_stop,
'y_tile_size': ingest_job.tile_size_y,
'z_start': ingest_job.z_start,
'z_stop': ingest_job.z_stop,
'z_tile_size': 1
}
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# Always the Boss cuboid z size for tile jobs.
args['z_chunk_size'] = 16
elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# tile_size_* holds the chunk size dimensions for volumetric jobs.
args['z_chunk_size'] = ingest_job.tile_size_z
else:
raise BossError(
"Ingest job's ingest_type has invalid value: {}".format(
self.job.ingest_type), ErrorCodes.UNABLE_TO_VALIDATE)
return args
def _generate_upload_queue_args(self, ingest_job):
"""
Generate dictionary to include in messages placed in the tile upload queue.
Args:
ingest_job (IngestJob):
Returns:
(dict)
Raises:
(BossError): If ingest_job.ingest_type invalid.
"""
bosskey = ingest_job.collection + CONNECTOR + ingest_job.experiment + CONNECTOR + ingest_job.channel
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
args = {
'job_id': ingest_job.id,
'upload_queue': ingest_job.upload_queue,
'ingest_queue': ingest_job.ingest_queue,
'resolution': ingest_job.resolution,
'project_info': lookup_key.split(CONNECTOR),
'ingest_type': ingest_job.ingest_type,
't_start': ingest_job.t_start,
't_stop': ingest_job.t_stop,
't_tile_size': 1,
'x_start': ingest_job.x_start,
'x_stop': ingest_job.x_stop,
'x_tile_size': ingest_job.tile_size_x,
'y_start': ingest_job.y_start,
'y_stop': ingest_job.y_stop,
'y_tile_size': ingest_job.tile_size_y,
'z_start': ingest_job.z_start,
'z_stop': ingest_job.z_stop,
'z_tile_size': 1
}
if ingest_job.ingest_type == IngestJob.TILE_INGEST:
# Always the Boss cuboid z size for tile jobs.
args['z_chunk_size'] = 16
elif ingest_job.ingest_type == IngestJob.VOLUMETRIC_INGEST:
# tile_size_* holds the chunk size dimensions for volumetric jobs.
args['z_chunk_size'] = ingest_job.tile_size_z
else:
raise BossError(
"Ingest job's ingest_type has invalid value: {}".format(
self.job.ingest_type), ErrorCodes.UNABLE_TO_VALIDATE)
return args
def populate_upload_queue(self):
"""Execute the populate_upload_queue Step Function
Returns:
string: ARN of the StepFunction Execution started
Raises:
BossError : if there is no valid ingest job
"""
if self.job is None:
raise BossError(
"Unable to generate upload tasks for the ingest service. Please specify a ingest job",
ErrorCodes.UNABLE_TO_VALIDATE)
ingest_job = self.job
bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
# TODO DP ???: create IngestJob method that creates the StepFunction arguments?
args = {
'upload_sfn': config['sfn']['upload_sfn'],
'job_id': ingest_job.id,
'upload_queue': ingest_job.upload_queue,
'ingest_queue': ingest_job.ingest_queue,
'resolution': ingest_job.resolution,
'project_info': lookup_key.split(CONNECTER),
't_start': ingest_job.t_start,
't_stop': ingest_job.t_stop,
't_tile_size': 1,
'x_start': ingest_job.x_start,
'x_stop': ingest_job.x_stop,
'x_tile_size': ingest_job.tile_size_x,
'y_start': ingest_job.y_start,
'y_stop': ingest_job.y_stop,
'y_tile_size': ingest_job.tile_size_y,
'z_start': ingest_job.z_start,
'z_stop': ingest_job.z_stop,
'z_tile_size': 16,
}
session = bossutils.aws.get_session()
populate_sfn = config['sfn']['populate_upload_queue']
arn = bossutils.aws.sfn_execute(session, populate_sfn, args)
return arn
def generate_upload_tasks(self, job_id=None):
"""
Generate upload tasks for the ingest job. This creates once task for each tile that has to be uploaded in the
ingest queue
Args:
job_id: Job id of the ingest queue. If not included this takes the current ingest job
Returns:
None
Raises:
BossError : if there is no valid ingest job
"""
if job_id is None and self.job is None:
raise BossError(
"Unable to generate upload tasks for the ingest service. Please specify a ingest job",
ErrorCodes.UNABLE_TO_VALIDATE)
elif job_id:
# Using the job id to get the job
try:
ingest_job = IngestJob.objects.get(id=job_id)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(job_id),
ErrorCodes.RESOURCE_NOT_FOUND)
else:
ingest_job = self.job
# Generate upload tasks for the ingest job
# Get the project information
bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
# Batch messages and write to file
base_file_name = 'tasks_' + lookup_key + '_' + str(ingest_job.id)
self.file_index = 0
# open file
f = io.StringIO()
header = {
'job_id': ingest_job.id,
'upload_queue_url': ingest_job.upload_queue,
'ingest_queue_url': ingest_job.ingest_queue
}
f.write(json.dumps(header))
f.write('\n')
num_msg_per_file = 0
for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
# For each time step, compute the chunks and tile keys
for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
for y in range(ingest_job.y_start, ingest_job.y_stop,
ingest_job.tile_size_y):
for x in range(ingest_job.x_start, ingest_job.x_stop,
ingest_job.tile_size_x):
# compute the chunk indices
chunk_x = int(x / ingest_job.tile_size_x)
chunk_y = int(y / ingest_job.tile_size_y)
chunk_z = int(z / 16)
# Compute the number of tiles in the chunk
if ingest_job.z_stop - z >= 16:
num_of_tiles = 16
else:
num_of_tiles = ingest_job.z_stop - z
# Generate the chunk key
chunk_key = (BossBackend(
self.config)).encode_chunk_key(
num_of_tiles, project_info,
ingest_job.resolution, chunk_x, chunk_y,
chunk_z, time_step)
self.num_of_chunks += 1
# get the tiles keys for this chunk
for tile in range(z, z + num_of_tiles):
# get the tile key
tile_key = (BossBackend(
self.config)).encode_tile_key(
project_info, ingest_job.resolution,
chunk_x, chunk_y, tile, time_step)
self.count_of_tiles += 1
# Generate the upload task msg
msg = chunk_key + ',' + tile_key + '\n'
f.write(msg)
num_msg_per_file += 1
# if there are 10 messages in the batch send it to the upload queue.
if num_msg_per_file == MAX_NUM_MSG_PER_FILE:
fname = base_file_name + '_' + str(
self.file_index + 1) + '.txt'
self.upload_task_file(fname, f.getvalue())
self.file_index += 1
f.close()
# status = self.send_upload_message_batch(batch_msg)
fname = base_file_name + '_' + str(
self.file_index + 1) + '.txt'
f = io.StringIO()
header = {
'job_id': ingest_job.id,
'upload_queue_url':
ingest_job.upload_queue,
'ingest_queue_url': ingest_job.ingest_queue
}
f.write(json.dumps(header))
f.write('\n')
num_msg_per_file = 0
# Edge case: the last batch size maybe smaller than 10
if num_msg_per_file != 0:
fname = base_file_name + '_' + str(self.file_index + 1) + '.txt'
self.upload_task_file(fname, f.getvalue())
f.close()
self.file_index += 1
num_msg_per_file = 0
# Update status
self.job.tile_count = self.count_of_tiles
self.job.save()
def generate_upload_tasks(self, job_id=None):
"""
Args:
job_id:
Returns:
"""
if job_id is None and self.job is None:
raise BossError(
"Unable to generate upload tasks for the ingest service. Please specify a ingest job",
ErrorCodes.UNABLE_TO_VALIDATE)
elif job_id:
# Using the job id to get the job
try:
ingest_job = IngestJob.objects.get(id=job_id)
except IngestJob.DoesNotExist:
raise BossError(
"Ingest job with id {} does not exist".format(job_id),
ErrorCodes.RESOURCE_NOT_FOUND)
else:
ingest_job = self.job
# Generate upload tasks for the ingest job
# Get the project information
bosskey = ingest_job.collection + CONNECTER + ingest_job.experiment + CONNECTER + ingest_job.channel_layer
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
# For each time step, compute the chunks and tile keys
for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
for y in range(ingest_job.y_start, ingest_job.y_stop,
ingest_job.tile_size_y):
for x in range(ingest_job.x_start, ingest_job.x_stop,
ingest_job.tile_size_x):
# compute the chunk indices
chunk_x = int(x / ingest_job.tile_size_x)
chunk_y = int(y / ingest_job.tile_size_y)
chunk_z = int(z / 16)
# Compute the number of tiles in the chunk
if ingest_job.z_stop - z >= 16:
num_of_tiles = 16
else:
num_of_tiles = ingest_job.z_stop - z
# Generate the chunk key
chunk_key = (BossBackend(
self.config)).encode_chunk_key(
num_of_tiles, project_info,
ingest_job.resolution, chunk_x, chunk_y,
chunk_z, time_step)
# get the tiles keys for this chunk
for tile in range(0, num_of_tiles):
# get the tile key
tile_key = (BossBackend(
self.config)).encode_tile_key(
project_info, ingest_job.resolution,
chunk_x, chunk_y, tile, time_step)
# Generate the upload task msg
msg = self.create_upload_task_message(
ingest_job.id, chunk_key, tile_key,
ingest_job.upload_queue,
ingest_job.ingest_queue)
# Upload the message
self.send_upload_task_message(msg)
def post(self, request, collection, experiment, channel_layer):
"""
Post a new Channel
Args:
request: DRF Request object
collection: Collection name
experiment: Experiment name
channel_layer: Channel or Layer name
Returns :
ChannelLayer
"""
channel_layer_data = request.data.copy()
channel_layer_data['name'] = channel_layer
try:
if 'channels' in channel_layer_data:
channels = dict(channel_layer_data)['channels']
else:
channels = []
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment, collection=collection_obj)
# Check for add permissions
if request.user.has_perm("add", experiment_obj):
channel_layer_data['experiment'] = experiment_obj.pk
channel_layer_data['is_channel'] = self.get_bool(channel_layer_data['is_channel'])
# layers require at least 1 channel
if (channel_layer_data['is_channel'] is False) and (len(channels) == 0):
return BossHTTPError("Invalid Request.Please specify a valid channel for the layer",
ErrorCodes.INVALID_POST_ARGUMENT)
serializer = ChannelLayerSerializer(data=channel_layer_data)
if serializer.is_valid():
serializer.save(creator=self.request.user)
channel_layer_obj = ChannelLayer.objects.get(name=channel_layer_data['name'],
experiment=experiment_obj)
# Layer?
if not channel_layer_obj.is_channel:
# Layers must map to at least 1 channel
for channel_id in channels:
# Is this a valid channel?
channel_obj = ChannelLayer.objects.get(pk=channel_id)
if channel_obj:
channel_layer_map = {'channel': channel_id, 'layer': channel_layer_obj.pk}
map_serializer = ChannelLayerMapSerializer(data=channel_layer_map)
if map_serializer.is_valid():
map_serializer.save()
# Assign permissions to the users primary group
BossPermissionManager.add_permissions_primary_group(self.request.user, channel_layer_obj)
# Add Lookup key
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk) + '&' + str(channel_layer_obj.pk)
boss_key = collection_obj.name + '&' + experiment_obj.name + '&' + channel_layer_obj.name
LookUpKey.add_lookup(lookup_key, boss_key, collection_obj.name, experiment_obj.name,
channel_layer_obj.name)
return Response(serializer.data, status=status.HTTP_201_CREATED)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('add', experiment)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except ChannelLayer.DoesNotExist:
return BossResourceNotFoundError(channel_layer)
except ValueError:
return BossHTTPError("Value Error in post data", ErrorCodes.TYPE_ERROR)
def put(self, request, collection, experiment, channel):
"""
Update new Channel
Args:
request: DRF Request object
collection: Collection name
experiment: Experiment name
channel: Channel name
Returns :
Channel
"""
if 'name' in request.data:
channel_name = request.data['name']
else:
channel_name = channel
try:
# Check if the object exists
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment, collection=collection_obj)
channel_obj = Channel.objects.get(name=channel, experiment=experiment_obj)
if request.user.has_perm("update", channel_obj):
# The source and related channels are names and need to be removed from the dict before serialization
source_channels = request.data.pop('sources', [])
related_channels = request.data.pop('related', [])
# Validate the source and related channels if they are incuded
channels = self.validate_source_related_channels(experiment_obj, source_channels, related_channels)
source_channels_objs = channels[0]
related_channels_objs = channels[1]
serializer = ChannelUpdateSerializer(channel_obj, data=request.data, partial=True)
if serializer.is_valid():
serializer.save()
channel_obj = Channel.objects.get(name=channel_name, experiment=experiment_obj)
# Save source and related channels if they are valid
channel_obj = self.update_source_related_channels(channel_obj, experiment_obj, source_channels_objs,
related_channels_objs)
# update the lookup key if you update the name
if 'name' in request.data and request.data['name'] != channel:
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk) + '&' \
+ str(channel_obj.pk)
boss_key = collection_obj.name + '&' + experiment_obj.name + '&' + request.data['name']
LookUpKey.update_lookup(lookup_key, boss_key, collection_obj.name, experiment_obj.name,
request.data['name'])
# return the object back to the user
channel = serializer.data['name']
channel_obj = Channel.objects.get(name=channel, experiment=experiment_obj)
serializer = ChannelReadSerializer(channel_obj)
return Response(serializer.data)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('update', channel)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except Channel.DoesNotExist:
return BossResourceNotFoundError(channel)
def post(self, request, collection, experiment, channel):
"""
Post a new Channel
Args:
request: DRF Request object
collection: Collection name
experiment: Experiment name
channel: Channel name
Returns :
Channel
"""
channel_data = request.data.copy()
channel_data['name'] = channel
try:
# Get the collection and experiment
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment, collection=collection_obj)
# Check for add permissions
if request.user.has_perm("add", experiment_obj):
channel_data['experiment'] = experiment_obj.pk
# The source and related channels are names and need to be removed from the dict before serialization
source_channels = channel_data.pop('sources', [])
related_channels = channel_data.pop('related', [])
# Source channels have to be included for new annotation channels
if 'type' in channel_data and channel_data['type'] == 'annotation' and len(source_channels) == 0:
return BossHTTPError("Annotation channels require the source channel to be set. "
"Specify a valid source channel in the post", ErrorCodes.INVALID_POST_ARGUMENT)
# Validate the source and related channels if they are incuded
channels = self.validate_source_related_channels(experiment_obj, source_channels, related_channels)
source_channels_objs = channels[0]
related_channels_objs = channels[1]
# Validate and create the channel
serializer = ChannelSerializer(data=channel_data)
if serializer.is_valid():
serializer.save(creator=self.request.user)
channel_obj = Channel.objects.get(name=channel_data['name'], experiment=experiment_obj)
# Save source and related channels if they are valid
channel_obj = self.add_source_related_channels(channel_obj, experiment_obj, source_channels_objs,
related_channels_objs)
# Assign permissions to the users primary group and admin group
BossPermissionManager.add_permissions_primary_group(self.request.user, channel_obj)
BossPermissionManager.add_permissions_admin_group(channel_obj)
# Add Lookup key
lookup_key = str(collection_obj.pk) + '&' + str(experiment_obj.pk) + '&' + str(channel_obj.pk)
boss_key = collection_obj.name + '&' + experiment_obj.name + '&' + channel_obj.name
LookUpKey.add_lookup(lookup_key, boss_key, collection_obj.name, experiment_obj.name,
channel_obj.name)
serializer = ChannelReadSerializer(channel_obj)
return Response(serializer.data, status=status.HTTP_201_CREATED)
else:
return BossHTTPError("{}".format(serializer.errors), ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('add', experiment)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except Channel.DoesNotExist:
return BossResourceNotFoundError(channel)
except BossError as err:
return err.to_http()
except ValueError:
return BossHTTPError("Value Error in post data", ErrorCodes.TYPE_ERROR)
def post(self, request, collection, experiment, channel):
"""
Post a new Channel
Args:
request: DRF Request object
collection: Collection name
experiment: Experiment name
channel: Channel name
Returns :
Channel
"""
channel_data = request.data.copy()
channel_data['name'] = channel
try:
is_admin = BossPermissionManager.is_in_group(
request.user, ADMIN_GRP)
if 'bucket' in channel_data and channel_data[
'bucket'] and not is_admin:
return BossHTTPError('Only admins can set bucket name',
ErrorCodes.MISSING_PERMISSION)
if 'cv_path' in channel_data and channel_data[
'cv_path'] and not is_admin:
return BossHTTPError('Only admins can set cv_path',
ErrorCodes.MISSING_PERMISSION)
# Get the collection and experiment
collection_obj = Collection.objects.get(name=collection)
experiment_obj = Experiment.objects.get(name=experiment,
collection=collection_obj)
# Check for add permissions
if request.user.has_perm("add", experiment_obj):
channel_data['experiment'] = experiment_obj.pk
use_cloudvol = channel_data.get(
'storage_type', None) == Channel.StorageType.CLOUD_VOLUME
cv_path = channel_data.get('cv_path', None)
if use_cloudvol and (cv_path is None or cv_path == ''):
channel_data[
'cv_path'] = f'/{collection}/{experiment}/{channel}'
if use_cloudvol:
# DX NOTE: For now we assume that cloudvolume channels are downsampled. This means
# that the num_hierarchy_levels in the experiment should be limited to the available
# mip levels in the cloudvolume layer.
channel_data['downsample_status'] = 'DOWNSAMPLED'
# The source and related channels are names and need to be removed from the dict before serialization
source_channels = channel_data.pop('sources', [])
related_channels = channel_data.pop('related', [])
# TODO: Removed source channel requirement for annotation channels. Future update should allow source channel from
# different collections.
# Source channels have to be included for new annotation channels
# if 'type' in channel_data and channel_data['type'] == 'annotation' and len(source_channels) == 0:
# return BossHTTPError("Annotation channels require the source channel to be set. "
# "Specify a valid source channel in the post", ErrorCodes.INVALID_POST_ARGUMENT)
# Validate the source and related channels if they are incuded
channels = self.validate_source_related_channels(
experiment_obj, source_channels, related_channels)
source_channels_objs = channels[0]
related_channels_objs = channels[1]
# Validate and create the channel
serializer = ChannelSerializer(data=channel_data)
if serializer.is_valid():
serializer.save(creator=self.request.user)
channel_obj = Channel.objects.get(
name=channel_data['name'], experiment=experiment_obj)
# Save source and related channels if they are valid
channel_obj = self.add_source_related_channels(
channel_obj, experiment_obj, source_channels_objs,
related_channels_objs)
# Assign permissions to the users primary group and admin group
BossPermissionManager.add_permissions_primary_group(
self.request.user, channel_obj)
BossPermissionManager.add_permissions_admin_group(
channel_obj)
# Add Lookup key
lookup_key = str(collection_obj.pk) + '&' + str(
experiment_obj.pk) + '&' + str(channel_obj.pk)
boss_key = collection_obj.name + '&' + experiment_obj.name + '&' + channel_obj.name
LookUpKey.add_lookup(lookup_key, boss_key,
collection_obj.name,
experiment_obj.name, channel_obj.name)
serializer = ChannelReadSerializer(channel_obj)
return Response(serializer.data,
status=status.HTTP_201_CREATED)
else:
return BossHTTPError("{}".format(serializer.errors),
ErrorCodes.INVALID_POST_ARGUMENT)
else:
return BossPermissionError('add', experiment)
except Collection.DoesNotExist:
return BossResourceNotFoundError(collection)
except Experiment.DoesNotExist:
return BossResourceNotFoundError(experiment)
except Channel.DoesNotExist:
return BossResourceNotFoundError(channel)
except BossError as err:
return err.to_http()
except ValueError:
return BossHTTPError("Value Error in post data",
ErrorCodes.TYPE_ERROR)
def test_upload_tile_index_table(self):
""""""
ingest_mgmr = IngestManager()
ingest_mgmr.validate_config_file(self.example_config_data)
ingest_mgmr.validate_properties()
ingest_mgmr.owner = self.user.pk
ingest_job = ingest_mgmr.create_ingest_job()
assert (ingest_job.id is not None)
# Get the chunks in this job
# Get the project information
bosskey = ingest_job.collection + '&' + ingest_job.experiment + '&' + ingest_job.channel_layer
lookup_key = (LookUpKey.get_lookup_key(bosskey)).lookup_key
[col_id, exp_id, ch_id] = lookup_key.split('&')
project_info = [col_id, exp_id, ch_id]
proj_name = ingest_job.collection + '&' + ingest_job.experiment
tile_index_db = BossTileIndexDB(proj_name)
tilebucket = TileBucket(str(col_id) + '&' + str(exp_id))
for time_step in range(ingest_job.t_start, ingest_job.t_stop, 1):
# For each time step, compute the chunks and tile keys
for z in range(ingest_job.z_start, ingest_job.z_stop, 16):
for y in range(ingest_job.y_start, ingest_job.y_stop,
ingest_job.tile_size_y):
for x in range(ingest_job.x_start, ingest_job.x_stop,
ingest_job.tile_size_x):
# compute the chunk indices
chunk_x = int(x / ingest_job.tile_size_x)
chunk_y = int(y / ingest_job.tile_size_y)
chunk_z = int(z / 16)
# Compute the number of tiles in the chunk
if ingest_job.z_stop - z >= 16:
num_of_tiles = 16
else:
num_of_tiles = ingest_job.z_stop - z
# Generate the chunk key
chunk_key = (BossBackend(
ingest_mgmr.config)).encode_chunk_key(
num_of_tiles, project_info,
ingest_job.resolution, chunk_x, chunk_y,
chunk_z, time_step)
# Upload the chunk to the tile index db
tile_index_db.createCuboidEntry(
chunk_key, ingest_job.id)
key_map = {}
for tile in range(0, num_of_tiles):
# get the object key and upload it
#tile_key = tilebucket.encodeObjectKey(ch_id, ingest_job.resolution,
# chunk_x, chunk_y, tile, time_step)
tile_key = 'fakekey' + str(tile)
tile_index_db.markTileAsUploaded(
chunk_key, tile_key)
# for each chunk key, delete entries from the tile_bucket
# Check if data has been uploaded
chunks = list(tile_index_db.getTaskItems(ingest_job.id))
assert (len(chunks) != 0)
ingest_mgmr.delete_tiles(ingest_job)
chunks = list(tile_index_db.getTaskItems(ingest_job.id))
assert (len(chunks) == 0)
