def enqueue_cubes(queue_arn, cubes):
"""Multiprocessing.Pool worker function for enqueuing a number of messages
Called by populate_cubes()
Args:
queue_arn (str): The target SQS queue URL
cubes (list[XYZ]): A list of XYZ cubes to enqueue
"""
try:
sqs = aws.get_session().resource('sqs')
queue = sqs.Queue(queue_arn)
count = 0
msgs = ({
'Id': str(id(cube)),
'MessageBody': json.dumps(cube)
} for cube in cubes)
for batch in chunk(msgs, 10): # 10 is the message batch limit for SQS
count += 1
if count % 500 == 0:
log.debug("Enqueued {} cubes".format(count * 10))
queue.send_messages(Entries=batch)
except Exception as ex:
log.exception("Error caught in process, raising to controller")
raise ResolutionHierarchyError(str(ex))
def verify_count(args):
"""Verify that the number of messages in a queue is the given number
Args:
args: {
'arn': ARN,
'count': 0,
}
Returns:
int: The total number of messages in the queue
Raises:
Error: If the count doesn't match the messages in the queue
"""
session = aws.get_session()
client = session.client('sqs')
resp = client.get_queue_attributes(QueueUrl = args['arn'],
AttributeNames = ['ApproximateNumberOfMessages'])
messages = int(resp['Attributes']['ApproximateNumberOfMessages'])
if messages != args['count']:
raise Exception('Counts do not match')
return args['count']
def invoke_lambdas(count, lambda_arn, lambda_args, dlq_arn):
"""Multiprocessing.Pool worker function for invoking a number of lambdas
Called by launch_lambdas()
The dlq_arn queue is only checked every 10 lambdas launched. This is so
that the queue is not hit too hard when invoking a large number of lambdas
via Multiprocessing.Pool.
Args:
count (int): The number of lambdas to launch
lambda_arn (str): Name or ARN of the lambda function to invoke
lambda_args (str): The lambda payload to pass when invoking
dlq_arn (str): ARN of the SQS DLQ to monitor for error messages
"""
try:
lambda_ = aws.get_session().client('lambda')
log.info("Launching {} lambdas".format(count))
for i in range(1, count+1):
if i % 500 == 0:
log.debug("Launched {} lambdas".format(i))
if i % 10 == 0:
if check_queue(dlq_arn) > 0:
raise FailedLambdaError()
lambda_.invoke(FunctionName = lambda_arn,
InvocationType = 'Event', # Async execution
Payload = lambda_args)
except Exception as ex:
log.exception("Error caught in process, raising to controller")
raise ResolutionHierarchyError(str(ex))
def invoke_lambdas(count, lambda_arn, lambda_args, dlq_arn):
"""Multiprocessing.Pool worker function for invoking a number of lambdas
Called by launch_lambdas()
The dlq_arn queue is only checked every 10 lambdas launched. This is so
that the queue is not hit too hard when invoking a large number of lambdas
via Multiprocessing.Pool.
Args:
count (int): The number of lambdas to launch
lambda_arn (str): Name or ARN of the lambda function to invoke
lambda_args (str): The lambda payload to pass when invoking
dlq_arn (str): ARN of the SQS DLQ to monitor for error messages
"""
try:
lambda_ = aws.get_session().client('lambda')
log.info("Launching {} lambdas".format(count))
for i in range(1, count + 1):
if i % 500 == 0:
log.debug("Launched {} lambdas".format(i))
if i % 10 == 0:
if check_queue(dlq_arn) > 0:
raise FailedLambdaError()
lambda_.invoke(
FunctionName=lambda_arn,
InvocationType='Event', # Async execution
Payload=lambda_args)
except Exception as ex:
log.exception("Error caught in process, raising to controller")
raise ResolutionHierarchyError(str(ex))
def verify_count(args):
"""Verify that the number of messages in a queue is at least the given number
Args:
args: {
'arn': ARN,
'count': 0,
}
Returns:
int: The total number of messages in the queue
Raises:
Error: If the messages in the queue is less then the count
"""
session = aws.get_session()
client = session.client('sqs')
resp = client.get_queue_attributes(QueueUrl = args['arn'],
AttributeNames = ['ApproximateNumberOfMessages'])
messages = int(resp['Attributes']['ApproximateNumberOfMessages'])
if messages > args['count']:
log.debug("More SQS messages then expected: required tiles {}, actual messages {}".format(args['count'],
messages))
elif messages < args['count']:
raise Exception('Not enough messages in queue')
return messages
def verify_count(args):
"""Verify that the number of messages in a queue is at least the given number
Args:
args: {
'arn': ARN,
'count': 0,
}
Returns:
int: The total number of messages in the queue
Raises:
Error: If the messages in the queue is less then the count
"""
session = aws.get_session()
client = session.client('sqs')
resp = client.get_queue_attributes(QueueUrl = args['arn'],
AttributeNames = ['ApproximateNumberOfMessages'])
messages = int(resp['Attributes']['ApproximateNumberOfMessages'])
if messages > args['count']:
log.debug("More SQS messages then expected: required tiles {}, actual messages {}".format(args['count'],
messages))
elif messages < args['count']:
raise Exception('Not enough messages in queue')
return messages
def enqueue_cubes(queue_arn, cubes):
"""Multiprocessing.Pool worker function for enqueuing a number of messages
Called by populate_cubes()
Args:
queue_arn (str): The target SQS queue URL
cubes (list[XYZ]): A list of XYZ cubes to enqueue
"""
try:
sqs = aws.get_session().resource('sqs')
queue = sqs.Queue(queue_arn)
count = 0
msgs = ({'Id': str(id(cube)),
'MessageBody': json.dumps(cube)}
for cube in cubes)
for batch in chunk(msgs, 10): # 10 is the message batch limit for SQS
count += 1
if count % 500 == 0:
log.debug ("Enqueued {} cubes".format(count * 10))
queue.send_messages(Entries=batch)
except Exception as ex:
log.exception("Error caught in process, raising to controller")
raise ResolutionHierarchyError(str(ex))
def update_visibility_timeout(queue_url, receipt_handle):
"""
Update the visibility timeout of the message for the current downsample job.
This keeps the message from getting redelivered while the downsample is
still running.
Args:
queue_url (str): URL of SQS queue.
receipt_handle (str): Message's receipt handle.
Returns:
Raises:
"""
try:
session = aws.get_session()
sqs = session.client('sqs')
sqs.change_message_visibility(
QueueUrl=queue_url,
ReceiptHandle=receipt_handle,
VisibilityTimeout=NEW_VISIBILITY_TIMEOUT.seconds)
except Exception as ex:
log.exception(
f'Error trying to update visibilty timeout of downsample job: {ex}'
)
raise
def ingest_populate(args):
"""Populate the ingest upload SQS Queue with tile information
Note: This activity will clear the upload queue of any existing
messages
Args:
args: {
'upload_sfn': ARN,
'job_id': '',
'upload_queue': ARN,
'ingest_queue': ARN,
'resolution': 0,
'project_info': [col_id, exp_id, ch_id],
't_start': 0,
't_stop': 0,
't_tile_size': 0,
'x_start': 0,
'x_stop': 0,
'x_tile_size': 0,
'y_start': 0,
'y_stop': 0
'y_tile_size': 0,
'z_start': 0,
'z_stop': 0
'z_tile_size': 16,
}
Returns:
{'arn': Upload queue ARN,
'count': Number of messages put into the queue}
"""
log.debug("Starting to populate upload queue")
clear_queue(args['upload_queue'])
results = fanout(aws.get_session(),
args['upload_sfn'],
split_args(args),
max_concurrent=MAX_NUM_PROCESSES,
rampup_delay=RAMPUP_DELAY,
rampup_backoff=RAMPUP_BACKOFF,
poll_delay=POLL_DELAY,
status_delay=STATUS_DELAY)
total_sent = reduce(lambda x, y: x + y, results, 0)
return {
'arn': args['upload_queue'],
'count': total_sent,
}
def clear_queue(arn):
"""Delete any existing messages in the given SQS queue
Args:
arn (string): SQS ARN of the queue to empty
"""
log.debug("Clearing queue {}".format(arn))
session = aws.get_session()
client = session.client('sqs')
client.purge_queue(QueueUrl = arn)
time.sleep(60)
def clear_queue(arn):
"""Delete any existing messages in the given SQS queue
Args:
arn (string): SQS ARN of the queue to empty
"""
log.debug("Clearing queue {}".format(arn))
session = aws.get_session()
client = session.client('sqs')
client.purge_queue(QueueUrl = arn)
time.sleep(60)
def delete_queue(queue_arn):
"""Delete a SQS queue
Args:
queue_arn (str): The URL of the queue
"""
session = aws.get_session()
sqs = session.client('sqs')
try:
resp = sqs.delete_queue(QueueUrl = queue_arn)
except:
log.exception("Could not delete status queue '{}'".format(queue_arn))
def delete_queue(queue_arn):
"""Delete a SQS queue
Args:
queue_arn (str): The URL of the queue
"""
session = aws.get_session()
sqs = session.client('sqs')
try:
resp = sqs.delete_queue(QueueUrl=queue_arn)
except:
log.exception("Could not delete status queue '{}'".format(queue_arn))
def create_queue(queue_name):
"""Create a SQS queue
Args:
queue_name (str): Name of the queue
Return:
str: URL of the queue
"""
session = aws.get_session()
sqs = session.client('sqs')
resp = sqs.create_queue(QueueName=queue_name)
url = resp['QueueUrl']
return url
def create_queue(queue_name):
"""Create a SQS queue
Args:
queue_name (str): Name of the queue
Return:
str: URL of the queue
"""
session = aws.get_session()
sqs = session.client('sqs')
resp = sqs.create_queue(QueueName = queue_name)
url = resp['QueueUrl']
return url
def check_queue(queue_arn):
"""Get the count of messages in the given queue
The count is a combination Approximate Number of Messages, Messages Delayed,
and Messages Not Visible.
If the queue_arn contains 'dlq' the first message will be read and the SNS
DLQ message decoded to print the error that caused the DLQ message.
Args:
queue_arn (str): The URL of the queue
Return:
int: The count of messages or zero if the queue could not be queried
"""
session = aws.get_session()
sqs = session.client('sqs')
try:
resp = sqs.get_queue_attributes(
QueueUrl=queue_arn,
AttributeNames=[
'ApproximateNumberOfMessages',
'ApproximateNumberOfMessagesDelayed',
'ApproximateNumberOfMessagesNotVisible'
])
except:
log.exception(
"Could not get message count for queue '{}'".format(queue_arn))
return 0
else:
# Include both the number of messages and the number of in-flight messages
message_count = int(resp['Attributes']['ApproximateNumberOfMessages']) + \
int(resp['Attributes']['ApproximateNumberOfMessagesDelayed']) + \
int(resp['Attributes']['ApproximateNumberOfMessagesNotVisible'])
if message_count > 0 and 'dlq' in queue_arn:
try:
resp = sqs.receive_message(QueueUrl=queue_arn)
for msg in resp['Messages']:
body = json.loads(msg['Body'])
error = body['Records'][0]['Sns']['MessageAttributes'][
'ErrorMessage']['Value']
log.debug("DLQ Error: {}".format(error))
except Exception as err:
log.exception(
"Problem getting DLQ error message: {}".format(err))
return message_count
def check_downsample_queue(args):
"""
Check queue for downsample jobs.
Also marks downsample as in progress if message found.
Args:
(dict): { 'queue_url': <URL of SQS queue>, 'sfn_arn': <arn of the downsample step fcn> }
Returns:
(dict): {
'start_downsample': True | False,
'queue_url': <URL of SQS queue>,
'sfn_arn': <arn of the downsample step fcn>,
'status': 'IN_PROGRESS',
'db_host': <host name of database>
'channel_id': <id of channel for downsample>
... }
if start_downsample, then args for downsample_channel() provided including
job_receipt_handle so message's visibility timeout can be
adjusted or be deleted from the queue. The message's contents are
provided in 'msg', if one is available.
"""
session = aws.get_session()
sqs = session.client('sqs')
resp = sqs.receive_message(QueueUrl=args['queue_url'],
WaitTimeSeconds=2,
MaxNumberOfMessages=1)
if 'Messages' not in resp or len(resp['Messages']) == 0:
return {'start_downsample': False}
msg = resp['Messages'][0]
job = json.loads(msg['Body'])
output = {
'start_downsample': True,
'job_receipt_handle': msg['ReceiptHandle'],
'queue_url': args['queue_url'],
'sfn_arn': args['sfn_arn'],
'status': DownsampleStatus.IN_PROGRESS,
'db_host': job['db_host'],
'channel_id': job['channel_id'],
'msg': job,
}
return output
def lambda_throttle_count(lambda_arn):
"""Read the Throttle count for the given Lambda function from Cloud Watch
The metric is read for the last minute, which is in the process of being updated
so the value read is not the final value that is recorded in Cloud Watch for the
given time period.
Args:
lambda_arn (str): ARN or Name of the lambda to get the metric for
Return:
float: The Sample Count for the Lambda's Throttle count
-1: If there was an error getting the metric
"""
session = aws.get_session()
cw = session.client('cloudwatch')
lambda_name = lambda_arn.split(':')[-1]
try:
end = datetime.now()
begin = end - timedelta(minutes=1)
resp = cw.get_metric_statistics(
Namespace='AWS/Lambda',
MetricName='Throttles',
# Limit the throttle count to only our target lambda function
Dimensions=[{
'Name': 'FunctionName',
'Value': lambda_name
}],
StartTime=begin,
EndTime=end,
Period=60,
Unit='Count',
Statistics=['SampleCount'])
if 'Datapoints' in resp and len(resp['Datapoints']) > 0:
if 'SampleCount' in resp['Datapoints'][0]:
return resp['Datapoints'][0]['SampleCount']
return 0.0
except Exception as err:
log.exception("Problem getting Lambda Throttle Count: {}".format(err))
return -1
def check_queue(queue_arn):
"""Get the count of messages in the given queue
The count is a combination Approximate Number of Messages, Messages Delayed,
and Messages Not Visible.
If the queue_arn contains 'dlq' the first message will be read and the SNS
DLQ message decoded to print the error that caused the DLQ message.
Args:
queue_arn (str): The URL of the queue
Return:
int: The count of messages or zero if the queue could not be queried
"""
session = aws.get_session()
sqs = session.client('sqs')
try:
resp = sqs.get_queue_attributes(QueueUrl = queue_arn,
AttributeNames = ['ApproximateNumberOfMessages',
'ApproximateNumberOfMessagesDelayed',
'ApproximateNumberOfMessagesNotVisible'])
except:
log.exception("Could not get message count for queue '{}'".format(queue_arn))
return 0
else:
# Include both the number of messages and the number of in-flight messages
message_count = int(resp['Attributes']['ApproximateNumberOfMessages']) + \
int(resp['Attributes']['ApproximateNumberOfMessagesDelayed']) + \
int(resp['Attributes']['ApproximateNumberOfMessagesNotVisible'])
if message_count > 0 and 'dlq' in queue_arn:
try:
resp = sqs.receive_message(QueueUrl = queue_arn)
for msg in resp['Messages']:
body = json.loads(msg['Body'])
error = body['Records'][0]['Sns']['MessageAttributes']['ErrorMessage']['Value']
log.debug("DLQ Error: {}".format(error))
except Exception as err:
log.exception("Problem getting DLQ error message: {}".format(err))
return message_count
def lambda_throttle_count(lambda_arn):
"""Read the Throttle count for the given Lambda function from Cloud Watch
The metric is read for the last minute, which is in the process of being updated
so the value read is not the final value that is recorded in Cloud Watch for the
given time period.
Args:
lambda_arn (str): ARN or Name of the lambda to get the metric for
Return:
float: The Sample Count for the Lambda's Throttle count
-1: If there was an error getting the metric
"""
session = aws.get_session()
cw = session.client('cloudwatch')
lambda_name = lambda_arn.split(':')[-1]
try:
end = datetime.now()
begin = end - timedelta(minutes=1)
resp = cw.get_metric_statistics(Namespace = 'AWS/Lambda',
MetricName = 'Throttles',
# Limit the throttle count to only our target lambda function
Dimensions = [{'Name': 'FunctionName', 'Value': lambda_name}],
StartTime = begin,
EndTime = end,
Period = 60,
Unit = 'Count',
Statistics = ['SampleCount'])
if 'Datapoints' in resp and len(resp['Datapoints']) > 0:
if 'SampleCount' in resp['Datapoints'][0]:
return resp['Datapoints'][0]['SampleCount']
return 0.0
except Exception as err:
log.exception("Problem getting Lambda Throttle Count: {}".format(err))
return -1
def delete_downsample_job(args):
"""
Delete the message for the finished downsample job from SQS.
Returns a dict that allows the last state to start a new instance of the
downsample step function.
Args:
args (dict): {
'sfn_arn': ARN of this step function,,
'queue_url': <URL of SQS queue>,
'job_receipt_handle': <msg's receipt handle,
'lookup_key': <full lookup key of channel>,
...
}
Returns:
(dict): {
'queue_url': <URL of SQS queue>,
'sfn_arn': <arn of the downsample step fcn>,
'lookup_key': <full lookup key of channel>,
}
"""
try:
session = aws.get_session()
sqs = session.client('sqs')
sqs.delete_message(QueueUrl=args['queue_url'],
ReceiptHandle=args['job_receipt_handle'])
log.info(
f"Deleting SQS message for downsample of {args['lookup_key']}")
return {
'sfn_arn': args['sfn_arn'],
'queue_url': args['queue_url'],
'lookup_key': args['lookup_key'],
}
except Exception as ex:
log.exception(f'Error trying to downsample job from SQS: {ex}')
raise
def start(request, resource):
"""Main code to start a downsample
Args:
request: DRF Request object
resource (BossResourceDjango): The channel to downsample
Returns:
(HttpResponse)
"""
channel = resource.get_channel()
chan_status = channel.downsample_status.upper()
if chan_status == Channel.DownsampleStatus.IN_PROGRESS:
return BossHTTPError(
"Channel is currently being downsampled. Invalid Request.",
ErrorCodes.INVALID_STATE)
elif chan_status == Channel.DownsampleStatus.QUEUED:
return BossHTTPError(
"Channel is already waiting to be downsampled. Invalid Request.",
ErrorCodes.INVALID_STATE)
elif chan_status == Channel.DownsampleStatus.DOWNSAMPLED and not request.user.is_staff:
return BossHTTPError(
"Channel is already downsampled. Invalid Request.",
ErrorCodes.INVALID_STATE)
if request.user.is_staff:
# DP HACK: allow admin users to override the coordinate frame
frame = request.data
else:
frame = {}
boss_config = BossConfig()
collection = resource.get_collection()
experiment = resource.get_experiment()
coord_frame = resource.get_coord_frame()
lookup_key = resource.get_lookup_key()
col_id, exp_id, ch_id = lookup_key.split("&")
def get_frame(idx):
return int(frame.get(idx, getattr(coord_frame, idx)))
downsample_sfn = boss_config['sfn']['downsample_sfn']
db_host = boss_config['aws']['db']
args = {
'lookup_key':
lookup_key,
'collection_id':
int(col_id),
'experiment_id':
int(exp_id),
'channel_id':
int(ch_id),
'annotation_channel':
not channel.is_image(),
'data_type':
resource.get_data_type(),
's3_bucket':
boss_config["aws"]["cuboid_bucket"],
's3_index':
boss_config["aws"]["s3-index-table"],
'x_start':
get_frame('x_start'),
'y_start':
get_frame('y_start'),
'z_start':
get_frame('z_start'),
'x_stop':
get_frame('x_stop'),
'y_stop':
get_frame('y_stop'),
'z_stop':
get_frame('z_stop'),
'resolution':
int(channel.base_resolution),
'resolution_max':
int(experiment.num_hierarchy_levels),
'res_lt_max':
int(channel.base_resolution) + 1 < int(
experiment.num_hierarchy_levels),
'type':
experiment.hierarchy_method,
'iso_resolution':
int(resource.get_isotropic_level()),
# This step function executes: boss-tools/activities/resolution_hierarchy.py
'downsample_volume_lambda':
boss_config['lambda']['downsample_volume'],
# Need to pass step function's ARN to itself, so it can start another
# instance of itself after finishing a downsample.
'sfn_arn':
downsample_sfn,
'db_host':
db_host,
'aws_region':
get_region(),
}
# Check that only administrators are triggering extra large downsamples
if ((not request.user.is_staff) and
((args['x_stop'] - args['x_start']) *\
(args['y_stop'] - args['y_start']) *\
(args['z_stop'] - args['z_start']) > settings.DOWNSAMPLE_MAX_SIZE)):
return BossHTTPError(
"Large downsamples require admin permissions to trigger. Invalid Request.",
ErrorCodes.INVALID_STATE)
session = get_session()
downsample_sqs = boss_config['aws']['downsample-queue']
try:
enqueue_job(session, args, downsample_sqs)
except BossError as be:
return BossHTTPError(be.message, be.error_code)
compute_usage_metrics(session, args, boss_config['system']['fqdn'],
request.user.username or "public", collection.name,
experiment.name, channel.name)
region = get_region()
account_id = get_account_id()
downsample_sfn_arn = f'arn:aws:states:{region}:{account_id}:stateMachine:{downsample_sfn}'
if not check_for_running_sfn(session, downsample_sfn_arn):
bossutils.aws.sfn_run(session, downsample_sfn_arn, {
'queue_url': downsample_sqs,
'sfn_arn': downsample_sfn_arn,
})
return HttpResponse(status=201)
def downsample_channel(args):
"""
Slice the given channel into chunks of 2x2x2 or 2x2x1 cubes that are then
sent to the downsample_volume lambda for downsampling into a 1x1x1 cube at
resolution + 1.
Makes use of the bossutils.multidimensional library for simplified vector
math.
Args:
args {
downsample_volume_sfn (ARN)
collection_id (int)
experiment_id (int)
channel_id (int)
annotation_channel (bool)
data_type (str) 'uint8' | 'uint16' | 'uint64'
s3_bucket (URL)
s3_index (URL)
id_index (URL)
x_start (int)
y_start (int)
z_start (int)
x_stop (int)
y_stop (int)
z_stop (int)
resolution (int) The resolution to downsample. Creates resolution + 1
resolution_max (int) The maximum resolution to generate
res_lt_max (bool) = args['resolution'] < (args['resolution_max'] - 1)
annotation_index_max (int) The maximum resolution to index annotation channel cubes at
When annotation_index_max = N, indices will exist for res 0 - (N - 1)
type (str) 'isotropic' | 'anisotropic'
iso_resolution (int) if resolution >= iso_resolution && type == 'anisotropic' downsample both
}
"""
#log.debug("Downsampling resolution " + str(args['resolution']))
resolution = args['resolution']
dim = XYZ(*CUBOIDSIZE[resolution])
#log.debug("Cube dimensions: {}".format(dim))
def frame(key):
return XYZ(args[key.format('x')], args[key.format('y')], args[key.format('z')])
# Figure out variables for isotropic, anisotropic, or isotropic and anisotropic
# downsampling. If both are happening, fanout one and then the other in series.
configs = []
if args['type'] == 'isotropic':
configs.append({
'name': 'isotropic',
'step': XYZ(2,2,2),
'iso_flag': False,
'frame_start_key': '{}_start',
'frame_stop_key': '{}_stop',
})
else:
configs.append({
'name': 'anisotropic',
'step': XYZ(2,2,1),
'iso_flag': False,
'frame_start_key': '{}_start',
'frame_stop_key': '{}_stop',
})
if resolution >= args['iso_resolution']: # DP TODO: Figure out how to launch aniso iso version with mutating arguments
configs.append({
'name': 'isotropic',
'step': XYZ(2,2,2),
'iso_flag': True,
'frame_start_key': 'iso_{}_start',
'frame_stop_key': 'iso_{}_stop',
})
for config in configs:
frame_start = frame(config['frame_start_key'])
frame_stop = frame(config['frame_stop_key'])
step = config['step']
use_iso_flag = config['iso_flag'] # If the resulting cube should be marked with the ISO flag
index_annotations = args['resolution'] < (args['annotation_index_max'] - 1)
# Round to the furthest full cube from the center of the data
cubes_start = frame_start // dim
cubes_stop = ceildiv(frame_stop, dim)
log.debug('Downsampling {} resolution {}'.format(config['name'], resolution))
log.debug("Frame corner: {}".format(frame_start))
log.debug("Frame extent: {}".format(frame_stop))
log.debug("Cubes corner: {}".format(cubes_start))
log.debug("Cubes extent: {}".format(cubes_stop))
log.debug("Downsample step: {}".format(step))
log.debug("Indexing Annotations: {}".format(index_annotations))
# Call the downsample_volume lambda to process the data
fanout(aws.get_session(),
args['downsample_volume_sfn'],
make_args(args, cubes_start, cubes_stop, step, dim, use_iso_flag, index_annotations),
max_concurrent = MAX_NUM_PROCESSES,
rampup_delay = RAMPUP_DELAY,
rampup_backoff = RAMPUP_BACKOFF,
poll_delay = POLL_DELAY,
status_delay = STATUS_DELAY)
# Resize the coordinate frame extents as the data shrinks
# DP NOTE: doesn't currently work correctly with non-zero frame starts
def resize(var, size):
start = config['frame_start_key'].format(var)
stop = config['frame_stop_key'].format(var)
args[start] //= size
args[stop] = ceildiv(args[stop], size)
resize('x', step.x)
resize('y', step.y)
resize('z', step.z)
# if next iteration will split into aniso and iso downsampling, copy the coordinate frame
if args['type'] != 'isotropic' and (resolution + 1) == args['iso_resolution']:
def copy(var):
args['iso_{}_start'.format(var)] = args['{}_start'.format(var)]
args['iso_{}_stop'.format(var)] = args['{}_stop'.format(var)]
copy('x')
copy('y')
copy('z')
# Advance the loop and recalculate the conditional
# Using max - 1 because resolution_max should not be a valid resolution
# and res < res_max will end with res = res_max - 1, which generates res_max resolution
args['resolution'] = resolution + 1
args['res_lt_max'] = args['resolution'] < (args['resolution_max'] - 1)
return args
def ingest_populate(args):
"""Populate the ingest upload SQS Queue with tile information
Note: This activity will clear the upload queue of any existing
messages
Args:
args: {
'upload_sfn': ARN,
'job_id': '',
'upload_queue': ARN,
'ingest_queue': ARN,
'ingest_type': int (0 == TILE, 1 == VOLUMETRIC),
'resolution': 0,
'project_info': [col_id, exp_id, ch_id],
't_start': 0,
't_stop': 0,
't_tile_size': 0,
'x_start': 0,
'x_stop': 0,
'x_tile_size': 0,
'y_start': 0,
'y_stop': 0
'y_tile_size': 0,
'z_start': 0,
'z_stop': 0
'z_tile_size': 1,
'z_chunk_size': 16 for Tile or Probably 64 for Volumetric
}
Returns:
{'arn': Upload queue ARN,
'count': Number of messages put into the queue}
"""
log.debug("Starting to populate upload queue")
args['MAX_NUM_ITEMS_PER_LAMBDA'] = MAX_NUM_ITEMS_PER_LAMBDA
if (args["ingest_type"] != 0) and (args["ingest_type"] != 1):
raise ValueError("{}".format("Unknown ingest_type: {}".format(args["ingest_type"])))
clear_queue(args['upload_queue'])
results = fanout(aws.get_session(),
args['upload_sfn'],
split_args(args),
max_concurrent=MAX_NUM_PROCESSES,
rampup_delay=RAMPUP_DELAY,
rampup_backoff=RAMPUP_BACKOFF,
poll_delay=POLL_DELAY,
status_delay=STATUS_DELAY)
# At least one None values in the return of fanout. This avoids an exception in those cases.
if results is None:
messages_uploaded = 0
else:
messages_uploaded = sum(filter(None, results))
if args["ingest_type"] == 0:
tile_count = get_tile_count(args)
if tile_count != messages_uploaded:
log.warning("Messages uploaded do not match tile count. tile count: {} messages uploaded: {}"
.format(tile_count, messages_uploaded))
else:
log.debug("tile count and messages uploaded match: {}".format(tile_count))
return {
'arn': args['upload_queue'],
'count': tile_count,
}
elif args["ingest_type"] == 1:
vol_count = get_volumetric_count(args)
if vol_count != messages_uploaded:
log.warning("Messages uploaded do not match volumetric count. volumetric count: {} messages uploaded: {}"
.format(vol_count, messages_uploaded))
else:
log.debug("volumetric count and messages uploaded match: {}".format(vol_count))
return {
'arn': args['upload_queue'],
'count': vol_count,
}
def ingest_populate(args):
"""Populate the ingest upload SQS Queue with tile information
Note: This activity will clear the upload queue of any existing
messages
Args:
args: {
'upload_sfn': ARN,
'job_id': '',
'upload_queue': ARN,
'ingest_queue': ARN,
'ingest_type': int (0 == TILE, 1 == VOLUMETRIC),
'resolution': 0,
'project_info': [col_id, exp_id, ch_id],
't_start': 0,
't_stop': 0,
't_tile_size': 0,
'x_start': 0,
'x_stop': 0,
'x_tile_size': 0,
'y_start': 0,
'y_stop': 0
'y_tile_size': 0,
'z_start': 0,
'z_stop': 0
'z_tile_size': 1,
'z_chunk_size': 16 for Tile or Probably 64 for Volumetric
}
Returns:
{'arn': Upload queue ARN,
'count': Number of messages put into the queue}
"""
log.debug("Starting to populate upload queue")
args['MAX_NUM_ITEMS_PER_LAMBDA'] = MAX_NUM_ITEMS_PER_LAMBDA
if (args["ingest_type"] != 0) and (args["ingest_type"] != 1):
raise ValueError("{}".format("Unknown ingest_type: {}".format(args["ingest_type"])))
clear_queue(args['upload_queue'])
results = fanout(aws.get_session(),
args['upload_sfn'],
split_args(args),
max_concurrent=MAX_NUM_PROCESSES,
rampup_delay=RAMPUP_DELAY,
rampup_backoff=RAMPUP_BACKOFF,
poll_delay=POLL_DELAY,
status_delay=STATUS_DELAY)
# At least one None values in the return of fanout. This avoids an exception in those cases.
if results is None:
messages_uploaded = 0
else:
messages_uploaded = sum(filter(None, results))
if args["ingest_type"] == 0:
tile_count = get_tile_count(args)
if tile_count != messages_uploaded:
log.warning("Messages uploaded do not match tile count. tile count: {} messages uploaded: {}"
.format(tile_count, messages_uploaded))
else:
log.debug("tile count and messages uploaded match: {}".format(tile_count))
return {
'arn': args['upload_queue'],
'count': tile_count,
}
elif args["ingest_type"] == 1:
vol_count = get_volumetric_count(args)
if vol_count != messages_uploaded:
log.warning("Messages uploaded do not match volumetric count. volumetric count: {} messages uploaded: {}"
.format(vol_count, messages_uploaded))
else:
log.debug("volumetric count and messages uploaded match: {}".format(vol_count))
return {
'arn': args['upload_queue'],
'count': vol_count,
}