def enqueue_cubes(queue_arn, cubes): """Multiprocessing.Pool worker function for enqueuing a number of messages Called by populate_cubes() Args: queue_arn (str): The target SQS queue URL cubes (list[XYZ]): A list of XYZ cubes to enqueue """ try: sqs = aws.get_session().resource('sqs') queue = sqs.Queue(queue_arn) count = 0 msgs = ({ 'Id': str(id(cube)), 'MessageBody': json.dumps(cube) } for cube in cubes) for batch in chunk(msgs, 10): # 10 is the message batch limit for SQS count += 1 if count % 500 == 0: log.debug("Enqueued {} cubes".format(count * 10)) queue.send_messages(Entries=batch) except Exception as ex: log.exception("Error caught in process, raising to controller") raise ResolutionHierarchyError(str(ex))
def verify_count(args): """Verify that the number of messages in a queue is the given number Args: args: { 'arn': ARN, 'count': 0, } Returns: int: The total number of messages in the queue Raises: Error: If the count doesn't match the messages in the queue """ session = aws.get_session() client = session.client('sqs') resp = client.get_queue_attributes(QueueUrl = args['arn'], AttributeNames = ['ApproximateNumberOfMessages']) messages = int(resp['Attributes']['ApproximateNumberOfMessages']) if messages != args['count']: raise Exception('Counts do not match') return args['count']
def invoke_lambdas(count, lambda_arn, lambda_args, dlq_arn): """Multiprocessing.Pool worker function for invoking a number of lambdas Called by launch_lambdas() The dlq_arn queue is only checked every 10 lambdas launched. This is so that the queue is not hit too hard when invoking a large number of lambdas via Multiprocessing.Pool. Args: count (int): The number of lambdas to launch lambda_arn (str): Name or ARN of the lambda function to invoke lambda_args (str): The lambda payload to pass when invoking dlq_arn (str): ARN of the SQS DLQ to monitor for error messages """ try: lambda_ = aws.get_session().client('lambda') log.info("Launching {} lambdas".format(count)) for i in range(1, count+1): if i % 500 == 0: log.debug("Launched {} lambdas".format(i)) if i % 10 == 0: if check_queue(dlq_arn) > 0: raise FailedLambdaError() lambda_.invoke(FunctionName = lambda_arn, InvocationType = 'Event', # Async execution Payload = lambda_args) except Exception as ex: log.exception("Error caught in process, raising to controller") raise ResolutionHierarchyError(str(ex))
def invoke_lambdas(count, lambda_arn, lambda_args, dlq_arn): """Multiprocessing.Pool worker function for invoking a number of lambdas Called by launch_lambdas() The dlq_arn queue is only checked every 10 lambdas launched. This is so that the queue is not hit too hard when invoking a large number of lambdas via Multiprocessing.Pool. Args: count (int): The number of lambdas to launch lambda_arn (str): Name or ARN of the lambda function to invoke lambda_args (str): The lambda payload to pass when invoking dlq_arn (str): ARN of the SQS DLQ to monitor for error messages """ try: lambda_ = aws.get_session().client('lambda') log.info("Launching {} lambdas".format(count)) for i in range(1, count + 1): if i % 500 == 0: log.debug("Launched {} lambdas".format(i)) if i % 10 == 0: if check_queue(dlq_arn) > 0: raise FailedLambdaError() lambda_.invoke( FunctionName=lambda_arn, InvocationType='Event', # Async execution Payload=lambda_args) except Exception as ex: log.exception("Error caught in process, raising to controller") raise ResolutionHierarchyError(str(ex))
def verify_count(args): """Verify that the number of messages in a queue is at least the given number Args: args: { 'arn': ARN, 'count': 0, } Returns: int: The total number of messages in the queue Raises: Error: If the messages in the queue is less then the count """ session = aws.get_session() client = session.client('sqs') resp = client.get_queue_attributes(QueueUrl = args['arn'], AttributeNames = ['ApproximateNumberOfMessages']) messages = int(resp['Attributes']['ApproximateNumberOfMessages']) if messages > args['count']: log.debug("More SQS messages then expected: required tiles {}, actual messages {}".format(args['count'], messages)) elif messages < args['count']: raise Exception('Not enough messages in queue') return messages
def verify_count(args): """Verify that the number of messages in a queue is at least the given number Args: args: { 'arn': ARN, 'count': 0, } Returns: int: The total number of messages in the queue Raises: Error: If the messages in the queue is less then the count """ session = aws.get_session() client = session.client('sqs') resp = client.get_queue_attributes(QueueUrl = args['arn'], AttributeNames = ['ApproximateNumberOfMessages']) messages = int(resp['Attributes']['ApproximateNumberOfMessages']) if messages > args['count']: log.debug("More SQS messages then expected: required tiles {}, actual messages {}".format(args['count'], messages)) elif messages < args['count']: raise Exception('Not enough messages in queue') return messages
def enqueue_cubes(queue_arn, cubes): """Multiprocessing.Pool worker function for enqueuing a number of messages Called by populate_cubes() Args: queue_arn (str): The target SQS queue URL cubes (list[XYZ]): A list of XYZ cubes to enqueue """ try: sqs = aws.get_session().resource('sqs') queue = sqs.Queue(queue_arn) count = 0 msgs = ({'Id': str(id(cube)), 'MessageBody': json.dumps(cube)} for cube in cubes) for batch in chunk(msgs, 10): # 10 is the message batch limit for SQS count += 1 if count % 500 == 0: log.debug ("Enqueued {} cubes".format(count * 10)) queue.send_messages(Entries=batch) except Exception as ex: log.exception("Error caught in process, raising to controller") raise ResolutionHierarchyError(str(ex))
def update_visibility_timeout(queue_url, receipt_handle): """ Update the visibility timeout of the message for the current downsample job. This keeps the message from getting redelivered while the downsample is still running. Args: queue_url (str): URL of SQS queue. receipt_handle (str): Message's receipt handle. Returns: Raises: """ try: session = aws.get_session() sqs = session.client('sqs') sqs.change_message_visibility( QueueUrl=queue_url, ReceiptHandle=receipt_handle, VisibilityTimeout=NEW_VISIBILITY_TIMEOUT.seconds) except Exception as ex: log.exception( f'Error trying to update visibilty timeout of downsample job: {ex}' ) raise
def ingest_populate(args): """Populate the ingest upload SQS Queue with tile information Note: This activity will clear the upload queue of any existing messages Args: args: { 'upload_sfn': ARN, 'job_id': '', 'upload_queue': ARN, 'ingest_queue': ARN, 'resolution': 0, 'project_info': [col_id, exp_id, ch_id], 't_start': 0, 't_stop': 0, 't_tile_size': 0, 'x_start': 0, 'x_stop': 0, 'x_tile_size': 0, 'y_start': 0, 'y_stop': 0 'y_tile_size': 0, 'z_start': 0, 'z_stop': 0 'z_tile_size': 16, } Returns: {'arn': Upload queue ARN, 'count': Number of messages put into the queue} """ log.debug("Starting to populate upload queue") clear_queue(args['upload_queue']) results = fanout(aws.get_session(), args['upload_sfn'], split_args(args), max_concurrent=MAX_NUM_PROCESSES, rampup_delay=RAMPUP_DELAY, rampup_backoff=RAMPUP_BACKOFF, poll_delay=POLL_DELAY, status_delay=STATUS_DELAY) total_sent = reduce(lambda x, y: x + y, results, 0) return { 'arn': args['upload_queue'], 'count': total_sent, }
def clear_queue(arn): """Delete any existing messages in the given SQS queue Args: arn (string): SQS ARN of the queue to empty """ log.debug("Clearing queue {}".format(arn)) session = aws.get_session() client = session.client('sqs') client.purge_queue(QueueUrl = arn) time.sleep(60)
def clear_queue(arn): """Delete any existing messages in the given SQS queue Args: arn (string): SQS ARN of the queue to empty """ log.debug("Clearing queue {}".format(arn)) session = aws.get_session() client = session.client('sqs') client.purge_queue(QueueUrl = arn) time.sleep(60)
def delete_queue(queue_arn): """Delete a SQS queue Args: queue_arn (str): The URL of the queue """ session = aws.get_session() sqs = session.client('sqs') try: resp = sqs.delete_queue(QueueUrl = queue_arn) except: log.exception("Could not delete status queue '{}'".format(queue_arn))
def delete_queue(queue_arn): """Delete a SQS queue Args: queue_arn (str): The URL of the queue """ session = aws.get_session() sqs = session.client('sqs') try: resp = sqs.delete_queue(QueueUrl=queue_arn) except: log.exception("Could not delete status queue '{}'".format(queue_arn))
def create_queue(queue_name): """Create a SQS queue Args: queue_name (str): Name of the queue Return: str: URL of the queue """ session = aws.get_session() sqs = session.client('sqs') resp = sqs.create_queue(QueueName=queue_name) url = resp['QueueUrl'] return url
def create_queue(queue_name): """Create a SQS queue Args: queue_name (str): Name of the queue Return: str: URL of the queue """ session = aws.get_session() sqs = session.client('sqs') resp = sqs.create_queue(QueueName = queue_name) url = resp['QueueUrl'] return url
def check_queue(queue_arn): """Get the count of messages in the given queue The count is a combination Approximate Number of Messages, Messages Delayed, and Messages Not Visible. If the queue_arn contains 'dlq' the first message will be read and the SNS DLQ message decoded to print the error that caused the DLQ message. Args: queue_arn (str): The URL of the queue Return: int: The count of messages or zero if the queue could not be queried """ session = aws.get_session() sqs = session.client('sqs') try: resp = sqs.get_queue_attributes( QueueUrl=queue_arn, AttributeNames=[ 'ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesDelayed', 'ApproximateNumberOfMessagesNotVisible' ]) except: log.exception( "Could not get message count for queue '{}'".format(queue_arn)) return 0 else: # Include both the number of messages and the number of in-flight messages message_count = int(resp['Attributes']['ApproximateNumberOfMessages']) + \ int(resp['Attributes']['ApproximateNumberOfMessagesDelayed']) + \ int(resp['Attributes']['ApproximateNumberOfMessagesNotVisible']) if message_count > 0 and 'dlq' in queue_arn: try: resp = sqs.receive_message(QueueUrl=queue_arn) for msg in resp['Messages']: body = json.loads(msg['Body']) error = body['Records'][0]['Sns']['MessageAttributes'][ 'ErrorMessage']['Value'] log.debug("DLQ Error: {}".format(error)) except Exception as err: log.exception( "Problem getting DLQ error message: {}".format(err)) return message_count
def check_downsample_queue(args): """ Check queue for downsample jobs. Also marks downsample as in progress if message found. Args: (dict): { 'queue_url': <URL of SQS queue>, 'sfn_arn': <arn of the downsample step fcn> } Returns: (dict): { 'start_downsample': True | False, 'queue_url': <URL of SQS queue>, 'sfn_arn': <arn of the downsample step fcn>, 'status': 'IN_PROGRESS', 'db_host': <host name of database> 'channel_id': <id of channel for downsample> ... } if start_downsample, then args for downsample_channel() provided including job_receipt_handle so message's visibility timeout can be adjusted or be deleted from the queue. The message's contents are provided in 'msg', if one is available. """ session = aws.get_session() sqs = session.client('sqs') resp = sqs.receive_message(QueueUrl=args['queue_url'], WaitTimeSeconds=2, MaxNumberOfMessages=1) if 'Messages' not in resp or len(resp['Messages']) == 0: return {'start_downsample': False} msg = resp['Messages'][0] job = json.loads(msg['Body']) output = { 'start_downsample': True, 'job_receipt_handle': msg['ReceiptHandle'], 'queue_url': args['queue_url'], 'sfn_arn': args['sfn_arn'], 'status': DownsampleStatus.IN_PROGRESS, 'db_host': job['db_host'], 'channel_id': job['channel_id'], 'msg': job, } return output
def lambda_throttle_count(lambda_arn): """Read the Throttle count for the given Lambda function from Cloud Watch The metric is read for the last minute, which is in the process of being updated so the value read is not the final value that is recorded in Cloud Watch for the given time period. Args: lambda_arn (str): ARN or Name of the lambda to get the metric for Return: float: The Sample Count for the Lambda's Throttle count -1: If there was an error getting the metric """ session = aws.get_session() cw = session.client('cloudwatch') lambda_name = lambda_arn.split(':')[-1] try: end = datetime.now() begin = end - timedelta(minutes=1) resp = cw.get_metric_statistics( Namespace='AWS/Lambda', MetricName='Throttles', # Limit the throttle count to only our target lambda function Dimensions=[{ 'Name': 'FunctionName', 'Value': lambda_name }], StartTime=begin, EndTime=end, Period=60, Unit='Count', Statistics=['SampleCount']) if 'Datapoints' in resp and len(resp['Datapoints']) > 0: if 'SampleCount' in resp['Datapoints'][0]: return resp['Datapoints'][0]['SampleCount'] return 0.0 except Exception as err: log.exception("Problem getting Lambda Throttle Count: {}".format(err)) return -1
def check_queue(queue_arn): """Get the count of messages in the given queue The count is a combination Approximate Number of Messages, Messages Delayed, and Messages Not Visible. If the queue_arn contains 'dlq' the first message will be read and the SNS DLQ message decoded to print the error that caused the DLQ message. Args: queue_arn (str): The URL of the queue Return: int: The count of messages or zero if the queue could not be queried """ session = aws.get_session() sqs = session.client('sqs') try: resp = sqs.get_queue_attributes(QueueUrl = queue_arn, AttributeNames = ['ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesDelayed', 'ApproximateNumberOfMessagesNotVisible']) except: log.exception("Could not get message count for queue '{}'".format(queue_arn)) return 0 else: # Include both the number of messages and the number of in-flight messages message_count = int(resp['Attributes']['ApproximateNumberOfMessages']) + \ int(resp['Attributes']['ApproximateNumberOfMessagesDelayed']) + \ int(resp['Attributes']['ApproximateNumberOfMessagesNotVisible']) if message_count > 0 and 'dlq' in queue_arn: try: resp = sqs.receive_message(QueueUrl = queue_arn) for msg in resp['Messages']: body = json.loads(msg['Body']) error = body['Records'][0]['Sns']['MessageAttributes']['ErrorMessage']['Value'] log.debug("DLQ Error: {}".format(error)) except Exception as err: log.exception("Problem getting DLQ error message: {}".format(err)) return message_count
def lambda_throttle_count(lambda_arn): """Read the Throttle count for the given Lambda function from Cloud Watch The metric is read for the last minute, which is in the process of being updated so the value read is not the final value that is recorded in Cloud Watch for the given time period. Args: lambda_arn (str): ARN or Name of the lambda to get the metric for Return: float: The Sample Count for the Lambda's Throttle count -1: If there was an error getting the metric """ session = aws.get_session() cw = session.client('cloudwatch') lambda_name = lambda_arn.split(':')[-1] try: end = datetime.now() begin = end - timedelta(minutes=1) resp = cw.get_metric_statistics(Namespace = 'AWS/Lambda', MetricName = 'Throttles', # Limit the throttle count to only our target lambda function Dimensions = [{'Name': 'FunctionName', 'Value': lambda_name}], StartTime = begin, EndTime = end, Period = 60, Unit = 'Count', Statistics = ['SampleCount']) if 'Datapoints' in resp and len(resp['Datapoints']) > 0: if 'SampleCount' in resp['Datapoints'][0]: return resp['Datapoints'][0]['SampleCount'] return 0.0 except Exception as err: log.exception("Problem getting Lambda Throttle Count: {}".format(err)) return -1
def delete_downsample_job(args): """ Delete the message for the finished downsample job from SQS. Returns a dict that allows the last state to start a new instance of the downsample step function. Args: args (dict): { 'sfn_arn': ARN of this step function,, 'queue_url': <URL of SQS queue>, 'job_receipt_handle': <msg's receipt handle, 'lookup_key': <full lookup key of channel>, ... } Returns: (dict): { 'queue_url': <URL of SQS queue>, 'sfn_arn': <arn of the downsample step fcn>, 'lookup_key': <full lookup key of channel>, } """ try: session = aws.get_session() sqs = session.client('sqs') sqs.delete_message(QueueUrl=args['queue_url'], ReceiptHandle=args['job_receipt_handle']) log.info( f"Deleting SQS message for downsample of {args['lookup_key']}") return { 'sfn_arn': args['sfn_arn'], 'queue_url': args['queue_url'], 'lookup_key': args['lookup_key'], } except Exception as ex: log.exception(f'Error trying to downsample job from SQS: {ex}') raise
def start(request, resource): """Main code to start a downsample Args: request: DRF Request object resource (BossResourceDjango): The channel to downsample Returns: (HttpResponse) """ channel = resource.get_channel() chan_status = channel.downsample_status.upper() if chan_status == Channel.DownsampleStatus.IN_PROGRESS: return BossHTTPError( "Channel is currently being downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) elif chan_status == Channel.DownsampleStatus.QUEUED: return BossHTTPError( "Channel is already waiting to be downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) elif chan_status == Channel.DownsampleStatus.DOWNSAMPLED and not request.user.is_staff: return BossHTTPError( "Channel is already downsampled. Invalid Request.", ErrorCodes.INVALID_STATE) if request.user.is_staff: # DP HACK: allow admin users to override the coordinate frame frame = request.data else: frame = {} boss_config = BossConfig() collection = resource.get_collection() experiment = resource.get_experiment() coord_frame = resource.get_coord_frame() lookup_key = resource.get_lookup_key() col_id, exp_id, ch_id = lookup_key.split("&") def get_frame(idx): return int(frame.get(idx, getattr(coord_frame, idx))) downsample_sfn = boss_config['sfn']['downsample_sfn'] db_host = boss_config['aws']['db'] args = { 'lookup_key': lookup_key, 'collection_id': int(col_id), 'experiment_id': int(exp_id), 'channel_id': int(ch_id), 'annotation_channel': not channel.is_image(), 'data_type': resource.get_data_type(), 's3_bucket': boss_config["aws"]["cuboid_bucket"], 's3_index': boss_config["aws"]["s3-index-table"], 'x_start': get_frame('x_start'), 'y_start': get_frame('y_start'), 'z_start': get_frame('z_start'), 'x_stop': get_frame('x_stop'), 'y_stop': get_frame('y_stop'), 'z_stop': get_frame('z_stop'), 'resolution': int(channel.base_resolution), 'resolution_max': int(experiment.num_hierarchy_levels), 'res_lt_max': int(channel.base_resolution) + 1 < int( experiment.num_hierarchy_levels), 'type': experiment.hierarchy_method, 'iso_resolution': int(resource.get_isotropic_level()), # This step function executes: boss-tools/activities/resolution_hierarchy.py 'downsample_volume_lambda': boss_config['lambda']['downsample_volume'], # Need to pass step function's ARN to itself, so it can start another # instance of itself after finishing a downsample. 'sfn_arn': downsample_sfn, 'db_host': db_host, 'aws_region': get_region(), } # Check that only administrators are triggering extra large downsamples if ((not request.user.is_staff) and ((args['x_stop'] - args['x_start']) *\ (args['y_stop'] - args['y_start']) *\ (args['z_stop'] - args['z_start']) > settings.DOWNSAMPLE_MAX_SIZE)): return BossHTTPError( "Large downsamples require admin permissions to trigger. Invalid Request.", ErrorCodes.INVALID_STATE) session = get_session() downsample_sqs = boss_config['aws']['downsample-queue'] try: enqueue_job(session, args, downsample_sqs) except BossError as be: return BossHTTPError(be.message, be.error_code) compute_usage_metrics(session, args, boss_config['system']['fqdn'], request.user.username or "public", collection.name, experiment.name, channel.name) region = get_region() account_id = get_account_id() downsample_sfn_arn = f'arn:aws:states:{region}:{account_id}:stateMachine:{downsample_sfn}' if not check_for_running_sfn(session, downsample_sfn_arn): bossutils.aws.sfn_run(session, downsample_sfn_arn, { 'queue_url': downsample_sqs, 'sfn_arn': downsample_sfn_arn, }) return HttpResponse(status=201)
def downsample_channel(args): """ Slice the given channel into chunks of 2x2x2 or 2x2x1 cubes that are then sent to the downsample_volume lambda for downsampling into a 1x1x1 cube at resolution + 1. Makes use of the bossutils.multidimensional library for simplified vector math. Args: args { downsample_volume_sfn (ARN) collection_id (int) experiment_id (int) channel_id (int) annotation_channel (bool) data_type (str) 'uint8' | 'uint16' | 'uint64' s3_bucket (URL) s3_index (URL) id_index (URL) x_start (int) y_start (int) z_start (int) x_stop (int) y_stop (int) z_stop (int) resolution (int) The resolution to downsample. Creates resolution + 1 resolution_max (int) The maximum resolution to generate res_lt_max (bool) = args['resolution'] < (args['resolution_max'] - 1) annotation_index_max (int) The maximum resolution to index annotation channel cubes at When annotation_index_max = N, indices will exist for res 0 - (N - 1) type (str) 'isotropic' | 'anisotropic' iso_resolution (int) if resolution >= iso_resolution && type == 'anisotropic' downsample both } """ #log.debug("Downsampling resolution " + str(args['resolution'])) resolution = args['resolution'] dim = XYZ(*CUBOIDSIZE[resolution]) #log.debug("Cube dimensions: {}".format(dim)) def frame(key): return XYZ(args[key.format('x')], args[key.format('y')], args[key.format('z')]) # Figure out variables for isotropic, anisotropic, or isotropic and anisotropic # downsampling. If both are happening, fanout one and then the other in series. configs = [] if args['type'] == 'isotropic': configs.append({ 'name': 'isotropic', 'step': XYZ(2,2,2), 'iso_flag': False, 'frame_start_key': '{}_start', 'frame_stop_key': '{}_stop', }) else: configs.append({ 'name': 'anisotropic', 'step': XYZ(2,2,1), 'iso_flag': False, 'frame_start_key': '{}_start', 'frame_stop_key': '{}_stop', }) if resolution >= args['iso_resolution']: # DP TODO: Figure out how to launch aniso iso version with mutating arguments configs.append({ 'name': 'isotropic', 'step': XYZ(2,2,2), 'iso_flag': True, 'frame_start_key': 'iso_{}_start', 'frame_stop_key': 'iso_{}_stop', }) for config in configs: frame_start = frame(config['frame_start_key']) frame_stop = frame(config['frame_stop_key']) step = config['step'] use_iso_flag = config['iso_flag'] # If the resulting cube should be marked with the ISO flag index_annotations = args['resolution'] < (args['annotation_index_max'] - 1) # Round to the furthest full cube from the center of the data cubes_start = frame_start // dim cubes_stop = ceildiv(frame_stop, dim) log.debug('Downsampling {} resolution {}'.format(config['name'], resolution)) log.debug("Frame corner: {}".format(frame_start)) log.debug("Frame extent: {}".format(frame_stop)) log.debug("Cubes corner: {}".format(cubes_start)) log.debug("Cubes extent: {}".format(cubes_stop)) log.debug("Downsample step: {}".format(step)) log.debug("Indexing Annotations: {}".format(index_annotations)) # Call the downsample_volume lambda to process the data fanout(aws.get_session(), args['downsample_volume_sfn'], make_args(args, cubes_start, cubes_stop, step, dim, use_iso_flag, index_annotations), max_concurrent = MAX_NUM_PROCESSES, rampup_delay = RAMPUP_DELAY, rampup_backoff = RAMPUP_BACKOFF, poll_delay = POLL_DELAY, status_delay = STATUS_DELAY) # Resize the coordinate frame extents as the data shrinks # DP NOTE: doesn't currently work correctly with non-zero frame starts def resize(var, size): start = config['frame_start_key'].format(var) stop = config['frame_stop_key'].format(var) args[start] //= size args[stop] = ceildiv(args[stop], size) resize('x', step.x) resize('y', step.y) resize('z', step.z) # if next iteration will split into aniso and iso downsampling, copy the coordinate frame if args['type'] != 'isotropic' and (resolution + 1) == args['iso_resolution']: def copy(var): args['iso_{}_start'.format(var)] = args['{}_start'.format(var)] args['iso_{}_stop'.format(var)] = args['{}_stop'.format(var)] copy('x') copy('y') copy('z') # Advance the loop and recalculate the conditional # Using max - 1 because resolution_max should not be a valid resolution # and res < res_max will end with res = res_max - 1, which generates res_max resolution args['resolution'] = resolution + 1 args['res_lt_max'] = args['resolution'] < (args['resolution_max'] - 1) return args
def ingest_populate(args): """Populate the ingest upload SQS Queue with tile information Note: This activity will clear the upload queue of any existing messages Args: args: { 'upload_sfn': ARN, 'job_id': '', 'upload_queue': ARN, 'ingest_queue': ARN, 'ingest_type': int (0 == TILE, 1 == VOLUMETRIC), 'resolution': 0, 'project_info': [col_id, exp_id, ch_id], 't_start': 0, 't_stop': 0, 't_tile_size': 0, 'x_start': 0, 'x_stop': 0, 'x_tile_size': 0, 'y_start': 0, 'y_stop': 0 'y_tile_size': 0, 'z_start': 0, 'z_stop': 0 'z_tile_size': 1, 'z_chunk_size': 16 for Tile or Probably 64 for Volumetric } Returns: {'arn': Upload queue ARN, 'count': Number of messages put into the queue} """ log.debug("Starting to populate upload queue") args['MAX_NUM_ITEMS_PER_LAMBDA'] = MAX_NUM_ITEMS_PER_LAMBDA if (args["ingest_type"] != 0) and (args["ingest_type"] != 1): raise ValueError("{}".format("Unknown ingest_type: {}".format(args["ingest_type"]))) clear_queue(args['upload_queue']) results = fanout(aws.get_session(), args['upload_sfn'], split_args(args), max_concurrent=MAX_NUM_PROCESSES, rampup_delay=RAMPUP_DELAY, rampup_backoff=RAMPUP_BACKOFF, poll_delay=POLL_DELAY, status_delay=STATUS_DELAY) # At least one None values in the return of fanout. This avoids an exception in those cases. if results is None: messages_uploaded = 0 else: messages_uploaded = sum(filter(None, results)) if args["ingest_type"] == 0: tile_count = get_tile_count(args) if tile_count != messages_uploaded: log.warning("Messages uploaded do not match tile count. tile count: {} messages uploaded: {}" .format(tile_count, messages_uploaded)) else: log.debug("tile count and messages uploaded match: {}".format(tile_count)) return { 'arn': args['upload_queue'], 'count': tile_count, } elif args["ingest_type"] == 1: vol_count = get_volumetric_count(args) if vol_count != messages_uploaded: log.warning("Messages uploaded do not match volumetric count. volumetric count: {} messages uploaded: {}" .format(vol_count, messages_uploaded)) else: log.debug("volumetric count and messages uploaded match: {}".format(vol_count)) return { 'arn': args['upload_queue'], 'count': vol_count, }
def ingest_populate(args): """Populate the ingest upload SQS Queue with tile information Note: This activity will clear the upload queue of any existing messages Args: args: { 'upload_sfn': ARN, 'job_id': '', 'upload_queue': ARN, 'ingest_queue': ARN, 'ingest_type': int (0 == TILE, 1 == VOLUMETRIC), 'resolution': 0, 'project_info': [col_id, exp_id, ch_id], 't_start': 0, 't_stop': 0, 't_tile_size': 0, 'x_start': 0, 'x_stop': 0, 'x_tile_size': 0, 'y_start': 0, 'y_stop': 0 'y_tile_size': 0, 'z_start': 0, 'z_stop': 0 'z_tile_size': 1, 'z_chunk_size': 16 for Tile or Probably 64 for Volumetric } Returns: {'arn': Upload queue ARN, 'count': Number of messages put into the queue} """ log.debug("Starting to populate upload queue") args['MAX_NUM_ITEMS_PER_LAMBDA'] = MAX_NUM_ITEMS_PER_LAMBDA if (args["ingest_type"] != 0) and (args["ingest_type"] != 1): raise ValueError("{}".format("Unknown ingest_type: {}".format(args["ingest_type"]))) clear_queue(args['upload_queue']) results = fanout(aws.get_session(), args['upload_sfn'], split_args(args), max_concurrent=MAX_NUM_PROCESSES, rampup_delay=RAMPUP_DELAY, rampup_backoff=RAMPUP_BACKOFF, poll_delay=POLL_DELAY, status_delay=STATUS_DELAY) # At least one None values in the return of fanout. This avoids an exception in those cases. if results is None: messages_uploaded = 0 else: messages_uploaded = sum(filter(None, results)) if args["ingest_type"] == 0: tile_count = get_tile_count(args) if tile_count != messages_uploaded: log.warning("Messages uploaded do not match tile count. tile count: {} messages uploaded: {}" .format(tile_count, messages_uploaded)) else: log.debug("tile count and messages uploaded match: {}".format(tile_count)) return { 'arn': args['upload_queue'], 'count': tile_count, } elif args["ingest_type"] == 1: vol_count = get_volumetric_count(args) if vol_count != messages_uploaded: log.warning("Messages uploaded do not match volumetric count. volumetric count: {} messages uploaded: {}" .format(vol_count, messages_uploaded)) else: log.debug("volumetric count and messages uploaded match: {}".format(vol_count)) return { 'arn': args['upload_queue'], 'count': vol_count, }