def __add_to_sqs(self, files):
prefix = util.get_stack_name_from_arn(
os.environ[c.ENV_DEPLOYMENT_STACK_ARN])
sqs = Sqs(self.context, "{0}{1}".format(prefix,
c.KEY_SQS_AMOEBA_SUFFIX))
sqs.set_queue_url(lowest_load_queue=False)
sqs.send_generic_message(json.dumps({"paths": files}))
def launch(event, lambdacontext):
util.debug_print("Start Amoeba Launcher")
context = dict({})
context[c.KEY_START_TIME] = time.time()
context[c.KEY_LAMBDA_FUNCTION] = lambdacontext.function_name if hasattr(
lambdacontext, 'function_name') else None
context[c.KEY_REQUEST_ID] = lambdacontext.aws_request_id if hasattr(
lambdacontext, 'aws_request_id') else None
prefix = util.get_stack_name_from_arn(
os.environ[c.ENV_DEPLOYMENT_STACK_ARN])
prefix = "{0}{1}".format(prefix, c.KEY_SQS_AMOEBA_SUFFIX)
db = DynamoDb(context)
sqs = Sqs(context, prefix, "sqs")
sqs.set_queue_url(lowest_load_queue=False)
if sqs.is_all_under_load:
sqs.add_fifo_queue(prefix)
elapsed = util.elapsed(context)
timeout = context[c.KEY_MAX_LAMBDA_TIME] * c.RATIO_OF_MAX_LAMBDA_TIME
map = {}
queues_checked = 0
number_of_queues = sqs.number_of_queues
sqs_delete_tokens = {}
while elapsed < timeout and queues_checked < number_of_queues:
messages = sqs.read_queue()
length = len(messages)
if sqs.queue_url not in sqs_delete_tokens:
sqs_delete_tokens[sqs.queue_url] = []
if length > 0:
for x in range(0, length):
message = messages[x]
body = json.loads(message["Body"])
paths = body["paths"]
msg_token = "{}{}{}".format(message['MessageId'],
context[c.KEY_SEPERATOR_CSV],
message['ReceiptHandle'])
sqs_delete_tokens[sqs.queue_url].append(msg_token)
for i in range(0, len(paths)):
path = paths[i]
parts = path.split(context[c.KEY_SEPERATOR_PARTITION])
filename = parts.pop()
directory = context[c.KEY_SEPERATOR_PARTITION].join(parts)
if directory not in map:
map[directory] = {"paths": [], "size": 0}
#lambda payload limit for Event invocation type 131072
sizeof = len(path) + map[directory]["size"]
is_invoked = map[directory].get("invoked", False)
if sizeof >= c.MAXIMUM_ASYNC_PAYLOAD_SIZE and not is_invoked:
invoke_lambda(context, directory,
map[directory]["paths"])
map[directory] = {
"paths": [],
"size": 0,
"invoked": True
}
else:
map[directory]["paths"].append(path)
map[directory]["size"] = sizeof
else:
queues_checked += 1
sqs.set_queue_url(lowest_load_queue=False)
elapsed = util.elapsed(context)
#Invoke a amoeba generator for each S3 leaf node
for directory, settings in iteritems(map):
is_invoked = settings.get("invoked", False)
#Amoeba's are not designed to have multiple amoebas working against one directory
#If the Amoeba has already been invoked due to payload size then we requeue the remaining paths
if is_invoked:
sqs.send_generic_message(json.dumps({"paths": settings["paths"]}))
else:
invoke_lambda(context, directory, settings["paths"])
context[c.KEY_THREAD_POOL] = ThreadPool(context, 8)
#Delete SQS messages that have been processed
for key, value in iteritems(sqs_delete_tokens):
sqs.delete_message_batch(value, key)
return custom_resource_response.success_response({"StatusCode": 200}, "*")