def deployment_stack_deleted(self, deployment_name, deployment_stack_id, deployment_access_stack_id): self._output_message( '\n{} deployment stack {} and access stack {} have been deleted.'. format(deployment_name, util.get_stack_name_from_arn(deployment_stack_id), util.get_stack_name_from_arn(deployment_access_stack_id)))
def __get_access_stack_parameters(context, deployment_name, deployment_stack_id = None, uploader = None): return { 'ConfigurationBucket': uploader.bucket if uploader else None, 'ConfigurationKey': uploader.key if uploader else None, 'ProjectResourceHandler': context.config.project_resource_handler_id, 'PlayerAccessTokenExchange': context.config.token_exchange_handler_id, 'ProjectStack': util.get_stack_name_from_arn(context.config.project_stack_id), 'DeploymentName': deployment_name, 'DeploymentStack': util.get_stack_name_from_arn(deployment_stack_id) if deployment_stack_id else None, 'DeploymentStackArn': deployment_stack_id }
def delete(self, stack_id, pending_resource_status=None): stack_name = util.get_stack_name_from_arn(stack_id) self.context.view.deleting_stack(stack_name, stack_id) self.__clean_undeltable_resources( stack_id, pending_resource_status=pending_resource_status) monitor = Monitor(self.context, stack_id, 'DELETE') cf = self.context.aws.client('cloudformation', region=util.get_region_from_arn(stack_id)) try: res = cf.delete_stack(StackName=stack_id) except ClientError as e: raise HandledError( 'Could not start delete of {} stack ({}).'.format( stack_name, stack_id), e) monitor.wait() self.__clean_log_groups( stack_id, pending_resource_status=pending_resource_status)
def __delete_custom_resource_lambdas(context, args): context.view.deleting_custom_resource_lambdas() stack_id = context.config.project_stack_id project_name = util.get_stack_name_from_arn(stack_id) region = util.get_region_from_arn(stack_id) lambda_client = context.aws.client('lambda', region=region) iam_client = context.aws.client('iam') delete_functions = [] delete_roles = [] prefixes = [ "{}-{}-".format(project_name, prefix) for prefix in resource_type_info.LAMBDA_TAGS ] # Iterate through all lambda functions and generate a list that begin with any of the prefixes associated with # custom resource handlers for response in lambda_client.get_paginator('list_functions').paginate(): for entry in response['Functions']: function_name = entry['FunctionName'] if any(function_name.startswith(prefix) for prefix in prefixes): delete_functions.append(function_name) delete_roles.append( aws_utils.get_role_name_from_role_arn(entry['Role'])) # Delete the functions and roles related to custom resource handlers for function_name, role_name in zip(delete_functions, delete_roles): lambda_client.delete_function(FunctionName=function_name) iam_client.delete_role_policy(RoleName=role_name, PolicyName="Default") iam_client.delete_role(RoleName=role_name) context.view.deleting_lambdas_completed(len(delete_functions))
def handler(event, context): print "Start FIFO Auto Scale" prefix = util.get_stack_name_from_arn(event[c.ENV_STACK_ID], False) request_type = event['RequestType'] assigned_suffix = event['ResourceProperties'].get('Suffix', None) type = event['ResourceProperties'].get('QueueType', "fifo") initial_number_of_queues = int(event['ResourceProperties'].get( 'IntialNumberOfQueues', 5)) if assigned_suffix: prefix = "{0}{1}".format(prefix, assigned_suffix) sqs = Sqs({}, queue_prefix=prefix, type=type) if request_type == 'Delete': sqs.delete_all_queues(prefix) else: queues = sqs.get_queues() number_of_queues = len(queues) #5 queues to start, each queue can support 300 send message calls per second. total: 1500 messages per second if number_of_queues < initial_number_of_queues: for i in range(number_of_queues, initial_number_of_queues): sqs.add_fifo_queue(prefix) return custom_resource_response.success_response({}, "*")
def __add_to_sqs(self, files): prefix = util.get_stack_name_from_arn( os.environ[c.ENV_DEPLOYMENT_STACK_ARN]) sqs = Sqs(self.context, "{0}{1}".format(prefix, c.KEY_SQS_AMOEBA_SUFFIX)) sqs.set_queue_url(lowest_load_queue=False) sqs.send_generic_message(json.dumps({"paths": files}))
def main(event, lambdacontext): starttime = time.time() queue_url = event.get(c.KEY_SQS_QUEUE_URL, None) print "Started consumer with queue url '{}'".format(queue_url) context = event.get("context", {}) context[c.KEY_SQS_QUEUE_URL] = queue_url context[c.KEY_LAMBDA_FUNCTION] = lambdacontext.function_name if hasattr(lambdacontext, 'function_name') else None context[c.KEY_REQUEST_ID] = lambdacontext.aws_request_id if hasattr(lambdacontext, 'aws_request_id') else None context[c.KEY_IS_LAMBDA_ENV] = context[c.KEY_REQUEST_ID] is not None prefix = util.get_stack_name_from_arn(os.environ[c.ENV_DEPLOYMENT_STACK_ARN]) context[c.KEY_STACK_PREFIX] = prefix context[c.KEY_SQS] = Sqs(context, "{0}_".format(prefix)) context[c.KEY_SQS_AMOEBA] = Sqs(context, "{0}{1}_".format(prefix, c.KEY_SQS_AMOEBA_SUFFIX)) context[c.KEY_SQS_AMOEBA].set_queue_url(lowest_load_queue=True) context[c.KEY_LAMBDA] = Lambda(context) context[c.KEY_CLOUDWATCH] = CloudWatch(context) context[c.KEY_THREAD_POOL] = ThreadPool(context, 8) context[c.KEY_METRIC_BUCKET] = os.environ[c.RES_S3_STORAGE] context[c.KEY_START_TIME] = starttime context[c.CW_ATTR_SAVE_DURATION] = context[c.KEY_CLOUDWATCH].avg_save_duration(util.get_cloudwatch_namespace(os.environ[c.ENV_DEPLOYMENT_STACK_ARN])) context[c.CW_ATTR_DELETE_DURATION] = context[c.KEY_CLOUDWATCH].avg_delete_duration(util.get_cloudwatch_namespace(os.environ[c.ENV_DEPLOYMENT_STACK_ARN])) context[c.KEY_SUCCEEDED_MSG_IDS] = [] process(context) del context gc.collect() return { 'StatusCode': 200 }
def delete(self, stack_id, pending_resource_status = None): stack_name = util.get_stack_name_from_arn(stack_id) self.context.view.deleting_stack(stack_name, stack_id) self.__clean_undeltable_resources(stack_id, pending_resource_status = pending_resource_status) monitor = Monitor(self.context, stack_id, 'DELETE') cf = self.context.aws.client('cloudformation', region=util.get_region_from_arn(stack_id)) failed_resources=[] attempts =0; while attempts < 5: try: res = cf.delete_stack(StackName = stack_id, RetainResources=list(failed_resources)) except ClientError as e: raise HandledError('Could not start delete of {} stack ({}).'.format(stack_name, stack_id), e) failed_resources = monitor.wait() if len(failed_resources)==0: break attempts+=1 self.__clean_log_groups(stack_id, pending_resource_status = pending_resource_status)
def update(self, stack_id, template_url, parameters={}, pending_resource_status={}, capabilities = []): stack_name = util.get_stack_name_from_arn(stack_id) self.context.view.updating_stack(stack_name, template_url, parameters) if pending_resource_status is not None: self.__clean_undeltable_resources(stack_id, pending_resource_status=pending_resource_status) monitor = Monitor(self.context, stack_id, 'UPDATE') cf = self.context.aws.client('cloudformation', region=util.get_region_from_arn(stack_id)) parameter_list = [ { 'ParameterKey': k, 'ParameterValue': v } for k, v in parameters.iteritems() ] try: res = cf.update_stack( StackName = stack_id, TemplateURL = template_url, Capabilities = capabilities, Parameters = parameter_list ) except ClientError as e: raise HandledError('Could not start update of {} stack ({}).'.format(stack_name, stack_id), e) monitor.wait() self.__clean_log_groups(stack_id, pending_resource_status = pending_resource_status)
def describe_resources( self, stack_id, recursive=True, optional=False ): region = util.get_region_from_arn(stack_id) cf = self.context.aws.client('cloudformation', region=region) self.context.view.describing_stack_resources(stack_id) try: res = cf.describe_stack_resources(StackName=stack_id) except ClientError as e: if optional and e.response['Error']['Code'] == 'ValidationError': return {} message = e.message if e.response['Error']['Code'] == 'ValidationError': message += ' Make sure the AWS credentials you are using have access to the project\'s resources.' raise HandledError('Could not get stack {} resource data. {}'.format( util.get_stack_name_from_arn(stack_id), message)) resource_descriptions = {} for entry in res['StackResources']: resource_descriptions[entry['LogicalResourceId']] = entry if recursive and entry['ResourceType'] == 'AWS::CloudFormation::Stack': physical_resource_id = entry.get('PhysicalResourceId', None) if physical_resource_id is not None: logical_resource_id = entry['LogicalResourceId'] nested_map = self.describe_resources(physical_resource_id) for k,v in nested_map.iteritems(): resource_descriptions[entry['LogicalResourceId'] + '.' + k] = v elif entry['ResourceType'] == 'Custom::CognitoUserPool': # User Pools require extra information (client id/secret) resource_descriptions[entry['LogicalResourceId']]['UserPoolClients'] = [] idp = self.context.aws.client('cognito-idp', region=region) pool_id = entry.get('PhysicalResourceId', None) # Lookup client IDs if the pool ID is valid. Valid pool ids must contain an underscore. # CloudFormation initializes the physical ID to a UUID without an underscore before the resource is created. # If the pool creation doesn't happen or it fails, the physical ID isn't updated to a valid value. if pool_id is not None and pool_id.find('_') >= 0: try: client_list = idp.list_user_pool_clients(UserPoolId=pool_id, MaxResults=60)['UserPoolClients'] except ClientError as e: client_list = {} if e.response['Error']['Code'] == 'ResourceNotFoundException': continue collected_details = {} for client in client_list: client_name = client['ClientName'] client_id = client['ClientId'] client_description = idp.describe_user_pool_client(UserPoolId=pool_id, ClientId=client_id)['UserPoolClient'] collected_details[client_name] = { 'ClientId': client_id } resource_descriptions[entry['LogicalResourceId']]['UserPoolClients'] = collected_details return resource_descriptions
def delete_stack(context, args): if context.config.project_stack_id is None: raise HandledError("Project stack does not exist.") if context.config.deployment_names: raise HandledError( 'The project has {} deployment stack(s): {}. You must delete these stacks before you can delete the project stack.' .format( len(context.config.deployment_names), ', '.join( deployment_name for deployment_name in context.config.deployment_names))) if context.stack.id_exists(context.config.project_stack_id): logs_bucket_id = context.stack.get_physical_resource_id( context.config.project_stack_id, 'Logs', optional=True, expected_type='AWS::S3::Bucket') pending_resource_status = __get_pending_resource_status(context, deleting=True) context.stack.confirm_stack_operation(context.config.project_stack_id, 'project stack', args, pending_resource_status) context.stack.delete(context.config.project_stack_id, pending_resource_status=pending_resource_status) if logs_bucket_id: s3 = context.aws.client('s3') # Check to see if the bucket still exists, old versions of project-template.json # don't have DeletionPolicy="Retain" on this bucket. try: s3.head_bucket(Bucket=logs_bucket_id) bucket_still_exists = True except: bucket_still_exists = False if bucket_still_exists: stack_name = util.get_stack_name_from_arn( context.config.project_stack_id) util.delete_bucket_contents(context, stack_name, "Logs", logs_bucket_id) context.view.deleting_bucket(logs_bucket_id) s3.delete_bucket(Bucket=logs_bucket_id) else: context.view.clearing_project_stack_id(context.config.project_stack_id) context.config.clear_project_stack_id()
def main(event, lambdacontext): global context global timestamp global aws_sqs start = time.time() ok_response = { 'StatusCode': 200, } refreshtime = datetime.datetime.utcnow() - datetime.timedelta(minutes=1) if context is None or aws_sqs is None or refreshtime > timestamp: context=dict({}) stack_id = os.environ[c.ENV_DEPLOYMENT_STACK_ARN] context[c.KEY_REQUEST_ID] = lambdacontext.aws_request_id if hasattr(lambdacontext, 'aws_request_id') else None db = DynamoDb(context) prefix = util.get_stack_name_from_arn(stack_id) aws_sqs = Sqs(context, queue_prefix="{}_".format(prefix)) aws_sqs.set_queue_url(True) timestamp = datetime.datetime.utcnow() else: context[c.KEY_SQS_QUEUE_URL] = aws_sqs.queue_url data = event.get(c.API_PARAM_PAYLOAD, {})[c.API_PARAM_DATA] source_IP = event.get(c.API_PARAM_SOURCE_IP, None) sensitivity_type = event.get(c.SQS_PARAM_SENSITIVITY_TYPE, sensitivity.SENSITIVITY_TYPE.NONE) compression_mode = event.get(c.SQS_PARAM_COMPRESSION_TYPE, compression.COMPRESSION_MODE.NONE) payload_type = event.get(c.SQS_PARAM_PAYLOAD_TYPE, payload.PAYLOAD_TYPE.CSV) compression_mode = CompressionClassFactory.instance(compression_mode) sensitivity_type = SensitivityClassFactory.instance(sensitivity_type) payload_type = PayloadClassFactory.instance(context, payload_type, compression_mode, sensitivity_type, source_IP) print "[{}]Using SQS queue URL '{}'".format(context[c.KEY_REQUEST_ID],aws_sqs.queue_url) if os.environ[c.ENV_VERBOSE]== "True": print "The post request contains a paylod of\n{}".format(data) if data is None: print "Terminating, there is no data." return ok_response total_metrics = "all" try: data_size = len(data) + sqs.message_overhead_size(sensitivity_type, compression_mode, payload_type) message_chunks, total_metrics = payload_type.chunk(data) for message in message_chunks: print "Sending a sqs message with {} bytes".format(len(message)) aws_sqs.send_message(message, sensitivity_type, compression_mode, payload_type) except Exception as e: traceback.print_exc() raise errors.ClientError(e.message) print "The job sent {} metric(s) to the FIFO queue '{}'".format(total_metrics, aws_sqs.queue_url) print "The job took {} seconds.".format(time.time() -start) return ok_response
def stack_changes(self, stack_id, stack_description, change_map): if stack_id: full_stack_description = '{} stack ({})'.format(stack_description, util.get_stack_name_from_arn(stack_id)) else: full_stack_description = '{} stack'.format(stack_description) self._output_message('\nThis operation will perform the following pending actions on the {} resources in AWS:'.format(full_stack_description)) if change_map: self.__view_resource_list(change_map) else: self._output_message('\n(no changes detected)') print ''
def handler(event, context): print "Start FIFO Auto Scale" prefix = util.get_stack_name_from_arn(event[c.ENV_STACK_ID], False) sqs = Sqs({}, queue_prefix=prefix) request_type = event['RequestType'] print request_type, prefix if request_type == 'Delete': sqs.delete_all_queues(prefix) else: queues = sqs.get_queues() print queues number_of_queues = len(queues) #5 queues to start, each queue can support 300 send message calls per second. total: 1500 messages per second for i in range(number_of_queues, 5): sqs.add_fifo_queue(prefix) return custom_resource_response.success_response({}, "*")
def __init__(self, context, stack_id, operation): self.context = context self.stack_id = stack_id self.stack_name = util.get_stack_name_from_arn(stack_id) self.operation = operation self.events_seen = {} self.success_status = operation + '_COMPLETE' self.finished_status = [ self.success_status, operation + '_FAILED', operation + '_ROLLBACK_COMPLETE', operation + '_ROLLBACK_FAILED', context.stack.STATUS_ROLLBACK_COMPLETE, context.stack.STATUS_ROLLBACK_FAILED ] self.client = self.context.aws.client('cloudformation', region=util.get_region_from_arn( self.stack_id)) self.client.verbose = False self.start_nested_stack_status = [ context.stack.STATUS_UPDATE_IN_PROGRESS, context.stack.STATUS_CREATE_IN_PROGRESS, context.stack.STATUS_DELETE_IN_PROGRESS ] self.end_nested_stack_status = [ context.stack.STATUS_UPDATE_COMPLETE, context.stack.STATUS_UPDATE_FAILED, context.stack.STATUS_CREATE_COMPLETE, context.stack.STATUS_CREATE_FAILED, context.stack.STATUS_DELETE_COMPLETE, context.stack.STATUS_DELETE_FAILED, context.stack.STATUS_ROLLBACK_COMPLETE, context.stack.STATUS_ROLLBACK_FAILED ] self.monitored_stacks = [stack_id] if operation != 'CREATE': self.__load_existing_events()
def main(event, lambdacontext): context = dict({}) stack_id = os.environ[c.ENV_DEPLOYMENT_STACK_ARN] context[c.KEY_LAMBDA_FUNCTION] = lambdacontext.function_name if hasattr( lambdacontext, 'function_name') else None context[c.KEY_REQUEST_ID] = lambdacontext.aws_request_id if hasattr( lambdacontext, 'aws_request_id') else None is_lambda = context[c.KEY_REQUEST_ID] is not None db = DynamoDb(context) if not is_lambda: import lambda_fifo_message_consumer as consumer prefix = util.get_stack_name_from_arn(stack_id) sqs = Sqs(context, "{0}_".format(prefix)) awslambda = Lambda(context) if sqs.is_all_under_load: sqs.add_fifo_queue(prefix) queues = sqs.get_queues() for queue_url in queues: payload = {c.KEY_SQS_QUEUE_URL: queue_url, "context": context} print "Starting {} with queue url '{}'".format( "lambda" if is_lambda else "thread", queue_url) if is_lambda: invoke(context, awslambda, payload) else: payload[c.ENV_STACK_ID] = event['StackId'] consumer.main( payload, type('obj', (object, ), {'function_name': context[c.KEY_LAMBDA_FUNCTION]})) print "{} {} lambdas have started".format(len(queues), context[c.KEY_LAMBDA_FUNCTION]) return custom_resource_response.success_response({}, "*")
def retrieving_mappings(self, deployment_name, deployment_stack_id): self._output_message( 'Loading mappings for deployment {} from stack {}.'.format( deployment_name, util.get_stack_name_from_arn(deployment_stack_id)))
def create_stack(context, args): # Has the project been initialized? if not context.config.project_initialized: raise HandledError('The project has not been initialized.') # Does a deployment with that name already exist? if context.config.deployment_stack_exists(args.deployment): raise HandledError('The project already has a {} deployment.'.format(args.deployment)) # Does deployment-template.json include resource group from a gem which isn't enabled for the project? for resource_group_name in context.resource_groups.keys(): __check_resource_group_gem_status(context, resource_group_name) # Is the project settings file writable? context.config.validate_writable(context.config.local_project_settings_path) # Is the deployment name valid? util.validate_stack_name(args.deployment) # If there is no project default deployment, make this the project default deployment if context.config.project_default_deployment is None: args.make_project_default = True # If there is no release deployment, make this the release deployment if context.config.release_deployment is None: args.make_release_deployment = True # Need to handle situations where the deployment and/or access stack were # not successfully created on previous attempts. pending_deployment_stack_id = context.config.get_pending_deployment_stack_id(args.deployment) pending_deployment_access_stack_id = context.config.get_pending_deployment_access_stack_id(args.deployment) pending_deployment_stack_status = context.stack.get_stack_status(pending_deployment_stack_id) pending_deployment_access_stack_status = context.stack.get_stack_status(pending_deployment_access_stack_id) # Does a stack with the name already exist? It's ok if a previous attempt # at creation left a stack with this name behind, we'll deal with that later. deployment_stack_name = args.stack_name or context.config.get_default_deployment_stack_name(args.deployment) deployment_region = util.get_region_from_arn(context.config.project_stack_id) if pending_deployment_stack_id is None or deployment_stack_name != util.get_stack_name_from_arn(pending_deployment_stack_id): if context.stack.name_exists(deployment_stack_name, deployment_region): raise HandledError('An AWS Cloud Formation stack with the name {} already exists in region {}. Use the --stack-name option to provide a different name.'.format(deployment_stack_name, deployment_region)) # Resource group (and other) file write checks create_and_validate_writable_list(context) # Is it ok to use AWS? pending_resource_status = __get_pending_combined_resource_status(context, args.deployment) capabilities = context.stack.confirm_stack_operation( None, # stack id 'deployment {}'.format(args.deployment), args, pending_resource_status, ignore_resource_types = [ 'Custom::EmptyDeployment' ] ) # We have the following scenerios to deal with: # # 1) This is the first attempt to create the deployment, or previous attempts didn't # get as far as creating any stacks. # # 2) The previous attempt failed to create or update the deployment stack, which was # left in a ROLLBACK_COMPLETED, UPDATE_ROLLBACK_FAILED, or ROLLBACK_FAILED state. This # stack must be deleted and a new one created. # # 3) The previous attempt created the deployment stack but failed to create the access # stack, leaving it in the ROLLBACK_COMPLETED state. In this case we update the deployment # stack (to make sure it reflects any changes that may have been made), delete the access # stack and attempt to create a new one. # # 4) Both the deployment and access stacks were created successfully, but the pending # stack id properites in the config were not replaced with the non-pending properties # (this could happen if someone kills the client during the access stack creation # process, which then runs to a successful completion). In this case we update both # stacks to make sure they reflect any changes, then replace the "pending" stack id # properties. project_uploader = ProjectUploader(context) deployment_uploader = project_uploader.get_deployment_uploader(args.deployment) template_url = before_update(context, deployment_uploader) deployment_stack_parameters = __get_deployment_stack_parameters(context, args.deployment, uploader = deployment_uploader) # wait a bit for S3 to help insure that templates can be read by cloud formation time.sleep(constant.STACK_UPDATE_DELAY_TIME) try: if pending_deployment_stack_status not in [None, context.stack.STATUS_ROLLBACK_COMPLETE, context.stack.STATUS_DELETE_COMPLETE, context.stack.STATUS_UPDATE_ROLLBACK_FAILED, context.stack.STATUS_ROLLBACK_FAILED]: # case 3 or 4 - deployment stack was previously created successfully, update it context.stack.update( pending_deployment_stack_id, template_url, deployment_stack_parameters, capabilities = capabilities ) deployment_stack_id = pending_deployment_stack_id else: if pending_deployment_stack_status in [context.stack.STATUS_ROLLBACK_COMPLETE, context.stack.STATUS_ROLLBACK_FAILED, context.stack.STATUS_UPDATE_ROLLBACK_FAILED]: # case 2 - deployment stack failed to create previously, delete it context.stack.delete(pending_deployment_stack_id) # case 1 and 2 - deployment stack wasn't creatred previously or was just # deleted, attempt to create it deployment_stack_id = context.stack.create_using_url( deployment_stack_name, template_url, deployment_region, deployment_stack_parameters, created_callback=lambda id: context.config.set_pending_deployment_stack_id(args.deployment, id), capabilities = capabilities) # Now create or update the access stack... context.view.processing_template('{} deployment'.format(args.deployment)) access_template_url = deployment_uploader.upload_content( constant.DEPLOYMENT_ACCESS_TEMPLATE_FILENAME, json.dumps(context.config.deployment_access_template_aggregator.effective_template, indent=4, sort_keys=True), 'processed deployment access temmplate') access_stack_parameters = __get_access_stack_parameters( context, args.deployment, deployment_stack_id = deployment_stack_id, uploader = deployment_uploader ) if pending_deployment_access_stack_status not in [None, context.stack.STATUS_ROLLBACK_COMPLETE, context.stack.STATUS_DELETE_COMPLETE]: # case 4 - access stack was previously created successfully but the pending # stack id properties were not replaced. Update the stack. context.stack.update( pending_deployment_access_stack_id, access_template_url, deployment_stack_parameters, capabilities = capabilities ) deployment_access_stack_id = pending_deployment_access_stack_id else: if pending_deployment_access_stack_status == context.stack.STATUS_ROLLBACK_COMPLETE: # case 3 - access stack failed to create previously, delete it context.stack.delete(pending_deployment_access_stack_id) # case 1 or 3 - access stack wasn't created before, or was just deleted. Attempt # to create. deployment_access_stack_name = deployment_stack_name + '-Access' deployment_access_stack_id = context.stack.create_using_url( deployment_access_stack_name, access_template_url, deployment_region, parameters = access_stack_parameters, created_callback=lambda id: context.config.set_pending_deployment_access_stack_id(args.deployment, id), capabilities = capabilities) except: context.config.force_gui_refresh() raise context.config.force_gui_refresh() context.config.finalize_deployment_stack_ids(args.deployment) context.view.deployment_stack_created(args.deployment, deployment_stack_id, deployment_access_stack_id) # Should the new deployment become the project default deployment or the release deployment? if args.make_project_default: context.config.set_project_default_deployment(args.deployment) mappings.update(context, util.Args()) context.view.default_deployment(context.config.user_default_deployment, context.config.project_default_deployment) if args.make_release_deployment: context.config.set_release_deployment(args.deployment) temp_args = util.Args() temp_args.release = True mappings.update(context, temp_args) context.view.release_deployment(context.config.release_deployment) after_update(context, deployment_uploader)
def launch(event, lambdacontext): util.debug_print("Start Amoeba Launcher") context = dict({}) context[c.KEY_START_TIME] = time.time() context[c.KEY_LAMBDA_FUNCTION] = lambdacontext.function_name if hasattr( lambdacontext, 'function_name') else None context[c.KEY_REQUEST_ID] = lambdacontext.aws_request_id if hasattr( lambdacontext, 'aws_request_id') else None prefix = util.get_stack_name_from_arn( os.environ[c.ENV_DEPLOYMENT_STACK_ARN]) prefix = "{0}{1}".format(prefix, c.KEY_SQS_AMOEBA_SUFFIX) db = DynamoDb(context) sqs = Sqs(context, prefix, "sqs") sqs.set_queue_url(lowest_load_queue=False) if sqs.is_all_under_load: sqs.add_fifo_queue(prefix) elapsed = util.elapsed(context) timeout = context[c.KEY_MAX_LAMBDA_TIME] * c.RATIO_OF_MAX_LAMBDA_TIME map = {} queues_checked = 0 number_of_queues = sqs.number_of_queues sqs_delete_tokens = {} while elapsed < timeout and queues_checked < number_of_queues: messages = sqs.read_queue() length = len(messages) if sqs.queue_url not in sqs_delete_tokens: sqs_delete_tokens[sqs.queue_url] = [] if length > 0: for x in range(0, length): message = messages[x] body = json.loads(message["Body"]) paths = body["paths"] msg_token = "{}{}{}".format(message['MessageId'], context[c.KEY_SEPERATOR_CSV], message['ReceiptHandle']) sqs_delete_tokens[sqs.queue_url].append(msg_token) for i in range(0, len(paths)): path = paths[i] parts = path.split(context[c.KEY_SEPERATOR_PARTITION]) filename = parts.pop() directory = context[c.KEY_SEPERATOR_PARTITION].join(parts) if directory not in map: map[directory] = {"paths": [], "size": 0} #lambda payload limit for Event invocation type 131072 sizeof = len(path) + map[directory]["size"] is_invoked = map[directory].get("invoked", False) if sizeof >= c.MAXIMUM_ASYNC_PAYLOAD_SIZE and not is_invoked: invoke_lambda(context, directory, map[directory]["paths"]) map[directory] = { "paths": [], "size": 0, "invoked": True } else: map[directory]["paths"].append(path) map[directory]["size"] = sizeof else: queues_checked += 1 sqs.set_queue_url(lowest_load_queue=False) elapsed = util.elapsed(context) #Invoke a amoeba generator for each S3 leaf node for directory, settings in iteritems(map): is_invoked = settings.get("invoked", False) #Amoeba's are not designed to have multiple amoebas working against one directory #If the Amoeba has already been invoked due to payload size then we requeue the remaining paths if is_invoked: sqs.send_generic_message(json.dumps({"paths": settings["paths"]})) else: invoke_lambda(context, directory, settings["paths"]) context[c.KEY_THREAD_POOL] = ThreadPool(context, 8) #Delete SQS messages that have been processed for key, value in iteritems(sqs_delete_tokens): sqs.delete_message_batch(value, key) return custom_resource_response.success_response({"StatusCode": 200}, "*")
def describing_stack(self, stack_id): if self.__verbose: self._output_message('Retrieving description of stack {}.'.format( util.get_stack_name_from_arn(stack_id)))
def retrieving_mappings(self, deployment_name, deployment_stack_id, role): self._output_message( "Loading mappings for deployment '{}' with role '{}' from stack '{}'." .format(deployment_name, role, util.get_stack_name_from_arn(deployment_stack_id)))
def __remove_bucket_contents(self, stack_id, logical_resource_id): physical_bucket_id = self.get_physical_resource_id( stack_id, logical_resource_id) stack_name = util.get_stack_name_from_arn(stack_id) util.delete_bucket_contents(self.context, stack_name, logical_resource_id, physical_bucket_id)
def get_project_stack_name(self): return util.get_stack_name_from_arn(self.project_stack_id)