def _create_cloud_watch_rule_from_meta(name, meta, region): # validation depends on rule type required_parameters = ['rule_type'] rule_type = meta.get('rule_type') if rule_type: if rule_type == 'schedule': required_parameters.append('expression') validate_params(name, meta, required_parameters) event_buses = meta.get('event_bus_accounts') response = CONN.cw_events(region).get_rule(name) if response: _LOG.warn('%s rule exists in %s.', name, region) return describe_rule(name=name, meta=meta, region=region, response=response) try: func = RULE_TYPES[rule_type] func(name, meta, CONN.cw_events(region)) if event_buses: time.sleep(5) _attach_tenant_rule_targets(name, region, event_buses) _LOG.info('Created cloud watch rule %s in %s.', name, region) response = CONN.cw_events(region).get_rule(name) time.sleep(5) return describe_rule(name=name, meta=meta, region=region, response=response) except KeyError: raise AssertionError( 'Invalid rule type: {0} for resource {1}. ' 'Please, change rule type with existing: ' 'schedule|ec2|api_call.'.format(rule_type, name))
def _create_api_gateway_from_meta(name, meta): """ Create API Gateway with all specified meta. :type name: str :type meta: dict """ required_parameters = ['resources', 'deploy_stage'] validate_params(name, meta, required_parameters) api_resources = meta['resources'] api_id = _API_GATEWAY_CONN.create_rest_api(name)['id'] if api_resources: args = __prepare_api_resources_args(api_id, api_resources) create_pool(_create_resource_from_metadata, args, 1) else: _LOG.info('There is no resources in %s API Gateway description.', name) # add headers # waiter b4 customization time.sleep(10) _LOG.debug('Customizing API Gateway responses...') # _customize_gateway_responses call is commented due to botocore # InternalFailure while performing the call. will be fixed later #_customize_gateway_responses(api_id) # deploy api __deploy_api_gateway(api_id, meta, api_resources) return describe_api_resources(api_id=api_id, meta=meta, name=name)
def _create_platform_application_from_meta(self, name, meta, region): required_parameters = ['platform', 'attributes'] validate_params(name, meta, required_parameters) arn = self.connection_provider.sns(region).get_platform_application( name) if arn: _LOG.warn( '{0} SNS platform application exists in region {1}.'.format( name, region)) return self.describe_sns_application(name, meta, region, arn) platform = meta['platform'] atrbts = meta['attributes'] try: arn = self.connection_provider.sns( region).create_platform_application( name=name, platform=platform, attributes=atrbts) except ClientError as e: exception_type = e.response['Error']['Code'] if exception_type == 'InvalidParameterException': _LOG.warn('SNS application %s is already exist.', name) else: raise e _LOG.info('SNS platform application %s in region %s has been created.', name, region) return self.describe_sns_application(name, meta, region, arn)
def _enable_autoscaling(autoscaling_config, name): targets = [] policies = [] for item in autoscaling_config: autoscaling_required_parameters = [ 'resource_name', 'dimension', 'min_capacity', 'max_capacity', 'role_name' ] validate_params(name, item, autoscaling_required_parameters) role_name = item['role_name'] role_arn = CONN.iam().check_if_role_exists(role_name) if role_arn: dimension = item['dimension'] resource_id, sc_targets = register_autoscaling_target( dimension, item, role_arn, name) targets.extend(sc_targets) _LOG.debug('Autoscaling %s is set up for %s', dimension, resource_id) autoscaling_policy = item.get('config') if autoscaling_policy: policy_name = autoscaling_policy['policy_name'] _LOG.debug('Going to set up autoscaling with ' 'policy %s', policy_name) sc_policies = put_autoscaling_policy(autoscaling_policy, dimension, policy_name, resource_id) policies.append(sc_policies) _LOG.debug('Policy %s is set up', policy_name) else: _LOG.warn('Role %s is not found, skip autoscaling config', role_name) return {'targets': targets, 'policies': policies}
def _create_api_gateway_from_meta(self, name, meta): """ Create API Gateway with all specified meta. :type name: str :type meta: dict """ required_parameters = ['resources', 'deploy_stage'] validate_params(name, meta, required_parameters) api_resources = meta['resources'] api_item = self.connection.create_rest_api( api_name=name, binary_media_types=meta.get('binary_media_types')) api_id = api_item['id'] # deploy authorizers authorizers = meta.get('authorizers', {}) for key, val in authorizers.items(): lambda_version = val.get('lambda_version') lambda_name = val.get('lambda_name') lambda_alias = val.get('lambda_alias') lambda_arn = self.lambda_res. \ resolve_lambda_arn_by_version_and_alias(lambda_name, lambda_version, lambda_alias) uri = 'arn:aws:apigateway:{0}:lambda:path/2015-03-31/' \ 'functions/{1}/invocations'.format(self.region, lambda_arn) self.connection.create_authorizer(api_id=api_id, name=key, type=val['type'], authorizer_uri=uri, identity_source=val.get( 'identity_source'), ttl=val.get('ttl')) self.lambda_res.add_invocation_permission( statement_id=api_id, name=lambda_arn, principal='apigateway.amazonaws.com') if api_resources: api_resp = meta.get('api_method_responses') api_integration_resp = meta.get('api_method_integration_responses') args = self.__prepare_api_resources_args(api_id, api_resources, api_resp, api_integration_resp) self.create_pool(self._create_resource_from_metadata, args, 1) else: _LOG.info('There is no resources in %s API Gateway description.', name) # add headers # waiter b4 customization time.sleep(10) _LOG.debug('Customizing API Gateway responses...') # _customize_gateway_responses call is commented due to botocore # InternalFailure while performing the call. will be fixed later # _customize_gateway_responses(api_id) # deploy api self.__deploy_api_gateway(api_id, meta, api_resources) return self.describe_api_resources(api_id=api_id, meta=meta, name=name)
def _create_dynamodb_table_from_meta(name, meta): """ Create Dynamo DB table from meta description after parameter validation. :type name: str :type meta: dict """ required_parameters = [ 'hash_key_name', 'hash_key_type', 'read_capacity', 'write_capacity' ] validate_params(name, meta, required_parameters) res = _DYNAMO_DB_CONN.describe_table(name) autoscaling_config = meta.get('autoscaling') if res: _LOG.warn('%s table exists.', name) if autoscaling_config: res['Autoscaling'] = _describe_autoscaling(autoscaling_config, name) return describe_table(name, meta, res) _DYNAMO_DB_CONN.create_table(name, meta['hash_key_name'], meta['hash_key_type'], meta.get('sort_key_name'), meta.get('sort_key_type'), meta['read_capacity'], meta['write_capacity'], global_indexes=meta.get('global_indexes'), local_indexes=meta.get('local_indexes'), wait=False) response = _DYNAMO_DB_CONN.describe_table(name) if not response: raise AssertionError( 'Table with name {0} has not been created!'.format(name)) # enabling stream if present stream_view_type = meta.get('stream_view_type') if stream_view_type: stream = _DYNAMO_DB_CONN.get_table_stream_arn(name) if stream: _LOG.warn('Stream %s exists.', name) else: try: _DYNAMO_DB_CONN.enable_table_stream(name, stream_view_type) except ClientError as e: # handle specific case for fantom stream enabling if 'ResourceInUseException' in str(e): _LOG.warn( 'Stream enabling currently in progress,' ' table: %s', name) else: raise e if autoscaling_config: _LOG.debug('Found autoscaling configuration for resource %s', name) sc_res = _enable_autoscaling(autoscaling_config, name) response['Autoscaling'] = sc_res _LOG.info('Created table %s.', name) return describe_table(name, meta, response)
def _create_sns_topic_trigger_from_meta(lambda_name, lambda_arn, role_name, trigger_meta): required_params = ['target_topic'] validate_params(lambda_name, trigger_meta, required_params) topic_name = trigger_meta['target_topic'] region = trigger_meta.get('region') create_sns_subscription_for_lambda(lambda_arn, topic_name, region) _LOG.info('Lambda %s subscribed to sns topic %s', lambda_name, trigger_meta['target_topic'])
def _create_cloud_watch_trigger_from_meta(topic_name, trigger_meta, region): required_parameters = ['target_rule'] validate_params(topic_name, trigger_meta, required_parameters) rule_name = trigger_meta['target_rule'] topic_arn = CONN.sns(region).get_topic_arn(topic_name) CONN.cw_events(region).add_rule_target(rule_name, topic_arn) CONN.sns(region).allow_service_invoke(topic_arn, 'events.amazonaws.com') _LOG.info('SNS topic %s subscribed to cloudwatch rule %s', topic_name, rule_name)
def _update_lambda(name, meta): _LOG.info('Updating lambda: {0}'.format(name)) req_params = ['runtime', 'memory', 'timeout', 'func_name'] validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] if not _S3_CONN.is_file_exists(CONFIG.deploy_target_bucket, key): raise AssertionError( 'Deployment package {0} does not exist ' 'in {1} bucket'.format(key, CONFIG.deploy_target_bucket)) response = _LAMBDA_CONN.get_function(name) if not response: raise AssertionError('{0} lambda does not exist.'.format(name)) publish_version = meta.get('publish_version', False) _LAMBDA_CONN.update_code_source( lambda_name=name, s3_bucket=CONFIG.deploy_target_bucket, s3_key=key, publish_version=publish_version) # AWS sometimes returns None after function creation, needs for stability time.sleep(10) response = _LAMBDA_CONN.get_function(name) _LOG.debug('Lambda describe result: {0}'.format(response)) code_sha_256 = response['Configuration']['CodeSha256'] publish_ver_response = _LAMBDA_CONN.publish_version( function_name=name, code_sha_256=code_sha_256) updated_version = publish_ver_response['Version'] _LOG.info( 'Version {0} for lambda {1} published'.format(updated_version, name)) alias_name = meta.get('alias') if alias_name: alias = _LAMBDA_CONN.get_alias(function_name=name, name=alias_name) if not alias: _LAMBDA_CONN.create_alias( function_name=name, name=alias_name, version=updated_version) _LOG.info( 'Alias {0} has been created for lambda {1}'.format(alias_name, name)) else: _LAMBDA_CONN.update_alias( function_name=name, alias_name=alias_name, function_version=updated_version) _LOG.info( 'Alias {0} has been updated for lambda {1}'.format(alias_name, name))
def _create_kinesis_stream_trigger_from_meta(lambda_name, lambda_arn, role_name, trigger_meta): required_parameters = ['target_stream', 'batch_size', 'starting_position'] validate_params(lambda_name, trigger_meta, required_parameters) stream_name = trigger_meta['target_stream'] stream = CONN.kinesis().get_stream(stream_name) stream_arn = stream['StreamDescription']['StreamARN'] stream_status = stream['StreamDescription']['StreamStatus'] # additional waiting for stream if stream_status != 'ACTIVE': _LOG.debug('Kinesis stream %s is not in active state,' ' waiting for activation...', stream_name) time.sleep(120) # TODO policy should be moved to meta policy_name = '{0}KinesisTo{1}Lambda'.format(stream_name, lambda_name) policy_document = { "Statement": [ { "Effect": "Allow", "Action": [ "lambda:InvokeFunction" ], "Resource": [ lambda_arn ] }, { "Action": [ "kinesis:DescribeStreams", "kinesis:DescribeStream", "kinesis:ListStreams", "kinesis:GetShardIterator", "Kinesis:GetRecords" ], "Effect": "Allow", "Resource": stream_arn } ], "Version": "2012-10-17" } CONN.iam().attach_inline_policy(role_name=role_name, policy_name=policy_name, policy_document=policy_document) _LOG.debug('Inline policy %s is attached to role %s', policy_name, role_name) _LOG.debug('Waiting for activation policy %s...', policy_name) time.sleep(10) _add_kinesis_event_source(lambda_arn, stream_arn, trigger_meta) _LOG.info('Lambda %s subscribed to kinesis stream %s', lambda_name, stream_name)
def create_lambda_layer_from_meta(self, name, meta, context=None): """ :param name: :param meta: :param context: because of usage in 'update' flow :return: """ req_params = ['runtimes', 'deployment_package'] validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] file_name = key.split('/')[-1] self.s3_conn.download_file(self.deploy_target_bucket, key, file_name) with open(file_name, 'rb') as file_data: file_body = file_data.read() import hashlib hash_object = hashlib.sha256() hash_object.update(file_body) existing_version = self._is_equal_lambda_layer(hash_object.digest(), name) if existing_version: existing_layer_arn = existing_version['LayerVersionArn'] _LOG.info('Layer {} with same content already ' 'exists in layer version {}.'.format( name, existing_layer_arn)) return { existing_layer_arn: build_description_obj(response=existing_version, name=name, meta=meta) } _LOG.debug('Creating lambda layer %s', name) args = { 'layer_name': name, 'runtimes': meta['runtimes'], 's3_bucket': self.deploy_target_bucket, 's3_key': meta[S3_PATH_NAME] } if meta.get('description'): args['description'] = meta['description'] if meta.get('license'): args['layer_license'] = meta['license'] response = self.lambda_conn.create_layer(**args) _LOG.info( 'Lambda Layer {0} version {1} was successfully created'.format( name, response['Version'])) layer_arn = response['LayerArn'] + ':' + str(response['Version']) del response['LayerArn'] return {layer_arn: build_description_obj(response, name, meta)}
def _create_cloud_watch_trigger_from_meta(lambda_name, lambda_arn, role_name, trigger_meta): required_parameters = ['target_rule'] validate_params(lambda_name, trigger_meta, required_parameters) rule_name = trigger_meta['target_rule'] rule_arn = CONN.cw_events().get_rule_arn(rule_name) CONN.cw_events().add_rule_target(rule_name, lambda_arn) _LAMBDA_CONN.add_invocation_permission(lambda_arn, 'events.amazonaws.com', rule_arn) _LOG.info('Lambda %s subscribed to cloudwatch rule %s', lambda_name, rule_name)
def _build_python_artifact(item, project_base_folder, project_path, root, target_folder): _LOG.debug('Building artifact in {0}'.format(target_folder)) lambda_config_dict = json.load(open(build_path(root, item))) req_params = ['lambda_path', 'name', 'version'] validate_params(root, lambda_config_dict, req_params) lambda_path = path_resolver(lambda_config_dict['lambda_path']) lambda_name = lambda_config_dict['name'] lambda_version = lambda_config_dict['version'] artifact_name = lambda_name + '-' + lambda_version # create folder to store artifacts artifact_path = build_path(target_folder, artifact_name) _LOG.debug('Artifacts path: {0}'.format(artifact_path)) os.makedirs(artifact_path) _LOG.debug('Folders are created') # install requirements.txt content # getting file content req_path = build_path(root, REQ_FILE_NAME) if os.path.exists(req_path): _LOG.debug('Going to install 3-rd party dependencies') with open(req_path) as f: req_list = f.readlines() req_list = [path_resolver(r.strip()) for r in req_list] _LOG.debug(str(req_list)) # install dependencies for lib in req_list: command = 'pip install {0} -t {1}'.format(lib, artifact_path) execute_command(command=command) _LOG.debug('3-rd party dependencies were installed successfully') # install local requirements local_req_path = build_path(root, LOCAL_REQ_FILE_NAME) if os.path.exists(local_req_path): _LOG.debug('Going to install local dependencies') _install_local_req(artifact_path, local_req_path, project_base_folder, project_path) _LOG.debug('Local dependencies were installed successfully') src_path = build_path(CONFIG.project_path, project_path, lambda_path) _copy_py_files(src_path, artifact_path) package_name = build_py_package_name(lambda_name, lambda_version) _zip_dir(artifact_path, build_path(target_folder, package_name)) # remove unused folder lock = threading.RLock() lock.acquire() try: shutil.rmtree(artifact_path) finally: lock.release() _LOG.info('Package {0} was created successfully'.format(package_name))
def _build_node_artifact(item, root, target_folder): _check_npm_is_installed() _LOG.debug('Building artifact in {0}'.format(target_folder)) lambda_config_dict = json.load(open(build_path(root, item))) _LOG.debug('Root path: {}'.format(root)) req_params = ['lambda_path', 'name', 'version'] validate_params(root, lambda_config_dict, req_params) lambda_name = lambda_config_dict['name'] lambda_version = lambda_config_dict['version'] artifact_name = lambda_name + '-' + lambda_version # create folder to store artifacts artifact_path = build_path(target_folder, artifact_name) _LOG.debug('Artifacts path: {0}'.format(artifact_path)) if not os.path.exists(artifact_path): os.makedirs(artifact_path) _LOG.debug('Folders are created') # getting file content req_path = build_path(root, NODE_REQ_FILE_NAME) try: if os.path.exists(req_path): command = 'npm install --prefix {0}'.format(root) execute_command(command=command) _LOG.debug('3-rd party dependencies were installed successfully') package_name = build_py_package_name(lambda_name, lambda_version) zip_dir(root, build_path(target_folder, package_name)) lock = threading.RLock() lock.acquire() try: # remove unused folder/files node_modules_path = os.path.join(root, 'node_modules') if os.path.exists(node_modules_path): shutil.rmtree(node_modules_path) # todo Investigate deleting package_lock file # shutil.rmtree(os.path.join(root, 'package_lock.json')) shutil.rmtree(artifact_path) except FileNotFoundError as e: _LOG.exception('Error occurred while temp files removing.') finally: lock.release() return 'Lambda package {0} was created successfully'.format( package_name) except Exception: _LOG.exception( 'Error occurred during the \'{0}\' lambda deployment package ' 'assembling'.format(lambda_name)) return 'Error occurred during the \'{0}\' lambda deployment package ' \ 'assembling'.format(lambda_name)
def _create_cloud_watch_trigger_from_meta(self, name, trigger_meta): required_parameters = ['target_rule', 'input', 'iam_role'] validate_params(name, trigger_meta, required_parameters) rule_name = trigger_meta['target_rule'] input = trigger_meta['input'] sf_role = trigger_meta['iam_role'] sf_arn = self._build_sm_arn(name, self.region) sf_description = self.sf_conn.describe_state_machine(arn=sf_arn) if sf_description.get('status') == 'ACTIVE': sf_role_arn = self.iam_conn.check_if_role_exists(sf_role) if sf_role_arn: self.cw_events_conn.add_rule_sf_target(rule_name, sf_arn, input, sf_role_arn) _LOG.info('State machine %s subscribed to cloudwatch rule %s', name, rule_name)
def _create_dynamodb_trigger_from_meta(lambda_name, lambda_arn, role_name, trigger_meta): required_parameters = ['target_table', 'batch_size'] validate_params(lambda_name, trigger_meta, required_parameters) table_name = trigger_meta['target_table'] if not CONN.dynamodb().is_stream_enabled(table_name): CONN.dynamodb().enable_table_stream(table_name) stream = CONN.dynamodb().get_table_stream_arn(table_name) # TODO support another sub type _LAMBDA_CONN.add_event_source(lambda_arn, stream, trigger_meta['batch_size'], start_position='LATEST') # start_position='LATEST' - in case we did not remove tables before _LOG.info('Lambda %s subscribed to dynamodb table %s', lambda_name, table_name)
def _create_sqs_trigger_from_meta(self, lambda_name, lambda_arn, role_name, trigger_meta): required_parameters = ['target_queue', 'batch_size'] validate_params(lambda_name, trigger_meta, required_parameters) target_queue = trigger_meta['target_queue'] if not self.sqs_conn.get_queue_url(target_queue, self.account_id): _LOG.debug('Queue %s does not exist', target_queue) return queue_arn = 'arn:aws:sqs:{0}:{1}:{2}'.format(self.region, self.account_id, target_queue) self.lambda_conn.add_event_source(lambda_arn, queue_arn, trigger_meta['batch_size']) _LOG.info('Lambda %s subscribed to SQS queue %s', lambda_name, target_queue)
def _create_s3_trigger_from_meta(lambda_name, lambda_arn, role_name, trigger_meta): required_parameters = ['target_bucket', 's3_events'] validate_params(lambda_name, trigger_meta, required_parameters) target_bucket = trigger_meta['target_bucket'] if not _S3_CONN.is_bucket_exists(target_bucket): _LOG.error( 'S3 bucket {0} event source for lambda {1} was not created.'. format(target_bucket, lambda_name)) return _LAMBDA_CONN.add_invocation_permission( lambda_arn, 's3.amazonaws.com', 'arn:aws:s3:::{0}'.format(target_bucket)) _S3_CONN.configure_event_source_for_lambda(target_bucket, lambda_arn, trigger_meta['s3_events']) _LOG.info('Lambda %s subscribed to S3 bucket %s', lambda_name, target_bucket)
def _create_alarm_from_meta(self, name, meta): """ Create alarm resource in AWS Cloud via meta description. :type name: str :type meta: dict """ required_parameters = [ 'metric_name', 'namespace', 'period', 'threshold', 'evaluation_periods', 'comparison_operator', 'statistic' ] validate_params(name, meta, required_parameters) if self.client.is_alarm_exists(name): _LOG.warn('%s alarm exists.', name) return self.describe_alarm(name, meta) params = dict(alarm_name=name, metric_name=meta['metric_name'], namespace=meta['namespace'], period=meta['period'], evaluation_periods=meta['evaluation_periods'], threshold=meta['threshold'], statistic=meta['statistic'], comparison_operator=meta['comparison_operator']) sns_topics = meta.get('sns_topics') sns_topic_arns = [] if sns_topics: for each in sns_topics: arn = self.sns_conn.get_topic_arn(each) sns_topic_arns.append(arn) if sns_topic_arns: params['alarm_actions'] = sns_topic_arns self.client.put_metric_alarm(**params) _LOG.info('Created alarm {0}.'.format(name)) return self.describe_alarm(name, meta)
def _create_lambda_from_meta(self, name, meta): _LOG.debug('Creating lambda %s', name) req_params = [ 'iam_role_name', 'runtime', 'memory', 'timeout', 'func_name' ] # Lambda configuration validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] if not self.s3_conn.is_file_exists(self.deploy_target_bucket, key): raise AssertionError( 'Error while creating lambda: %s;' 'Deployment package %s does not exist ' 'in %s bucket', name, key, self.deploy_target_bucket) lambda_def = self.lambda_conn.get_function(name) if lambda_def: _LOG.warn('%s lambda exists.', name) return self.describe_lambda(name, meta, lambda_def) role_name = meta['iam_role_name'] role_arn = self.iam_conn.check_if_role_exists(role_name) if not role_arn: raise AssertionError('Role {} does not exist; ' 'Lambda {} failed to be configured.'.format( role_name, name)) dl_type = meta.get('dl_resource_type') if dl_type: dl_type = dl_type.lower() dl_name = meta.get('dl_resource_name') dl_target_arn = 'arn:aws:{0}:{1}:{2}:{3}'.format( dl_type, self.region, self.account_id, dl_name) if dl_type and dl_name else None publish_version = meta.get('publish_version', False) lambda_layers_arns = [] layer_meta = meta.get('layers') if layer_meta: for layer_name in layer_meta: layer_arn = self.lambda_conn.get_lambda_layer_arn(layer_name) if not layer_arn: raise AssertionError( 'Could not link lambda layer {} to lambda {} ' 'due to layer absence!'.format(layer_name, name)) lambda_layers_arns.append(layer_arn) self.lambda_conn.create_lambda( lambda_name=name, func_name=meta['func_name'], role=role_arn, runtime=meta['runtime'].lower(), memory=meta['memory'], timeout=meta['timeout'], s3_bucket=self.deploy_target_bucket, s3_key=key, env_vars=meta.get('env_variables'), vpc_sub_nets=meta.get('subnet_ids'), vpc_security_group=meta.get('security_group_ids'), dl_target_arn=dl_target_arn, tracing_mode=meta.get('tracing_mode'), publish_version=publish_version, layers=lambda_layers_arns) _LOG.debug('Lambda created %s', name) # AWS sometimes returns None after function creation, needs for stability time.sleep(10) log_group_name = name retention = meta.get('logs_expiration') if retention: self.cw_logs_conn.create_log_group_with_retention_days( group_name=log_group_name, retention_in_days=retention) lambda_def = self.__describe_lambda_by_version( name) if publish_version else self.lambda_conn.get_function(name) version = lambda_def['Configuration']['Version'] self._setup_function_concurrency(name=name, meta=meta) # enabling aliases # aliases can be enabled only and for $LATEST alias = meta.get('alias') if alias: _LOG.debug('Creating alias') _LOG.debug( self.lambda_conn.create_alias(function_name=name, name=alias, version=version)) arn = self.build_lambda_arn_with_alias(lambda_def, alias) \ if publish_version or alias else \ lambda_def['Configuration']['FunctionArn'] _LOG.debug('arn value: ' + str(arn)) if meta.get('event_sources'): for trigger_meta in meta.get('event_sources'): trigger_type = trigger_meta['resource_type'] func = self.CREATE_TRIGGER[trigger_type] func(self, name, arn, role_name, trigger_meta) # concurrency configuration self._manage_provisioned_concurrency_configuration( function_name=name, meta=meta, lambda_def=lambda_def) return self.describe_lambda(name, meta, lambda_def)
def _create_lambda_from_meta(name, meta): req_params = ['iam_role_name', 'runtime', 'memory', 'timeout', 'func_name'] # Lambda configuration validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] if not _S3_CONN.is_file_exists(CONFIG.deploy_target_bucket, key): raise AssertionError('Deployment package %s does not exist ' 'in %s bucket', key, CONFIG.deploy_target_bucket) response = _LAMBDA_CONN.get_function(name) if response: _LOG.warn('%s lambda exists.', name) return describe_lambda(name, meta, response) role_name = meta['iam_role_name'] role_arn = CONN.iam().check_if_role_exists(role_name) if not role_arn: raise AssertionError('Role {0} does not exist.'.format(role_name)) dl_type = meta.get('dl_resource_type') if dl_type: dl_type = dl_type.lower() dl_name = meta.get('dl_resource_name') dl_target_arn = 'arn:aws:{0}:{1}:{2}:{3}'.format(dl_type, CONFIG.region, CONFIG.account_id, dl_name) if dl_type and dl_name else None publish_version = meta.get('publish_version', False) _LAMBDA_CONN.create_lambda( lambda_name=name, func_name=meta['func_name'], role=role_arn, runtime=meta['runtime'].lower(), memory=meta['memory'], timeout=meta['timeout'], s3_bucket=CONFIG.deploy_target_bucket, s3_key=key, env_vars=meta.get('env_variables'), vpc_sub_nets=meta.get('subnet_ids'), vpc_security_group=meta.get('security_group_ids'), dl_target_arn=dl_target_arn, tracing_mode=meta.get('tracing_mode'), publish_version=publish_version ) # AWS sometimes returns None after function creation, needs for stability time.sleep(10) response = __describe_lambda_by_version( name) if publish_version else _LAMBDA_CONN.get_function(name) version = response['Configuration']['Version'] con_exec = meta.get('concurrent_executions') if con_exec: _LOG.debug('Going to set up concurrency executions') unresolved_exec = _LAMBDA_CONN.get_unresolved_concurrent_executions() if con_exec <= unresolved_exec: _LAMBDA_CONN.put_function_concurrency( function_name=name, concurrent_executions=con_exec) _LOG.debug('Concurrency is enabled for %s lambda', name) else: _LOG.warn( 'Account does not have any unresolved executions.' ' Current size - %s', unresolved_exec) # enabling aliases # aliases can be enabled only and for $LATEST alias = meta.get('alias') if alias: _LOG.debug('Creating alias') _LOG.debug(_LAMBDA_CONN.create_alias(function_name=name, name=alias, version=version)) arn = build_lambda_arn_with_alias(response, alias) if publish_version or alias else \ response['Configuration']['FunctionArn'] _LOG.debug('arn value: ' + str(arn)) if meta.get('event_sources'): for trigger_meta in meta.get('event_sources'): trigger_type = trigger_meta['resource_type'] func = CREATE_TRIGGER[trigger_type] func(name, arn, role_name, trigger_meta) _LOG.info('Created lambda %s.', name) return describe_lambda(name, meta, response)
def _update_lambda(self, name, meta, context): _LOG.info('Updating lambda: {0}'.format(name)) req_params = ['runtime', 'memory', 'timeout', 'func_name'] validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] if not self.s3_conn.is_file_exists(self.deploy_target_bucket, key): raise AssertionError('Deployment package {0} does not exist ' 'in {1} bucket'.format( key, self.deploy_target_bucket)) response = self.lambda_conn.get_function(name) if not response: raise AssertionError('{0} lambda does not exist.'.format(name)) publish_version = meta.get('publish_version', False) self.lambda_conn.update_code_source( lambda_name=name, s3_bucket=self.deploy_target_bucket, s3_key=key, publish_version=publish_version) # update lambda layers version if meta.get('layers'): layers = meta.get('layers') updated_layers_arns = [ layer_arn for layer_arn, body in context.items() if body.get('resource_name') in layers ] self.lambda_conn.update_lambda_configuration( lambda_name=name, layers=updated_layers_arns) # AWS sometimes returns None after function creation, needs for stability time.sleep(10) response = self.lambda_conn.get_function(name) _LOG.debug('Lambda describe result: {0}'.format(response)) code_sha_256 = response['Configuration']['CodeSha256'] publish_ver_response = self.lambda_conn.publish_version( function_name=name, code_sha_256=code_sha_256) updated_version = publish_ver_response['Version'] _LOG.info('Version {0} for lambda {1} published'.format( updated_version, name)) alias_name = meta.get('alias') if alias_name: alias = self.lambda_conn.get_alias(function_name=name, name=alias_name) if not alias: self.lambda_conn.create_alias(function_name=name, name=alias_name, version=updated_version) _LOG.info('Alias {0} has been created for lambda {1}'.format( alias_name, name)) else: self.lambda_conn.update_alias(function_name=name, alias_name=alias_name, function_version=updated_version) _LOG.info('Alias {0} has been updated for lambda {1}'.format( alias_name, name)) req_max_concurrency = meta.get(LAMBDA_MAX_CONCURRENCY) existing_max_concurrency = self.lambda_conn.describe_function_concurrency( name=name) if req_max_concurrency and existing_max_concurrency: if existing_max_concurrency != req_max_concurrency: self._set_function_concurrency(name=name, meta=meta) elif not req_max_concurrency and existing_max_concurrency: self.lambda_conn.delete_function_concurrency_config(name=name) elif req_max_concurrency and not existing_max_concurrency: self._set_function_concurrency(name=name, meta=meta) self._manage_provisioned_concurrency_configuration(function_name=name, meta=meta, lambda_def=context)