def _create_kinesis_stream_trigger_from_meta(lambda_name, lambda_arn, role_name, trigger_meta): required_parameters = ['target_stream', 'batch_size', 'starting_position'] validate_params(lambda_name, trigger_meta, required_parameters) stream_name = trigger_meta['target_stream'] stream = CONN.kinesis().get_stream(stream_name) stream_arn = stream['StreamDescription']['StreamARN'] stream_status = stream['StreamDescription']['StreamStatus'] # additional waiting for stream if stream_status != 'ACTIVE': _LOG.debug('Kinesis stream %s is not in active state,' ' waiting for activation...', stream_name) time.sleep(120) # TODO policy should be moved to meta policy_name = '{0}KinesisTo{1}Lambda'.format(stream_name, lambda_name) policy_document = { "Statement": [ { "Effect": "Allow", "Action": [ "lambda:InvokeFunction" ], "Resource": [ lambda_arn ] }, { "Action": [ "kinesis:DescribeStreams", "kinesis:DescribeStream", "kinesis:ListStreams", "kinesis:GetShardIterator", "Kinesis:GetRecords" ], "Effect": "Allow", "Resource": stream_arn } ], "Version": "2012-10-17" } CONN.iam().attach_inline_policy(role_name=role_name, policy_name=policy_name, policy_document=policy_document) _LOG.debug('Inline policy %s is attached to role %s', policy_name, role_name) _LOG.debug('Waiting for activation policy %s...', policy_name) time.sleep(10) _add_kinesis_event_source(lambda_arn, stream_arn, trigger_meta) _LOG.info('Lambda %s subscribed to kinesis stream %s', lambda_name, stream_name)
def _enable_autoscaling(autoscaling_config, name): targets = [] policies = [] for item in autoscaling_config: autoscaling_required_parameters = [ 'resource_name', 'dimension', 'min_capacity', 'max_capacity', 'role_name' ] validate_params(name, item, autoscaling_required_parameters) role_name = item['role_name'] role_arn = CONN.iam().check_if_role_exists(role_name) if role_arn: dimension = item['dimension'] resource_id, sc_targets = register_autoscaling_target( dimension, item, role_arn, name) targets.extend(sc_targets) _LOG.debug('Autoscaling %s is set up for %s', dimension, resource_id) autoscaling_policy = item.get('config') if autoscaling_policy: policy_name = autoscaling_policy['policy_name'] _LOG.debug('Going to set up autoscaling with ' 'policy %s', policy_name) sc_policies = put_autoscaling_policy(autoscaling_policy, dimension, policy_name, resource_id) policies.append(sc_policies) _LOG.debug('Policy %s is set up', policy_name) else: _LOG.warn('Role %s is not found, skip autoscaling config', role_name) return {'targets': targets, 'policies': policies}
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ from botocore.exceptions import ClientError from syndicate.commons.log_helper import get_logger from syndicate.connection.helper import retry from syndicate.core import CONFIG, CONN from syndicate.core.helper import create_pool, prettify_json, unpack_kwargs from syndicate.core.resources.helper import (build_description_obj, resolve_dynamic_identifier) _IAM_CONN = CONN.iam() _LOG = get_logger('syndicate.core.resources.iam_resource') def remove_policies(args): create_pool(_remove_policy, args) @unpack_kwargs def _remove_policy(arn, config): policy_name = config['resource_name'] try: _IAM_CONN.remove_policy(arn) _LOG.info('IAM policy %s was removed.', policy_name) except ClientError as e:
def _create_lambda_from_meta(name, meta): _LOG.debug('Creating lambda %s', name) req_params = ['iam_role_name', 'runtime', 'memory', 'timeout', 'func_name'] # Lambda configuration validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] if not _S3_CONN.is_file_exists(CONFIG.deploy_target_bucket, key): raise AssertionError( 'Error while creating lambda: %s;' 'Deployment package %s does not exist ' 'in %s bucket', name, key, CONFIG.deploy_target_bucket) lambda_def = _LAMBDA_CONN.get_function(name) if lambda_def: _LOG.warn('%s lambda exists.', name) return describe_lambda(name, meta, lambda_def) role_name = meta['iam_role_name'] role_arn = CONN.iam().check_if_role_exists(role_name) if not role_arn: raise AssertionError('Role {} does not exist; ' 'Lambda {} failed to be configured.'.format( role_name, name)) dl_type = meta.get('dl_resource_type') if dl_type: dl_type = dl_type.lower() dl_name = meta.get('dl_resource_name') dl_target_arn = 'arn:aws:{0}:{1}:{2}:{3}'.format( dl_type, CONFIG.region, CONFIG.account_id, dl_name) if dl_type and dl_name else None publish_version = meta.get('publish_version', False) lambda_layers_arns = [] layer_meta = meta.get('layers') if layer_meta: for layer_name in layer_meta: layer_arn = _LAMBDA_CONN.get_lambda_layer_arn(layer_name) if not layer_arn: raise AssertionError( 'Could not link lambda layer {} to lambda {} ' 'due to layer absence!'.format(layer_name, name)) lambda_layers_arns.append(layer_arn) _LAMBDA_CONN.create_lambda( lambda_name=name, func_name=meta['func_name'], role=role_arn, runtime=meta['runtime'].lower(), memory=meta['memory'], timeout=meta['timeout'], s3_bucket=CONFIG.deploy_target_bucket, s3_key=key, env_vars=meta.get('env_variables'), vpc_sub_nets=meta.get('subnet_ids'), vpc_security_group=meta.get('security_group_ids'), dl_target_arn=dl_target_arn, tracing_mode=meta.get('tracing_mode'), publish_version=publish_version, layers=lambda_layers_arns) _LOG.debug('Lambda created %s', name) # AWS sometimes returns None after function creation, needs for stability time.sleep(10) lambda_def = __describe_lambda_by_version( name) if publish_version else _LAMBDA_CONN.get_function(name) version = lambda_def['Configuration']['Version'] _setup_function_concurrency(name=name, meta=meta) # enabling aliases # aliases can be enabled only and for $LATEST alias = meta.get('alias') if alias: _LOG.debug('Creating alias') _LOG.debug( _LAMBDA_CONN.create_alias(function_name=name, name=alias, version=version)) arn = build_lambda_arn_with_alias(lambda_def, alias) \ if publish_version or alias else \ lambda_def['Configuration']['FunctionArn'] _LOG.debug('arn value: ' + str(arn)) if meta.get('event_sources'): for trigger_meta in meta.get('event_sources'): trigger_type = trigger_meta['resource_type'] func = CREATE_TRIGGER[trigger_type] func(name, arn, role_name, trigger_meta) # concurrency configuration _manage_provisioned_concurrency_configuration(function_name=name, meta=meta, lambda_def=lambda_def) return describe_lambda(name, meta, lambda_def)
def _create_lambda_from_meta(name, meta): req_params = ['iam_role_name', 'runtime', 'memory', 'timeout', 'func_name'] # Lambda configuration validate_params(name, meta, req_params) key = meta[S3_PATH_NAME] if not _S3_CONN.is_file_exists(CONFIG.deploy_target_bucket, key): raise AssertionError('Deployment package %s does not exist ' 'in %s bucket', key, CONFIG.deploy_target_bucket) response = _LAMBDA_CONN.get_function(name) if response: _LOG.warn('%s lambda exists.', name) return describe_lambda(name, meta, response) role_name = meta['iam_role_name'] role_arn = CONN.iam().check_if_role_exists(role_name) if not role_arn: raise AssertionError('Role {0} does not exist.'.format(role_name)) dl_type = meta.get('dl_resource_type') if dl_type: dl_type = dl_type.lower() dl_name = meta.get('dl_resource_name') dl_target_arn = 'arn:aws:{0}:{1}:{2}:{3}'.format(dl_type, CONFIG.region, CONFIG.account_id, dl_name) if dl_type and dl_name else None publish_version = meta.get('publish_version', False) _LAMBDA_CONN.create_lambda( lambda_name=name, func_name=meta['func_name'], role=role_arn, runtime=meta['runtime'].lower(), memory=meta['memory'], timeout=meta['timeout'], s3_bucket=CONFIG.deploy_target_bucket, s3_key=key, env_vars=meta.get('env_variables'), vpc_sub_nets=meta.get('subnet_ids'), vpc_security_group=meta.get('security_group_ids'), dl_target_arn=dl_target_arn, tracing_mode=meta.get('tracing_mode'), publish_version=publish_version ) # AWS sometimes returns None after function creation, needs for stability time.sleep(10) response = __describe_lambda_by_version( name) if publish_version else _LAMBDA_CONN.get_function(name) version = response['Configuration']['Version'] con_exec = meta.get('concurrent_executions') if con_exec: _LOG.debug('Going to set up concurrency executions') unresolved_exec = _LAMBDA_CONN.get_unresolved_concurrent_executions() if con_exec <= unresolved_exec: _LAMBDA_CONN.put_function_concurrency( function_name=name, concurrent_executions=con_exec) _LOG.debug('Concurrency is enabled for %s lambda', name) else: _LOG.warn( 'Account does not have any unresolved executions.' ' Current size - %s', unresolved_exec) # enabling aliases # aliases can be enabled only and for $LATEST alias = meta.get('alias') if alias: _LOG.debug('Creating alias') _LOG.debug(_LAMBDA_CONN.create_alias(function_name=name, name=alias, version=version)) arn = build_lambda_arn_with_alias(response, alias) if publish_version or alias else \ response['Configuration']['FunctionArn'] _LOG.debug('arn value: ' + str(arn)) if meta.get('event_sources'): for trigger_meta in meta.get('event_sources'): trigger_type = trigger_meta['resource_type'] func = CREATE_TRIGGER[trigger_type] func(name, arn, role_name, trigger_meta) _LOG.info('Created lambda %s.', name) return describe_lambda(name, meta, response)