def test_sample_lambda_nodejs_with_env(awsclient, vendored_folder,
cleanup_lambdas_deprecated, cleanup_roles):
log.info('running test_sample_lambda_nodejs_with_env')
lambda_folder = './resources/sample_lambda_nodejs_with_env/'
temp_string = utils.random_string()
lambda_name = 'jenkins_test_sample-lambda-nodejs6_10_' + temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
# create the function
create_lambda_helper(awsclient, lambda_name, role_arn,
here(lambda_folder + 'index.js'),
lambda_handler='index.handler',
folders_from_file=[],
runtime='nodejs6.10',
environment={"MYVALUE": "FOO"}
)
cleanup_roles.append(role_name)
cleanup_lambdas_deprecated.append(lambda_name)
payload = '{"ramuda_action": "getenv"}' # provided by our test sample
result = invoke(awsclient, lambda_name, payload)
env = json.loads(result)
assert 'MYVALUE' in env
assert env['MYVALUE'] == 'FOO'
def test_create_stack_rolearn(
awsclient, cleanup_stack_simple_stack, temp_cloudformation_policy,
cleanup_roles):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py')
)
# create role to use for cloudformation deployment
role = create_role_helper(
awsclient,
'unittest_%s_kumo' % utils.random_string(),
policies=[
temp_cloudformation_policy,
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
principal_service=['cloudformation.amazonaws.com']
)
cleanup_roles.append(role['RoleName'])
config_rolearn = deepcopy(config_simple_stack)
config_rolearn['stack']['RoleARN'] = role['Arn']
exit_code = deploy_stack(awsclient, {}, config_rolearn,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
def test_wire_unwire_new_events_s3(awsclient, vendored_folder, temp_bucket,
cleanup_lambdas, cleanup_roles):
log.info('running test_wire_unwire_new_events_s3')
# create a lambda function
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
events = [{
"event_source": {
"arn": "arn:aws:s3:::" + temp_bucket,
"events": ['s3:ObjectCreated:*'],
"suffix": ".gz"
}
}]
cleanup_lambdas.append((lambda_name, events))
# wire the function with the bucket
exit_code = wire(awsclient, events, lambda_name)
assert exit_code == 0
# put a file into the bucket
awsclient.get_client('s3').put_object(
ACL='public-read',
Body=b'this is some content',
Bucket=temp_bucket,
Key='test_file.gz',
)
# validate function call
time.sleep(20) # sleep till the event arrived
assert int(_get_count(awsclient, lambda_name)) == 1
# unwire the function
exit_code = unwire(awsclient, events, lambda_name)
assert exit_code == 0
# put in another file
awsclient.get_client('s3').put_object(
ACL='public-read',
Body=b'this is some content',
Bucket=temp_bucket,
Key='test_file_2.gz',
)
# validate function not called
time.sleep(10)
assert int(_get_count(awsclient, lambda_name)) == 1
def test_deprecated_schedule_event_source(awsclient, vendored_folder,
cleanup_lambdas_deprecated,
cleanup_roles):
log.info('running test_schedule_event_source')
# include reading config from settings file
config = {
"lambda": {
"events": {
"timeSchedules": [{
"ruleName": "unittest-dev-lambda-schedule",
"ruleDescription": "run every 1 minute",
"scheduleExpression": "rate(1 minute)"
}]
}
}
}
# for time_event in time_event_sources:
time_event = config['lambda'].get('events', []).get('timeSchedules', [])[0]
rule_name = time_event.get('ruleName')
rule_description = time_event.get('ruleDescription')
schedule_expression = time_event.get('scheduleExpression')
# now, I need a lambda function that registers the calls!!
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
cleanup_lambdas_deprecated.append(lambda_name)
# lookup lambda arn
lambda_client = awsclient.get_client('lambda')
alias_name = 'ACTIVE'
lambda_arn = lambda_client.get_alias(FunctionName=lambda_name,
Name=alias_name)['AliasArn']
# create scheduled event source
rule_arn = _lambda_add_time_schedule_event_source(awsclient, rule_name,
rule_description,
schedule_expression,
lambda_arn)
_lambda_add_invoke_permission(awsclient, lambda_name,
'events.amazonaws.com', rule_arn)
time.sleep(180) # wait for at least 2 invocations
count = _get_count(awsclient, lambda_name)
assert int(count) >= 2
def test_update_lambda(awsclient, vendored_folder, cleanup_lambdas_deprecated,
cleanup_roles):
log.info('running test_update_lambda')
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
# create the function
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient, lambda_name, role_arn,
'./resources/sample_lambda/handler.py')
# update the function
create_lambda_helper(awsclient, lambda_name, role_arn,
'./resources/sample_lambda/handler_v2.py')
cleanup_lambdas_deprecated.append(lambda_name)
def test_event_source_lifecycle_cloudwatch_pattern(awsclient, vendored_folder,
cleanup_lambdas_deprecated,
cleanup_roles):
log.info('running test_event_source_lifecycle_cloudwatch_pattern')
lambda_folder = './resources/sample_lambda_event_pattern/'
temp_string = utils.random_string()
lambda_name = 'jenkins_test_sample-lambda-event-pattern_' + temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
# create the function
create_lambda_helper(awsclient,
lambda_name,
role_arn,
here(lambda_folder + 'handler.py'),
lambda_handler='handler.handler',
folders_from_file=[],
runtime='python2.7')
cleanup_roles.append(role_name)
cleanup_lambdas_deprecated.append(lambda_name)
# lookup lambda arn
# us-east-1 is the only region that implements lambda@edge
lambda_client = awsclient.get_client('lambda')
alias_name = 'ACTIVE'
lambda_arn = lambda_client.get_alias(FunctionName=lambda_name,
Name=alias_name)['AliasArn']
# define event source
evt_source = {
"name": "ssm_parameter_changed",
"input_path": "$.detail",
"pattern": {
"source": ["aws.ssm"],
"detail-type": ["Parameter Store Change"]
}
}
# event source lifecycle
_add_event_source(awsclient, evt_source, lambda_arn)
status = _get_event_source_status(awsclient, evt_source, lambda_arn)
assert status['EventSourceArn']
_remove_event_source(awsclient, evt_source, lambda_arn)
def test_wire_unwire_new_events_sns(awsclient, vendored_folder,
cleanup_lambdas, cleanup_roles,
temp_sns_topic):
log.info('running test_wire_unwire_new_events_sns')
# create a lambda function
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
# schedule expressions:
# http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html
events = [{
"event_source": {
"arn": temp_sns_topic[1],
"events": ["sns:Publish"]
}
}]
cleanup_lambdas.append((lambda_name, events))
# wire the function with the bucket
exit_code = wire(awsclient, events, lambda_name)
assert exit_code == 0
assert int(_get_count(awsclient, lambda_name)) == 0
# send message via sns
send_message(awsclient, temp_sns_topic[1], {"foo": "bar"})
time.sleep(10) # give a little time for the message to arrive
assert int(_get_count(awsclient, lambda_name)) == 1
# unwire the function
exit_code = unwire(awsclient, events, lambda_name)
assert exit_code == 0
def test_wire_unwire_new_events_cloudwatch_schedule(awsclient, vendored_folder,
cleanup_lambdas,
cleanup_roles):
log.info('running test_wire_unwire_new_events_cloudwatch_schedule')
# create a lambda function
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
# schedule expressions:
# http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html
events = [{
"event_source": {
"name": "execute_backup",
"schedule": "rate(1 minute)"
}
}]
cleanup_lambdas.append((lambda_name, events))
# wire the function with the bucket
exit_code = wire(awsclient, events, lambda_name)
assert exit_code == 0
assert int(_get_count(awsclient, lambda_name)) == 0
time.sleep(70) # sleep till scheduled event
assert int(_get_count(awsclient, lambda_name)) == 1
# unwire the function
exit_code = unwire(awsclient, events, lambda_name)
assert exit_code == 0
time.sleep(70)
assert int(_get_count(awsclient, lambda_name)) == 1
def test_lambda_add_invoke_permission(awsclient, vendored_folder,
temp_bucket, cleanup_lambdas_deprecated,
cleanup_roles):
log.info('running test_lambda_add_invoke_permission')
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient, lambda_name, role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
cleanup_lambdas_deprecated.append(lambda_name)
bucket_name = temp_bucket
s3_arn = 'arn:aws:s3:::' + bucket_name
response = _lambda_add_invoke_permission(
awsclient, lambda_name, 's3.amazonaws.com', s3_arn)
# {"Statement":"{\"Condition\":{\"ArnLike\":{\"AWS:SourceArn\":\"arn:aws:s3:::unittest-lambda-s3-bucket-coedce\"}},\"Action\":[\"lambda:InvokeFunction\"],\"Resource\":\"arn:aws:lambda:eu-west-1:188084614522:function:jenkins_test_coedce:ACTIVE\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"s3.amazonaws.com\"},\"Sid\":\"07c77fac-68ff-11e6-97f8-c4850848610b\"}"}
assert_not_in('Error', response)
assert_in('lambda:InvokeFunction', response['Statement'])
def test_update_stack_rolearn(awsclient, simple_cloudformation_stack,
temp_cloudformation_policy, cleanup_roles):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py')
)
# create role to use for cloudformation update
role = create_role_helper(
awsclient,
'unittest_%s_kumo' % utils.random_string(),
policies=[
temp_cloudformation_policy,
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
principal_service=['cloudformation.amazonaws.com']
)
cleanup_roles.append(role['RoleName'])
config_rolearn = deepcopy(config_simple_stack)
config_rolearn['stack']['RoleARN'] = role['Arn']
change_set_name, stackname, change_set_type = \
create_change_set(awsclient, {}, config_rolearn,
cloudformation_simple_stack)
assert stackname == _get_stack_name(config_rolearn)
assert change_set_name != ''
assert change_set_type == 'UPDATE'
describe_change_set(awsclient, change_set_name, stackname)
# update the stack
changed = get_parameter_diff(awsclient, config_rolearn)
assert not changed
exit_code = deploy_stack(awsclient, {}, config_rolearn,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
def test_deploy_delete_cmds(awsclient, vendored_folder, cleanup_roles,
temp_bucket):
log.info('running test_create_lambda')
temp_string = utils.random_string()
lambda_name = 'jenkins_test_' + temp_string
log.info(lambda_name)
role = create_role_helper(
awsclient,
'unittest_%s_lambda' % temp_string,
policies=[
'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole',
'arn:aws:iam::aws:policy/AWSLambdaExecute'
])
cleanup_roles.append(role['RoleName'])
config = {
"lambda": {
"name": lambda_name,
"description": "unittest for ramuda",
"role": role['Arn'],
"handlerFunction": "handler.handle",
"handlerFile": "./resources/sample_lambda/handler.py",
"timeout": 300,
"memorySize": 256,
"events": {
"s3Sources": [{
"bucket": temp_bucket,
"type": "s3:ObjectCreated:*",
"suffix": ".gz"
}],
"timeSchedules": [{
"ruleName": "infra-dev-sample-lambda-jobr-T1",
"ruleDescription": "run every 5 min from 0-5",
"scheduleExpression": "cron(0/5 0-5 ? * * *)"
}, {
"ruleName":
"infra-dev-sample-lambda-jobr-T2",
"ruleDescription":
"run every 5 min from 8-23:59",
"scheduleExpression":
"cron(0/5 8-23:59 ? * * *)"
}]
},
"vpc": {
"subnetIds": [
"subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-d5ffb0b1",
"subnet-e9db9f9f"
],
"securityGroups": ["sg-660dd700"]
}
},
"bundling": {
"zip":
"bundle.zip",
"folders": [{
"source": "./vendored",
"target": "."
}, {
"source": "./impl",
"target": "impl"
}]
},
"deployment": {
"region": "eu-west-1"
}
}
tooldata = get_tooldata(awsclient, 'ramuda', 'deploy', config=config)
tooldata['context']['_arguments'] = {'--keep': False}
bundle((tooldata['context'], {'ramuda': tooldata['config']}))
deploy_cmd(False, **tooldata)
# now we use the delete cmd to remove the lambda function
tooldata['context']['command'] = 'delete'
delete_cmd(True, lambda_name, True, **tooldata)
def test_deprecated_wire_unwire_lambda_with_s3(awsclient, vendored_folder,
cleanup_lambdas_deprecated,
cleanup_roles, temp_bucket):
log.info('running test_wire_unwire_lambda_with_s3')
# create a lambda function
temp_string = utils.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
cleanup_lambdas_deprecated.append(lambda_name)
bucket_name = temp_bucket
config = {
"lambda": {
"events": {
"s3Sources": [{
"bucket": bucket_name,
"type": "s3:ObjectCreated:*",
"suffix": ".gz"
}]
}
}
}
# wire the function with the bucket
s3_event_sources = config['lambda'].get('events', []).get('s3Sources', [])
time_event_sources = config['lambda'].get('events',
[]).get('timeSchedules', [])
exit_code = wire_deprecated(awsclient, lambda_name, s3_event_sources,
time_event_sources)
assert exit_code == 0
# put a file into the bucket
awsclient.get_client('s3').put_object(
ACL='public-read',
Body=b'this is some content',
Bucket=bucket_name,
Key='test_file.gz',
)
# validate function call
time.sleep(20) # sleep till the event arrived
assert int(_get_count(awsclient, lambda_name)) == 1
# unwire the function
exit_code = unwire_deprecated(awsclient, lambda_name, s3_event_sources,
time_event_sources)
assert exit_code == 0
# put in another file
awsclient.get_client('s3').put_object(
ACL='public-read',
Body=b'this is some content',
Bucket=bucket_name,
Key='test_file_2.gz',
)
# validate function not called
time.sleep(10)
assert int(_get_count(awsclient, lambda_name)) == 1
def test_create_lambda(awsclient, vendored_folder, cleanup_lambdas_deprecated,
cleanup_roles):
log.info('running test_create_lambda')
temp_string = utils.random_string()
lambda_name = 'jenkins_test_' + temp_string
log.info(lambda_name)
role = create_role_helper(
awsclient,
'unittest_%s_lambda' % temp_string,
policies=[
'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole',
'arn:aws:iam::aws:policy/AWSLambdaExecute']
)
cleanup_roles.append(role['RoleName'])
config = {
"lambda": {
"name": "dp-dev-sample-lambda-jobr1",
"description": "lambda test for ramuda",
"role": "'unused'",
"handlerFunction": "handler.handle",
"handlerFile": "./resources/sample_lambda/handler.py",
"timeout": 300,
"memorySize": 256,
"events": {
"s3Sources": [
{
"bucket": "jobr-test",
"type": "s3:ObjectCreated:*",
"suffix": ".gz"
}
],
"timeSchedules": [
{
"ruleName": "infra-dev-sample-lambda-jobr-T1",
"ruleDescription": "run every 5 min from 0-5",
"scheduleExpression": "cron(0/5 0-5 ? * * *)"
},
{
"ruleName": "infra-dev-sample-lambda-jobr-T2",
"ruleDescription": "run every 5 min from 8-23:59",
"scheduleExpression": "cron(0/5 8-23:59 ? * * *)"
}
]
},
"vpc": {
"subnetIds": [
"subnet-d5ffb0b1",
"subnet-d5ffb0b1",
"subnet-d5ffb0b1",
"subnet-e9db9f9f"
],
"securityGroups": [
"sg-660dd700"
]
}
},
"bundling": {
"zip": "bundle.zip",
"folders": [
{
"source": "./vendored",
"target": "."
},
{
"source": "./impl",
"target": "impl"
}
]
},
"deployment": {
"region": "eu-west-1"
}
}
lambda_description = config['lambda'].get('description')
role_arn = role['Arn']
lambda_handler = config['lambda'].get('handlerFunction')
handler_filename = config['lambda'].get('handlerFile')
timeout = int(config['lambda'].get('timeout'))
memory_size = int(config['lambda'].get('memorySize'))
zip_name = config['bundling'].get('zip')
folders_from_file = config['bundling'].get('folders')
subnet_ids = config['lambda'].get('vpc', {}).get('subnetIds', None)
security_groups = config['lambda'].get('vpc', {}).get('securityGroups',
None)
region = config['deployment'].get('region')
artifact_bucket = config['deployment'].get('artifactBucket', None)
zipfile = get_zipped_file(
handler_filename,
folders_from_file,
)
deploy_lambda(
awsclient=awsclient,
function_name=lambda_name,
role=role_arn,
handler_filename=handler_filename,
handler_function=lambda_handler,
folders=folders_from_file,
description=lambda_description,
timeout=timeout,
memory=memory_size,
artifact_bucket=artifact_bucket,
zipfile=zipfile
)
cleanup_lambdas_deprecated.append(lambda_name)
def test_create_update_stack_artifactbucket(awsclient, temp_cloudformation_policy,
cleanup_roles, cleanup_buckets):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py')
)
upload_conf = {
'stack': {
'StackName': "infra-dev-kumo-sample-stack",
'artifactBucket': "unittest-kumo-artifact-bucket"
},
'parameters': {
'InstanceType': "t2.micro",
}
}
region = awsclient.get_client('s3').meta.region_name
account = os.getenv('ACCOUNT', None)
# add account prefix to artifact bucket config
if account:
upload_conf['stack']['artifactBucket'] = \
'%s-unittest-kumo-artifact-bucket' % account
artifact_bucket = _get_artifact_bucket(upload_conf)
prepare_artifacts_bucket(awsclient, artifact_bucket)
cleanup_buckets.append(artifact_bucket)
dest_key = 'kumo/%s/%s-cloudformation.json' % (region,
_get_stack_name(upload_conf))
expected_s3url = 'https://s3-%s.amazonaws.com/%s/%s' % (region,
artifact_bucket,
dest_key)
actual_s3url = _s3_upload(awsclient, upload_conf,
generate_template({}, upload_conf, cloudformation_simple_stack))
assert expected_s3url == actual_s3url
# create role to use for cloudformation update
role = create_role_helper(
awsclient,
'unittest_%s_kumo' % utils.random_string(),
policies=[
temp_cloudformation_policy,
'arn:aws:iam::aws:policy/AWSCodeDeployReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
principal_service=['cloudformation.amazonaws.com']
)
cleanup_roles.append(role['RoleName'])
# create
exit_code = deploy_stack(awsclient, {}, upload_conf,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
stack_id = get_stack_id(awsclient, upload_conf['stack']['StackName'])
wait_for_stack_create_complete(awsclient, stack_id)
# update (as a change we add the RoleARN)
upload_conf['stack']['RoleARN'] = role['Arn']
# update the stack
changed = get_parameter_diff(awsclient, upload_conf)
assert not changed
exit_code = deploy_stack(awsclient, {}, upload_conf,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
wait_for_stack_update_complete(awsclient, stack_id)
# cleanup
exit_code = delete_stack(awsclient, upload_conf)
assert exit_code == 0
wait_for_stack_delete_complete(awsclient, stack_id)