def test_lambda_add_invoke_permission(awsclient, vendored_folder, temp_bucket,
cleanup_lambdas, cleanup_roles):
log.info('running test_lambda_add_invoke_permission')
temp_string = helpers.random_string()
# print(temp_string)
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
cleanup_lambdas.append(lambda_name)
bucket_name = temp_bucket
s3_arn = 'arn:aws:s3:::' + bucket_name
response = _lambda_add_invoke_permission(awsclient, lambda_name,
's3.amazonaws.com', s3_arn)
# {"Statement":"{\"Condition\":{\"ArnLike\":{\"AWS:SourceArn\":\"arn:aws:s3:::unittest-lambda-s3-bucket-coedce\"}},\"Action\":[\"lambda:InvokeFunction\"],\"Resource\":\"arn:aws:lambda:eu-west-1:188084614522:function:jenkins_test_coedce:ACTIVE\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"s3.amazonaws.com\"},\"Sid\":\"07c77fac-68ff-11e6-97f8-c4850848610b\"}"}
assert_not_in('Error', response)
assert_in('lambda:InvokeFunction', response['Statement'])
def temp_bucket(awsclient):
# create a bucket
temp_string = helpers.random_string()
bucket_name = 'unittest-lambda-s3-event-source-%s' % temp_string
create_bucket(awsclient, bucket_name)
yield bucket_name
# cleanup
delete_bucket(awsclient, bucket_name)
def test_schedule_event_source(awsclient, vendored_folder, cleanup_lambdas,
cleanup_roles):
log.info('running test_schedule_event_source')
# include reading config from settings file
config = {
"lambda": {
"events": {
"timeSchedules": [{
"ruleName": "unittest-dev-lambda-schedule",
"ruleDescription": "run every 1 minute",
"scheduleExpression": "rate(1 minute)"
}]
}
}
}
# for time_event in time_event_sources:
time_event = config['lambda'].get('events', []).get('timeSchedules', [])[0]
rule_name = time_event.get('ruleName')
rule_description = time_event.get('ruleDescription')
schedule_expression = time_event.get('scheduleExpression')
# now, I need a lambda function that registers the calls!!
temp_string = helpers.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
cleanup_lambdas.append(lambda_name)
# lookup lambda arn
lambda_client = awsclient.get_client('lambda')
alias_name = 'ACTIVE'
lambda_arn = lambda_client.get_alias(FunctionName=lambda_name,
Name=alias_name)['AliasArn']
# create scheduled event source
rule_arn = _lambda_add_time_schedule_event_source(awsclient, rule_name,
rule_description,
schedule_expression,
lambda_arn)
_lambda_add_invoke_permission(awsclient, lambda_name,
'events.amazonaws.com', rule_arn)
time.sleep(180) # wait for at least 2 invocations
count = _get_count(awsclient, lambda_name)
assert_greater_equal(int(count), 2)
def test_random_string_recording(awsclient):
# record and playback cases are identical for this test
lines = []
for i in range(5):
lines.append(helpers.random_string())
prefix = 'tests.test_helpers_aws.test_random_string_recording'
record_dir = os.path.join(here('./resources/placebo_awsclient'), prefix)
random_string_filename = 'random_string.txt'
with open(os.path.join(record_dir, random_string_filename), 'r') as rfile:
rlines = [l.strip() for l in rfile.readlines()]
assert lines == rlines
def test_update_lambda(awsclient, vendored_folder, cleanup_lambdas,
cleanup_roles):
log.info('running test_update_lambda')
temp_string = helpers.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
# create the function
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient, lambda_name, role_arn,
'./resources/sample_lambda/handler.py')
# update the function
create_lambda_helper(awsclient, lambda_name, role_arn,
'./resources/sample_lambda/handler_v2.py')
cleanup_lambdas.append(lambda_name)
def temp_lambda(awsclient):
# provide a lambda function and cleanup after test suite
temp_string = helpers.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
# create the function
role_arn = create_lambda_role_helper(awsclient, role_name)
create_lambda_helper(
awsclient,
lambda_name,
role_arn,
# './resources/sample_lambda/handler.py',
here('./resources/sample_lambda/handler.py'),
lambda_handler='handler.handle')
yield lambda_name, role_name, role_arn
# cleanup
delete_lambda(awsclient, lambda_name)
delete_role_helper(awsclient, role_name)
def test_create_api(awsclient, cleanup_api_keys, cleanup_apis):
log.info('running test_create_api')
temp_string = helpers.random_string()
api_name = 'unittest-gcdt-sample-api-%s' % temp_string
api_key_name = 'unittest-gcdt-sample-api-key-%s' % temp_string
api_description = 'Gcdt sample API based on dp api-mock'
target_stage = 'mock'
api_key = create_api_key(awsclient, api_name, api_key_name)
cleanup_api_keys.append(api_key)
lambdas = []
deploy_api(awsclient=awsclient,
api_name=api_name,
api_description=api_description,
stage_name=target_stage,
api_key=api_key,
lambdas=lambdas)
cleanup_apis.append(api_name)
def test_random_string():
ts = random_string()
print(ts)
assert_equal(len(ts), 6)
nose.tools.assert_not_equal(ts, random_string())
def test_bucket_does_not_exist(awsclient):
temp_string = helpers.random_string()
bucket_name = 'unittest-lambda-s3-event-source-%s' % temp_string
assert not bucket_exists(awsclient, bucket_name)
def test_deploy_delete_cmds(awsclient, vendored_folder, cleanup_roles,
temp_bucket):
log.info('running test_create_lambda')
temp_string = helpers.random_string()
lambda_name = 'jenkins_test_' + temp_string
log.info(lambda_name)
role = create_role_helper(
awsclient,
'unittest_%s_lambda' % temp_string,
policies=[
'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole',
'arn:aws:iam::aws:policy/AWSLambdaExecute'
])
cleanup_roles.append(role['RoleName'])
config = {
"lambda": {
"name": lambda_name,
"description": "unittest for ramuda",
"role": role['Arn'],
"handlerFunction": "handler.handle",
"handlerFile": "./resources/sample_lambda/handler.py",
"timeout": 300,
"memorySize": 256,
"events": {
"s3Sources": [{
"bucket": temp_bucket,
"type": "s3:ObjectCreated:*",
"suffix": ".gz"
}],
"timeSchedules": [{
"ruleName": "infra-dev-sample-lambda-jobr-T1",
"ruleDescription": "run every 5 min from 0-5",
"scheduleExpression": "cron(0/5 0-5 ? * * *)"
}, {
"ruleName":
"infra-dev-sample-lambda-jobr-T2",
"ruleDescription":
"run every 5 min from 8-23:59",
"scheduleExpression":
"cron(0/5 8-23:59 ? * * *)"
}]
},
"vpc": {
"subnetIds": [
"subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-d5ffb0b1",
"subnet-e9db9f9f"
],
"securityGroups": ["sg-660dd700"]
}
},
"bundling": {
"zip":
"bundle.zip",
"folders": [{
"source": "./vendored",
"target": "."
}, {
"source": "./impl",
"target": "impl"
}]
},
"deployment": {
"region": "eu-west-1"
}
}
tooldata = get_tooldata(awsclient, 'ramuda', 'deploy', config=config)
bundle((tooldata['context'], {'ramuda': tooldata['config']}))
deploy_cmd(**tooldata)
# now we use the delete cmd to remove the lambda function
tooldata['context']['command'] = 'delete'
delete_cmd(True, lambda_name, **tooldata)
def test_create_lambda(awsclient, vendored_folder, cleanup_lambdas,
cleanup_roles):
log.info('running test_create_lambda')
temp_string = helpers.random_string()
lambda_name = 'jenkins_test_' + temp_string
log.info(lambda_name)
role = create_role_helper(
awsclient,
'unittest_%s_lambda' % temp_string,
policies=[
'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole',
'arn:aws:iam::aws:policy/AWSLambdaExecute'
])
cleanup_roles.append(role['RoleName'])
config = {
"lambda": {
"name": "dp-dev-sample-lambda-jobr1",
"description": "lambda test for ramuda",
"role": "'unused'",
"handlerFunction": "handler.handle",
"handlerFile": "./resources/sample_lambda/handler.py",
"timeout": 300,
"memorySize": 256,
"events": {
"s3Sources": [{
"bucket": "jobr-test",
"type": "s3:ObjectCreated:*",
"suffix": ".gz"
}],
"timeSchedules": [{
"ruleName": "infra-dev-sample-lambda-jobr-T1",
"ruleDescription": "run every 5 min from 0-5",
"scheduleExpression": "cron(0/5 0-5 ? * * *)"
}, {
"ruleName":
"infra-dev-sample-lambda-jobr-T2",
"ruleDescription":
"run every 5 min from 8-23:59",
"scheduleExpression":
"cron(0/5 8-23:59 ? * * *)"
}]
},
"vpc": {
"subnetIds": [
"subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-d5ffb0b1",
"subnet-e9db9f9f"
],
"securityGroups": ["sg-660dd700"]
}
},
"bundling": {
"zip":
"bundle.zip",
"folders": [{
"source": "./vendored",
"target": "."
}, {
"source": "./impl",
"target": "impl"
}]
},
"deployment": {
"region": "eu-west-1"
}
}
lambda_description = config['lambda'].get('description')
# print (role)
role_arn = role['Arn']
lambda_handler = config['lambda'].get('handlerFunction')
handler_filename = config['lambda'].get('handlerFile')
timeout = int(config['lambda'].get('timeout'))
memory_size = int(config['lambda'].get('memorySize'))
zip_name = config['bundling'].get('zip')
folders_from_file = config['bundling'].get('folders')
subnet_ids = config['lambda'].get('vpc', {}).get('subnetIds', None)
security_groups = config['lambda'].get('vpc',
{}).get('securityGroups', None)
region = config['deployment'].get('region')
artifact_bucket = config['deployment'].get('artifactBucket', None)
zipfile = _get_zipped_file(
handler_filename,
folders_from_file,
)
deploy_lambda(awsclient=awsclient,
function_name=lambda_name,
role=role_arn,
handler_filename=handler_filename,
handler_function=lambda_handler,
folders=folders_from_file,
description=lambda_description,
timeout=timeout,
memory=memory_size,
artifact_bucket=artifact_bucket,
zipfile=zipfile)
# TODO improve this (by using a waiter??)
cleanup_lambdas.append(lambda_name)
def test_wire_unwire_lambda_with_s3(awsclient, vendored_folder,
cleanup_lambdas, cleanup_roles,
temp_bucket):
log.info('running test_wire_unwire_lambda_with_s3')
# create a lambda function
temp_string = helpers.random_string()
lambda_name = 'jenkins_test_%s' % temp_string
role_name = 'unittest_%s_lambda' % temp_string
role_arn = create_lambda_role_helper(awsclient, role_name)
cleanup_roles.append(role_name)
create_lambda_helper(awsclient,
lambda_name,
role_arn,
'./resources/sample_lambda/handler_counter.py',
lambda_handler='handler_counter.handle')
cleanup_lambdas.append(lambda_name)
bucket_name = temp_bucket
config = {
"lambda": {
"events": {
"s3Sources": [{
"bucket": bucket_name,
"type": "s3:ObjectCreated:*",
"suffix": ".gz"
}]
}
}
}
# wire the function with the bucket
s3_event_sources = config['lambda'].get('events', []).get('s3Sources', [])
time_event_sources = config['lambda'].get('events',
[]).get('timeSchedules', [])
exit_code = wire(awsclient, lambda_name, s3_event_sources,
time_event_sources)
assert_equal(exit_code, 0)
# put a file into the bucket
awsclient.get_client('s3').put_object(
ACL='public-read',
Body=b'this is some content',
Bucket=bucket_name,
Key='test_file.gz',
)
# validate function call
time.sleep(20) # sleep till the event arrived
assert_equal(int(_get_count(awsclient, lambda_name)), 1)
# unwire the function
exit_code = unwire(awsclient, lambda_name, s3_event_sources,
time_event_sources)
assert_equal(exit_code, 0)
# put in another file
awsclient.get_client('s3').put_object(
ACL='public-read',
Body=b'this is some content',
Bucket=bucket_name,
Key='test_file_2.gz',
)
# validate function not called
time.sleep(10)
assert int(_get_count(awsclient, lambda_name)) == 1