def test_sample_lambda_nodejs_with_env(awsclient, vendored_folder, cleanup_lambdas_deprecated, cleanup_roles): log.info('running test_sample_lambda_nodejs_with_env') lambda_folder = './resources/sample_lambda_nodejs_with_env/' temp_string = utils.random_string() lambda_name = 'jenkins_test_sample-lambda-nodejs6_10_' + temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) # create the function create_lambda_helper(awsclient, lambda_name, role_arn, here(lambda_folder + 'index.js'), lambda_handler='index.handler', folders_from_file=[], runtime='nodejs6.10', environment={"MYVALUE": "FOO"} ) cleanup_roles.append(role_name) cleanup_lambdas_deprecated.append(lambda_name) payload = '{"ramuda_action": "getenv"}' # provided by our test sample result = invoke(awsclient, lambda_name, payload) env = json.loads(result) assert 'MYVALUE' in env assert env['MYVALUE'] == 'FOO'
def test_create_stack_rolearn( awsclient, cleanup_stack_simple_stack, temp_cloudformation_policy, cleanup_roles): # create a stack we use for the test lifecycle cloudformation_simple_stack, _ = load_cloudformation_template( here('resources/simple_cloudformation_stack/cloudformation.py') ) # create role to use for cloudformation deployment role = create_role_helper( awsclient, 'unittest_%s_kumo' % utils.random_string(), policies=[ temp_cloudformation_policy, 'arn:aws:iam::aws:policy/AmazonS3FullAccess' ], principal_service=['cloudformation.amazonaws.com'] ) cleanup_roles.append(role['RoleName']) config_rolearn = deepcopy(config_simple_stack) config_rolearn['stack']['RoleARN'] = role['Arn'] exit_code = deploy_stack(awsclient, {}, config_rolearn, cloudformation_simple_stack, override_stack_policy=False) assert exit_code == 0
def test_wire_unwire_new_events_s3(awsclient, vendored_folder, temp_bucket, cleanup_lambdas, cleanup_roles): log.info('running test_wire_unwire_new_events_s3') # create a lambda function temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_counter.py', lambda_handler='handler_counter.handle') events = [{ "event_source": { "arn": "arn:aws:s3:::" + temp_bucket, "events": ['s3:ObjectCreated:*'], "suffix": ".gz" } }] cleanup_lambdas.append((lambda_name, events)) # wire the function with the bucket exit_code = wire(awsclient, events, lambda_name) assert exit_code == 0 # put a file into the bucket awsclient.get_client('s3').put_object( ACL='public-read', Body=b'this is some content', Bucket=temp_bucket, Key='test_file.gz', ) # validate function call time.sleep(20) # sleep till the event arrived assert int(_get_count(awsclient, lambda_name)) == 1 # unwire the function exit_code = unwire(awsclient, events, lambda_name) assert exit_code == 0 # put in another file awsclient.get_client('s3').put_object( ACL='public-read', Body=b'this is some content', Bucket=temp_bucket, Key='test_file_2.gz', ) # validate function not called time.sleep(10) assert int(_get_count(awsclient, lambda_name)) == 1
def test_deprecated_schedule_event_source(awsclient, vendored_folder, cleanup_lambdas_deprecated, cleanup_roles): log.info('running test_schedule_event_source') # include reading config from settings file config = { "lambda": { "events": { "timeSchedules": [{ "ruleName": "unittest-dev-lambda-schedule", "ruleDescription": "run every 1 minute", "scheduleExpression": "rate(1 minute)" }] } } } # for time_event in time_event_sources: time_event = config['lambda'].get('events', []).get('timeSchedules', [])[0] rule_name = time_event.get('ruleName') rule_description = time_event.get('ruleDescription') schedule_expression = time_event.get('scheduleExpression') # now, I need a lambda function that registers the calls!! temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_counter.py', lambda_handler='handler_counter.handle') cleanup_lambdas_deprecated.append(lambda_name) # lookup lambda arn lambda_client = awsclient.get_client('lambda') alias_name = 'ACTIVE' lambda_arn = lambda_client.get_alias(FunctionName=lambda_name, Name=alias_name)['AliasArn'] # create scheduled event source rule_arn = _lambda_add_time_schedule_event_source(awsclient, rule_name, rule_description, schedule_expression, lambda_arn) _lambda_add_invoke_permission(awsclient, lambda_name, 'events.amazonaws.com', rule_arn) time.sleep(180) # wait for at least 2 invocations count = _get_count(awsclient, lambda_name) assert int(count) >= 2
def test_update_lambda(awsclient, vendored_folder, cleanup_lambdas_deprecated, cleanup_roles): log.info('running test_update_lambda') temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string # create the function role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler.py') # update the function create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_v2.py') cleanup_lambdas_deprecated.append(lambda_name)
def test_event_source_lifecycle_cloudwatch_pattern(awsclient, vendored_folder, cleanup_lambdas_deprecated, cleanup_roles): log.info('running test_event_source_lifecycle_cloudwatch_pattern') lambda_folder = './resources/sample_lambda_event_pattern/' temp_string = utils.random_string() lambda_name = 'jenkins_test_sample-lambda-event-pattern_' + temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) # create the function create_lambda_helper(awsclient, lambda_name, role_arn, here(lambda_folder + 'handler.py'), lambda_handler='handler.handler', folders_from_file=[], runtime='python2.7') cleanup_roles.append(role_name) cleanup_lambdas_deprecated.append(lambda_name) # lookup lambda arn # us-east-1 is the only region that implements lambda@edge lambda_client = awsclient.get_client('lambda') alias_name = 'ACTIVE' lambda_arn = lambda_client.get_alias(FunctionName=lambda_name, Name=alias_name)['AliasArn'] # define event source evt_source = { "name": "ssm_parameter_changed", "input_path": "$.detail", "pattern": { "source": ["aws.ssm"], "detail-type": ["Parameter Store Change"] } } # event source lifecycle _add_event_source(awsclient, evt_source, lambda_arn) status = _get_event_source_status(awsclient, evt_source, lambda_arn) assert status['EventSourceArn'] _remove_event_source(awsclient, evt_source, lambda_arn)
def test_wire_unwire_new_events_sns(awsclient, vendored_folder, cleanup_lambdas, cleanup_roles, temp_sns_topic): log.info('running test_wire_unwire_new_events_sns') # create a lambda function temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_counter.py', lambda_handler='handler_counter.handle') # schedule expressions: # http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html events = [{ "event_source": { "arn": temp_sns_topic[1], "events": ["sns:Publish"] } }] cleanup_lambdas.append((lambda_name, events)) # wire the function with the bucket exit_code = wire(awsclient, events, lambda_name) assert exit_code == 0 assert int(_get_count(awsclient, lambda_name)) == 0 # send message via sns send_message(awsclient, temp_sns_topic[1], {"foo": "bar"}) time.sleep(10) # give a little time for the message to arrive assert int(_get_count(awsclient, lambda_name)) == 1 # unwire the function exit_code = unwire(awsclient, events, lambda_name) assert exit_code == 0
def test_wire_unwire_new_events_cloudwatch_schedule(awsclient, vendored_folder, cleanup_lambdas, cleanup_roles): log.info('running test_wire_unwire_new_events_cloudwatch_schedule') # create a lambda function temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_counter.py', lambda_handler='handler_counter.handle') # schedule expressions: # http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html events = [{ "event_source": { "name": "execute_backup", "schedule": "rate(1 minute)" } }] cleanup_lambdas.append((lambda_name, events)) # wire the function with the bucket exit_code = wire(awsclient, events, lambda_name) assert exit_code == 0 assert int(_get_count(awsclient, lambda_name)) == 0 time.sleep(70) # sleep till scheduled event assert int(_get_count(awsclient, lambda_name)) == 1 # unwire the function exit_code = unwire(awsclient, events, lambda_name) assert exit_code == 0 time.sleep(70) assert int(_get_count(awsclient, lambda_name)) == 1
def test_lambda_add_invoke_permission(awsclient, vendored_folder, temp_bucket, cleanup_lambdas_deprecated, cleanup_roles): log.info('running test_lambda_add_invoke_permission') temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_counter.py', lambda_handler='handler_counter.handle') cleanup_lambdas_deprecated.append(lambda_name) bucket_name = temp_bucket s3_arn = 'arn:aws:s3:::' + bucket_name response = _lambda_add_invoke_permission( awsclient, lambda_name, 's3.amazonaws.com', s3_arn) # {"Statement":"{\"Condition\":{\"ArnLike\":{\"AWS:SourceArn\":\"arn:aws:s3:::unittest-lambda-s3-bucket-coedce\"}},\"Action\":[\"lambda:InvokeFunction\"],\"Resource\":\"arn:aws:lambda:eu-west-1:188084614522:function:jenkins_test_coedce:ACTIVE\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"s3.amazonaws.com\"},\"Sid\":\"07c77fac-68ff-11e6-97f8-c4850848610b\"}"} assert_not_in('Error', response) assert_in('lambda:InvokeFunction', response['Statement'])
def test_update_stack_rolearn(awsclient, simple_cloudformation_stack, temp_cloudformation_policy, cleanup_roles): # create a stack we use for the test lifecycle cloudformation_simple_stack, _ = load_cloudformation_template( here('resources/simple_cloudformation_stack/cloudformation.py') ) # create role to use for cloudformation update role = create_role_helper( awsclient, 'unittest_%s_kumo' % utils.random_string(), policies=[ temp_cloudformation_policy, 'arn:aws:iam::aws:policy/AmazonS3FullAccess' ], principal_service=['cloudformation.amazonaws.com'] ) cleanup_roles.append(role['RoleName']) config_rolearn = deepcopy(config_simple_stack) config_rolearn['stack']['RoleARN'] = role['Arn'] change_set_name, stackname, change_set_type = \ create_change_set(awsclient, {}, config_rolearn, cloudformation_simple_stack) assert stackname == _get_stack_name(config_rolearn) assert change_set_name != '' assert change_set_type == 'UPDATE' describe_change_set(awsclient, change_set_name, stackname) # update the stack changed = get_parameter_diff(awsclient, config_rolearn) assert not changed exit_code = deploy_stack(awsclient, {}, config_rolearn, cloudformation_simple_stack, override_stack_policy=False) assert exit_code == 0
def test_deploy_delete_cmds(awsclient, vendored_folder, cleanup_roles, temp_bucket): log.info('running test_create_lambda') temp_string = utils.random_string() lambda_name = 'jenkins_test_' + temp_string log.info(lambda_name) role = create_role_helper( awsclient, 'unittest_%s_lambda' % temp_string, policies=[ 'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole', 'arn:aws:iam::aws:policy/AWSLambdaExecute' ]) cleanup_roles.append(role['RoleName']) config = { "lambda": { "name": lambda_name, "description": "unittest for ramuda", "role": role['Arn'], "handlerFunction": "handler.handle", "handlerFile": "./resources/sample_lambda/handler.py", "timeout": 300, "memorySize": 256, "events": { "s3Sources": [{ "bucket": temp_bucket, "type": "s3:ObjectCreated:*", "suffix": ".gz" }], "timeSchedules": [{ "ruleName": "infra-dev-sample-lambda-jobr-T1", "ruleDescription": "run every 5 min from 0-5", "scheduleExpression": "cron(0/5 0-5 ? * * *)" }, { "ruleName": "infra-dev-sample-lambda-jobr-T2", "ruleDescription": "run every 5 min from 8-23:59", "scheduleExpression": "cron(0/5 8-23:59 ? * * *)" }] }, "vpc": { "subnetIds": [ "subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-e9db9f9f" ], "securityGroups": ["sg-660dd700"] } }, "bundling": { "zip": "bundle.zip", "folders": [{ "source": "./vendored", "target": "." }, { "source": "./impl", "target": "impl" }] }, "deployment": { "region": "eu-west-1" } } tooldata = get_tooldata(awsclient, 'ramuda', 'deploy', config=config) tooldata['context']['_arguments'] = {'--keep': False} bundle((tooldata['context'], {'ramuda': tooldata['config']})) deploy_cmd(False, **tooldata) # now we use the delete cmd to remove the lambda function tooldata['context']['command'] = 'delete' delete_cmd(True, lambda_name, True, **tooldata)
def test_deprecated_wire_unwire_lambda_with_s3(awsclient, vendored_folder, cleanup_lambdas_deprecated, cleanup_roles, temp_bucket): log.info('running test_wire_unwire_lambda_with_s3') # create a lambda function temp_string = utils.random_string() lambda_name = 'jenkins_test_%s' % temp_string role_name = 'unittest_%s_lambda' % temp_string role_arn = create_lambda_role_helper(awsclient, role_name) cleanup_roles.append(role_name) create_lambda_helper(awsclient, lambda_name, role_arn, './resources/sample_lambda/handler_counter.py', lambda_handler='handler_counter.handle') cleanup_lambdas_deprecated.append(lambda_name) bucket_name = temp_bucket config = { "lambda": { "events": { "s3Sources": [{ "bucket": bucket_name, "type": "s3:ObjectCreated:*", "suffix": ".gz" }] } } } # wire the function with the bucket s3_event_sources = config['lambda'].get('events', []).get('s3Sources', []) time_event_sources = config['lambda'].get('events', []).get('timeSchedules', []) exit_code = wire_deprecated(awsclient, lambda_name, s3_event_sources, time_event_sources) assert exit_code == 0 # put a file into the bucket awsclient.get_client('s3').put_object( ACL='public-read', Body=b'this is some content', Bucket=bucket_name, Key='test_file.gz', ) # validate function call time.sleep(20) # sleep till the event arrived assert int(_get_count(awsclient, lambda_name)) == 1 # unwire the function exit_code = unwire_deprecated(awsclient, lambda_name, s3_event_sources, time_event_sources) assert exit_code == 0 # put in another file awsclient.get_client('s3').put_object( ACL='public-read', Body=b'this is some content', Bucket=bucket_name, Key='test_file_2.gz', ) # validate function not called time.sleep(10) assert int(_get_count(awsclient, lambda_name)) == 1
def test_create_lambda(awsclient, vendored_folder, cleanup_lambdas_deprecated, cleanup_roles): log.info('running test_create_lambda') temp_string = utils.random_string() lambda_name = 'jenkins_test_' + temp_string log.info(lambda_name) role = create_role_helper( awsclient, 'unittest_%s_lambda' % temp_string, policies=[ 'arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole', 'arn:aws:iam::aws:policy/AWSLambdaExecute'] ) cleanup_roles.append(role['RoleName']) config = { "lambda": { "name": "dp-dev-sample-lambda-jobr1", "description": "lambda test for ramuda", "role": "'unused'", "handlerFunction": "handler.handle", "handlerFile": "./resources/sample_lambda/handler.py", "timeout": 300, "memorySize": 256, "events": { "s3Sources": [ { "bucket": "jobr-test", "type": "s3:ObjectCreated:*", "suffix": ".gz" } ], "timeSchedules": [ { "ruleName": "infra-dev-sample-lambda-jobr-T1", "ruleDescription": "run every 5 min from 0-5", "scheduleExpression": "cron(0/5 0-5 ? * * *)" }, { "ruleName": "infra-dev-sample-lambda-jobr-T2", "ruleDescription": "run every 5 min from 8-23:59", "scheduleExpression": "cron(0/5 8-23:59 ? * * *)" } ] }, "vpc": { "subnetIds": [ "subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-d5ffb0b1", "subnet-e9db9f9f" ], "securityGroups": [ "sg-660dd700" ] } }, "bundling": { "zip": "bundle.zip", "folders": [ { "source": "./vendored", "target": "." }, { "source": "./impl", "target": "impl" } ] }, "deployment": { "region": "eu-west-1" } } lambda_description = config['lambda'].get('description') role_arn = role['Arn'] lambda_handler = config['lambda'].get('handlerFunction') handler_filename = config['lambda'].get('handlerFile') timeout = int(config['lambda'].get('timeout')) memory_size = int(config['lambda'].get('memorySize')) zip_name = config['bundling'].get('zip') folders_from_file = config['bundling'].get('folders') subnet_ids = config['lambda'].get('vpc', {}).get('subnetIds', None) security_groups = config['lambda'].get('vpc', {}).get('securityGroups', None) region = config['deployment'].get('region') artifact_bucket = config['deployment'].get('artifactBucket', None) zipfile = get_zipped_file( handler_filename, folders_from_file, ) deploy_lambda( awsclient=awsclient, function_name=lambda_name, role=role_arn, handler_filename=handler_filename, handler_function=lambda_handler, folders=folders_from_file, description=lambda_description, timeout=timeout, memory=memory_size, artifact_bucket=artifact_bucket, zipfile=zipfile ) cleanup_lambdas_deprecated.append(lambda_name)
def test_create_update_stack_artifactbucket(awsclient, temp_cloudformation_policy, cleanup_roles, cleanup_buckets): # create a stack we use for the test lifecycle cloudformation_simple_stack, _ = load_cloudformation_template( here('resources/simple_cloudformation_stack/cloudformation.py') ) upload_conf = { 'stack': { 'StackName': "infra-dev-kumo-sample-stack", 'artifactBucket': "unittest-kumo-artifact-bucket" }, 'parameters': { 'InstanceType': "t2.micro", } } region = awsclient.get_client('s3').meta.region_name account = os.getenv('ACCOUNT', None) # add account prefix to artifact bucket config if account: upload_conf['stack']['artifactBucket'] = \ '%s-unittest-kumo-artifact-bucket' % account artifact_bucket = _get_artifact_bucket(upload_conf) prepare_artifacts_bucket(awsclient, artifact_bucket) cleanup_buckets.append(artifact_bucket) dest_key = 'kumo/%s/%s-cloudformation.json' % (region, _get_stack_name(upload_conf)) expected_s3url = 'https://s3-%s.amazonaws.com/%s/%s' % (region, artifact_bucket, dest_key) actual_s3url = _s3_upload(awsclient, upload_conf, generate_template({}, upload_conf, cloudformation_simple_stack)) assert expected_s3url == actual_s3url # create role to use for cloudformation update role = create_role_helper( awsclient, 'unittest_%s_kumo' % utils.random_string(), policies=[ temp_cloudformation_policy, 'arn:aws:iam::aws:policy/AWSCodeDeployReadOnlyAccess', 'arn:aws:iam::aws:policy/AmazonS3FullAccess' ], principal_service=['cloudformation.amazonaws.com'] ) cleanup_roles.append(role['RoleName']) # create exit_code = deploy_stack(awsclient, {}, upload_conf, cloudformation_simple_stack, override_stack_policy=False) assert exit_code == 0 stack_id = get_stack_id(awsclient, upload_conf['stack']['StackName']) wait_for_stack_create_complete(awsclient, stack_id) # update (as a change we add the RoleARN) upload_conf['stack']['RoleARN'] = role['Arn'] # update the stack changed = get_parameter_diff(awsclient, upload_conf) assert not changed exit_code = deploy_stack(awsclient, {}, upload_conf, cloudformation_simple_stack, override_stack_policy=False) assert exit_code == 0 wait_for_stack_update_complete(awsclient, stack_id) # cleanup exit_code = delete_stack(awsclient, upload_conf) assert exit_code == 0 wait_for_stack_delete_complete(awsclient, stack_id)