def test_load_cloudformation_template_from_cwd(temp_folder):
# prepare dummy template for test
open('cloudformation.py', 'w').write('def plus(a, b):\n return a+b\n')
module, success = load_cloudformation_template()
assert_true(success, True)
assert_equal(module.plus(1, 2), 3)
def test_new_cloudformation_template_hooks():
template_path = here(
'resources/simple_cloudformation_stack_hooks/cloudformation.py')
module, success = load_cloudformation_template(template_path)
assert_equal(success, True)
assert module.COUNTER['register'] == 1
def test_create_stack_rolearn(awsclient, cleanup_stack_simple_stack,
temp_cloudformation_policy, cleanup_roles):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
# create role to use for cloudformation deployment
role = create_role_helper(
awsclient,
'unittest_%s_kumo' % utils.random_string(),
policies=[
temp_cloudformation_policy,
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
principal_service=['cloudformation.amazonaws.com'])
cleanup_roles.append(role['RoleName'])
config_rolearn = deepcopy(config_simple_stack)
config_rolearn['stack']['RoleARN'] = role['Arn']
exit_code = deploy_stack(awsclient, {},
config_rolearn,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
def test_s3_upload(cleanup_buckets, awsclient):
#upload_conf = ConfigFactory.parse_file(
# here('resources/simple_cloudformation_stack/settings_upload_dev.conf')
#)
upload_conf = {
'stack': {
'StackName': "infra-dev-kumo-sample-stack",
'artifactBucket': "unittest-kumo-artifact-bucket"
},
'parameters': {
'InstanceType': "t2.micro"
}
}
region = awsclient.get_client('s3').meta.region_name
account = os.getenv('ACCOUNT', None)
# add account prefix to artifact bucket config
if account:
upload_conf['stack']['artifactBucket'] = \
'%s-unittest-kumo-artifact-bucket' % account
artifact_bucket = _get_artifact_bucket(upload_conf)
prepare_artifacts_bucket(awsclient, artifact_bucket)
cleanup_buckets.append(artifact_bucket)
dest_key = 'kumo/%s/%s-cloudformation.json' % (
region, _get_stack_name(upload_conf))
expected_s3url = 'https://s3-%s.amazonaws.com/%s/%s' % (
region, artifact_bucket, dest_key)
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
actual_s3url = _s3_upload(
awsclient, upload_conf,
generate_template({}, upload_conf, cloudformation_simple_stack))
assert expected_s3url == actual_s3url
def test_load_cloudformation_template(cleanup_tempfiles):
tf = NamedTemporaryFile(delete=False, suffix='py')
open(tf.name, 'w').write('def plus(a, b):\n return a+b')
cleanup_tempfiles.append(tf.name)
module, success = load_cloudformation_template(tf.name)
assert_equal(success, True)
assert_equal(module.plus(1, 2), 3)
def test_generate_template_invalid_arguments():
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation_invalid_arguments.py')
)
context = {'foo': 'bar'}
config_simple_stack = fix_old_kumo_config(read_json_config(
here('resources/simple_cloudformation_stack/gcdt_dev.json')
))['kumo']
with pytest.raises(Exception) as einfo:
generate_template(context, config_simple_stack, cloudformation_simple_stack)
assert einfo.match(r"Arguments of 'generate_template' not as expected: \['invalid_context', 'invalid_config'\]")
def test_output_deployment(cleanup_stack_tenkai, awsclient, logcapture):
logcapture.level = logging.INFO
are_credentials_still_valid(awsclient)
# Set up stack with an ec2 deployment
cloudformation, _ = load_cloudformation_template(
here('resources/sample_codedeploy_app/cloudformation.py'))
exit_code = deploy_stack(awsclient, {},
config_sample_codeploy_stack,
cloudformation,
override_stack_policy=False)
assert_equal(exit_code, 0)
stack_name = _get_stack_name(config_sample_codeploy_stack)
stack_output = get_outputs_for_stack(awsclient, stack_name)
app_name = stack_output.get('ApplicationName', None)
deployment_group = stack_output.get('DeploymentGroupName', None)
not_working_deploy_dir = here(
'./resources/sample_codedeploy_app/not_working')
os.chdir(not_working_deploy_dir)
folders = [{'source': 'codedeploy', 'target': ''}]
# test deployment which should exit with exit code 1
deploy_id_1 = tenkai_deploy(awsclient, app_name, deployment_group,
'CodeDeployDefault.AllAtOnce',
'7finity-infra-dev-deployment',
bundle_revision(folders))
exit_code = output_deployment_status(awsclient, deploy_id_1)
assert exit_code == 1
output_deployment_summary(awsclient, deploy_id_1)
output_deployment_diagnostics(awsclient, deploy_id_1, 'unknown_log_group')
records = list(logcapture.actual())
assert ('gcdt.tenkai_core', 'INFO',
'Instance ID Status Most recent event') in records
#assert ('gcdt.tenkai_core', 'INFO', u'\x1b[35mi-0396d1ca00089c672 \x1b[39m Failed ValidateService') in records
assert ('gcdt.tenkai_core', 'INFO',
u'Error Code: ScriptFailed') in records
assert ('gcdt.tenkai_core', 'INFO', u'Script Name: appspec.sh') in records
assert (
'gcdt.tenkai_core', 'INFO',
'Message: Script at specified location: appspec.sh run as user root failed with exit code 1'
) in records
assert (
'gcdt.tenkai_core', 'INFO',
u'Log Tail: LifecycleEvent - ApplicationStart\nScript - appspec.sh\n[stdout]LIFECYCLE_EVENT=ApplicationStart\n[stderr]mv: cannot stat \u2018not-existing-file.txt\u2019: No such file or directory\n'
) in records
def test_kumo_utils_ensure_autoscaling_ebs_tags(cleanup_stack_autoscaling,
awsclient):
are_credentials_still_valid(awsclient)
cloudformation_autoscaling, _ = load_cloudformation_template(
here(
'resources/sample_autoscaling_cloudformation_stack/cloudformation.py'
))
exit_code = deploy_stack(awsclient,
config_autoscaling,
cloudformation_autoscaling,
override_stack_policy=False)
assert_equal(exit_code, 0)
stack_name = _get_stack_name(config_autoscaling)
stack_output = get_outputs_for_stack(awsclient, stack_name)
as_group_name = stack_output.get('AutoScalingGroupName', None)
assert_is_not_none(as_group_name)
tags_v1 = [{'Key': 'kumo-test', 'Value': 'version1'}]
ensure_ebs_volume_tags_autoscaling_group(awsclient, as_group_name, tags_v1)
autoscale_filter = {
'Name': 'tag:aws:autoscaling:groupName',
'Values': [as_group_name]
}
client_ec2 = awsclient.get_client('ec2')
response = client_ec2.describe_instances(Filters=[autoscale_filter])
for r in response['Reservations']:
for i in r['Instances']:
volumes = client_ec2.describe_volumes(
Filters=[{
'Name': 'attachment.instance-id',
'Values': [i['InstanceId']]
}])
for vol in volumes['Volumes']:
for tag in tags_v1:
assert check_volume_tagged(vol, tag)
tags_v2 = [{'Key': 'kumo-test', 'Value': 'version2'}]
ensure_ebs_volume_tags_autoscaling_group(awsclient, as_group_name, tags_v2)
for r in response['Reservations']:
for i in r['Instances']:
volumes = client_ec2.describe_volumes(
Filters=[{
'Name': 'attachment.instance-id',
'Values': [i['InstanceId']]
}])
for vol in volumes['Volumes']:
for tag in tags_v2:
assert_true(check_volume_tagged(vol, tag))
for tag in tags_v1:
assert_false(check_volume_tagged(vol, tag))
def test_generate_template_with_arguments():
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation_with_arguments.py')
)
context = {'foo': 'bar'}
config_simple_stack = fix_old_kumo_config(read_json_config(
here('resources/simple_cloudformation_stack/gcdt_dev.json')
))['kumo']
expected_template_body = read_json_config(
here('resources/simple_cloudformation_stack/expected_template_body.json')
)
template_body = json.loads(
generate_template(context, config_simple_stack, cloudformation_simple_stack)
)
assert template_body == expected_template_body
def test_describe_change_set_on_new_stack(awsclient):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
change_set_name, stackname, change_set_type = \
create_change_set(awsclient, {}, config_simple_stack,
cloudformation_simple_stack)
assert stackname == _get_stack_name(config_simple_stack)
assert change_set_name != ''
assert change_set_type == 'CREATE'
describe_change_set(awsclient, change_set_name, stackname)
# clean up
# even if we delete the change_Set we need to delete our stack which
# is in state "REVIEW_IN_PROGRESS"
awsclient.get_client('cloudformation').delete_stack(StackName=stackname, )
def test_simple_cloudformation_stack():
# read the template
template_path = here(
'resources/simple_cloudformation_stack/cloudformation.py')
config_path = here(
'resources/simple_cloudformation_stack/settings_dev.conf')
cloudformation, success = load_cloudformation_template(template_path)
assert_true(success)
# read the configuration
config = ConfigFactory.parse_file(config_path)
expected_templ_file_name = '%s-generated-cf-template.json' % \
_get_stack_name(config)
actual = generate_template_file(config, cloudformation)
assert_equal(actual, expected_templ_file_name)
def simple_cloudformation_stack(awsclient):
# create a stack we use for the test lifecycle
#print_parameter_diff(awsclient, config_simple_stack)
are_credentials_still_valid(awsclient)
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
exit_code = deploy_stack(awsclient,
config_simple_stack,
cloudformation_simple_stack,
override_stack_policy=False)
assert not exit_code
yield 'infra-dev-kumo-sample-stack'
# cleanup
exit_code = delete_stack(awsclient, config_simple_stack)
# check whether delete was completed!
assert not exit_code, 'delete_stack was not completed please make sure to clean up the stack manually'
def sample_codedeploy_app(awsclient):
are_credentials_still_valid(awsclient)
# Set up stack with an ec2 and deployment
cloudformation, _ = load_cloudformation_template(
here('resources/sample_codedeploy_app/cloudformation.py'))
exit_code = deploy_stack(awsclient, {},
config_sample_codeploy_stack,
cloudformation,
override_stack_policy=False)
assert_equal(exit_code, 0)
yield
# cleanup
exit_code = delete_stack(awsclient, config_sample_codeploy_stack)
# check whether delete was completed!
assert_false(
exit_code, 'delete_stack was not completed\n' +
'please make sure to clean up the stack manually')
def simple_cloudformation_stack_with_rds(awsclient):
# create a stack we use for the test lifecycle
stack_name = "infra-dev-kumo-sample-stack-with-rds"
are_credentials_still_valid(awsclient)
cloudformation_simple_stack_with_rds, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack_with_rds/cloudformation.py'
))
exit_code = deploy_stack(awsclient, {},
config_rds_stack,
cloudformation_simple_stack_with_rds,
override_stack_policy=False)
assert not exit_code
yield stack_name
# cleanup
exit_code = delete_stack(awsclient, config_rds_stack)
# check whether delete was completed!
assert not exit_code, 'delete_stack was not completed please make sure to clean up the stack manually'
def test_kumo_context_contains_stack_output(awsclient):
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
context = {}
exit_code = deploy_stack(awsclient,
context,
config_simple_stack,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
assert 'stack_output' in context
assert len(context['stack_output']) == 1
assert context['stack_output'][0]['Description'] == 'Name of S3 bucket'
assert context['stack_output'][0]['OutputKey'] == 'BucketName'
assert context['stack_output'][0]['OutputValue'].startswith(
'infra-dev-kumo-sample-stack-s3bucket1')
# cleanup
exit_code = delete_stack(awsclient, config_simple_stack)
assert exit_code == 0
def test_kumo_stack_lifecycle(awsclient, simple_cloudformation_stack):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
# preview (with identical stack)
# TODO: add more asserts!
change_set_name, stackname = \
create_change_set(awsclient, config_simple_stack,
cloudformation_simple_stack)
assert_equal(stackname, _get_stack_name(config_simple_stack))
assert_is_not(change_set_name, '')
describe_change_set(awsclient, change_set_name, stackname)
# update the stack
print_parameter_diff(awsclient, config_simple_stack)
exit_code = deploy_stack(awsclient,
config_simple_stack,
cloudformation_simple_stack,
override_stack_policy=False)
assert_equal(exit_code, 0)
def sample_cloudformation_stack_with_hooks(awsclient):
# create a stack we use for the test lifecycle
are_credentials_still_valid(awsclient)
cloudformation_stack, _ = load_cloudformation_template(
here(
'resources/sample_cloudformation_stack_with_hooks/cloudformation.py'
))
config_stack = read_json_config(
here('resources/sample_cloudformation_stack_with_hooks/gcdt_dev.json')
)['kumo']
exit_code = deploy_stack(awsclient,
config_stack,
cloudformation_stack,
override_stack_policy=False)
assert not exit_code
yield 'infra-dev-kumo-sample-stack-with-hooks'
# cleanup
exit_code = delete_stack(awsclient, config_stack)
# check whether delete was completed!
assert not exit_code, 'delete_stack was not completed please make sure to clean up the stack manually'
def test_tenkai_exit_codes(cleanup_stack_tenkai, awsclient):
are_credentials_still_valid(awsclient)
# Set up stack with an ec2 deployment
cloudformation, _ = load_cloudformation_template(
here('resources/sample_codedeploy_app/cloudformation.py'))
exit_code = deploy_stack(awsclient, {},
config_sample_codeploy_stack,
cloudformation,
override_stack_policy=False)
assert_equal(exit_code, 0)
stack_name = _get_stack_name(config_sample_codeploy_stack)
stack_output = get_outputs_for_stack(awsclient, stack_name)
app_name = stack_output.get('ApplicationName', None)
deployment_group = stack_output.get('DeploymentGroupName', None)
cwd = here('.')
not_working_deploy_dir = here(
'./resources/sample_codedeploy_app/not_working')
working_deploy_dir = here('./resources/sample_codedeploy_app/working')
os.chdir(not_working_deploy_dir)
folders = [{'source': 'codedeploy', 'target': ''}]
# test deployment which should exit with exit code 1
deploy_id_1 = tenkai_deploy(awsclient, app_name, deployment_group,
'CodeDeployDefault.AllAtOnce',
'7finity-infra-dev-deployment',
bundle_revision(folders))
exit_code = output_deployment_status(awsclient, deploy_id_1)
assert exit_code == 1
# test deployment which should exit with exit code 0
os.chdir(working_deploy_dir)
deploy_id_2 = tenkai_deploy(awsclient, app_name, deployment_group,
'CodeDeployDefault.AllAtOnce',
'7finity-infra-dev-deployment',
bundle_revision(folders))
exit_code = output_deployment_status(awsclient, deploy_id_2)
assert exit_code == 0
os.chdir(cwd)
def test_kumo_stack_lifecycle(awsclient, simple_cloudformation_stack):
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
# preview (with identical stack)
# TODO: add more asserts!
change_set_name, stackname, change_set_type = \
create_change_set(awsclient, {}, config_simple_stack,
cloudformation_simple_stack)
assert stackname == _get_stack_name(config_simple_stack)
assert change_set_name != ''
assert change_set_type == 'UPDATE'
describe_change_set(awsclient, change_set_name, stackname)
# update the stack
changed = get_parameter_diff(awsclient, config_simple_stack)
assert not changed
exit_code = deploy_stack(awsclient, {},
config_simple_stack,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
def test_simple_cloudformation_stack():
# read the template
template_path = here(
'resources/simple_cloudformation_stack/cloudformation.py')
#config_path = here(
# 'resources/simple_cloudformation_stack/settings_dev.conf')
cloudformation, success = load_cloudformation_template(template_path)
assert_true(success)
config = {
'stack': {
'StackName': "infra-dev-kumo-sample-stack"
},
'parameters': {
'InstanceType': "t2.micro"
}
}
expected_templ_file_name = '%s-generated-cf-template.json' % \
_get_stack_name(config)
actual = write_template_to_file(config, generate_template({}, config, cloudformation))
assert_equal(actual, expected_templ_file_name)
def test_s3_upload(cleanup_buckets, awsclient):
upload_conf = ConfigFactory.parse_file(
here('resources/simple_cloudformation_stack/settings_upload_dev.conf'))
region = awsclient.get_client('s3').meta.region_name
account = os.getenv('ACCOUNT', None)
# add account prefix to artifact bucket config
if account:
upload_conf['cloudformation']['artifactBucket'] = \
'%s-unittest-kumo-artifact-bucket' % account
artifact_bucket = _get_artifact_bucket(upload_conf)
prepare_artifacts_bucket(awsclient, artifact_bucket)
cleanup_buckets.append(artifact_bucket)
dest_key = 'kumo/%s/%s-cloudformation.json' % (
region, _get_stack_name(upload_conf))
expected_s3url = 'https://s3-%s.amazonaws.com/%s/%s' % (
region, artifact_bucket, dest_key)
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
actual_s3url = _s3_upload(awsclient, upload_conf,
cloudformation_simple_stack)
assert expected_s3url == actual_s3url
def test_update_stack_rolearn(awsclient, simple_cloudformation_stack,
temp_cloudformation_policy, cleanup_roles):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
# create role to use for cloudformation update
role = create_role_helper(
awsclient,
'unittest_%s_kumo' % utils.random_string(),
policies=[
temp_cloudformation_policy,
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
principal_service=['cloudformation.amazonaws.com'])
cleanup_roles.append(role['RoleName'])
config_rolearn = deepcopy(config_simple_stack)
config_rolearn['stack']['RoleARN'] = role['Arn']
change_set_name, stackname, change_set_type = \
create_change_set(awsclient, {}, config_rolearn,
cloudformation_simple_stack)
assert stackname == _get_stack_name(config_rolearn)
assert change_set_name != ''
assert change_set_type == 'UPDATE'
describe_change_set(awsclient, change_set_name, stackname)
# update the stack
changed = get_parameter_diff(awsclient, config_rolearn)
assert not changed
exit_code = deploy_stack(awsclient, {},
config_rolearn,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
def test_kumo_utils_ensure_ebs_tags(cleanup_stack_ec2, awsclient):
are_credentials_still_valid(awsclient)
cloudformation_ec2, _ = load_cloudformation_template(
here('resources/sample_ec2_cloudformation_stack/cloudformation.py'))
exit_code = deploy_stack(awsclient,
config_ec2,
cloudformation_ec2,
override_stack_policy=False)
assert_equal(exit_code, 0)
stack_name = _get_stack_name(config_ec2)
stack_output = get_outputs_for_stack(awsclient, stack_name)
instance_id = stack_output.get('InstanceId', None)
assert_is_not_none(instance_id)
tags = [{'Key': 'kumo-test', 'Value': 'Success'}]
ensure_ebs_volume_tags_ec2_instance(awsclient, instance_id, tags)
client_ec2 = awsclient.get_client('ec2')
volumes = client_ec2.describe_volumes(Filters=[{
'Name': 'attachment.instance-id',
'Values': [instance_id]
}])
for vol in volumes['Volumes']:
for tag in tags:
assert_true(check_volume_tagged(vol, tag))
def test_create_update_stack_artifactbucket(awsclient,
temp_cloudformation_policy,
cleanup_roles, cleanup_buckets):
# create a stack we use for the test lifecycle
cloudformation_simple_stack, _ = load_cloudformation_template(
here('resources/simple_cloudformation_stack/cloudformation.py'))
upload_conf = {
'stack': {
'StackName': "infra-dev-kumo-sample-stack",
'artifactBucket': "unittest-kumo-artifact-bucket"
},
'parameters': {
'InstanceType': "t2.micro",
}
}
region = awsclient.get_client('s3').meta.region_name
account = os.getenv('ACCOUNT', None)
# add account prefix to artifact bucket config
if account:
upload_conf['stack']['artifactBucket'] = \
'%s-unittest-kumo-artifact-bucket' % account
artifact_bucket = _get_artifact_bucket(upload_conf)
prepare_artifacts_bucket(awsclient, artifact_bucket)
cleanup_buckets.append(artifact_bucket)
dest_key = 'kumo/%s/%s-cloudformation.json' % (
region, _get_stack_name(upload_conf))
expected_s3url = 'https://s3-%s.amazonaws.com/%s/%s' % (
region, artifact_bucket, dest_key)
actual_s3url = _s3_upload(
awsclient, upload_conf,
generate_template({}, upload_conf, cloudformation_simple_stack))
assert expected_s3url == actual_s3url
# create role to use for cloudformation update
role = create_role_helper(
awsclient,
'unittest_%s_kumo' % utils.random_string(),
policies=[
temp_cloudformation_policy,
'arn:aws:iam::aws:policy/AWSCodeDeployReadOnlyAccess',
'arn:aws:iam::aws:policy/AmazonS3FullAccess'
],
principal_service=['cloudformation.amazonaws.com'])
cleanup_roles.append(role['RoleName'])
# create
exit_code = deploy_stack(awsclient, {},
upload_conf,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
stack_id = get_stack_id(awsclient, upload_conf['stack']['StackName'])
wait_for_stack_create_complete(awsclient, stack_id)
# update (as a change we add the RoleARN)
upload_conf['stack']['RoleARN'] = role['Arn']
# update the stack
changed = get_parameter_diff(awsclient, upload_conf)
assert not changed
exit_code = deploy_stack(awsclient, {},
upload_conf,
cloudformation_simple_stack,
override_stack_policy=False)
assert exit_code == 0
wait_for_stack_update_complete(awsclient, stack_id)
# cleanup
exit_code = delete_stack(awsclient, upload_conf)
assert exit_code == 0
wait_for_stack_delete_complete(awsclient, stack_id)
def test_cloudformation_template_not_available():
module, success = load_cloudformation_template()
assert_equal(module, None)
assert_equal(success, False)
