def test_resolve_referenced_resource(monkeypatch): boto3 = MagicMock() resource = { 'StackResourceDetail': { 'ResourceStatus': 'CREATE_COMPLETE', 'ResourceType': 'AWS::EC2::Something', 'PhysicalResourceId': 'some-resource' } } boto3.describe_stack_resource.return_value = resource monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) ref = {'Fn::GetAtt': ['RefSecGroup', 'GroupId']} assert ref == resolve_referenced_resource(ref, 'region') ref = {'Stack': 'stack', 'LogicalId': 'id'} assert 'some-resource' == resolve_referenced_resource(ref, 'region') resource['StackResourceDetail']['ResourceStatus'] = 'CREATE_IN_PROGRESS' try: resolve_referenced_resource(ref, 'region') except ValueError: pass else: assert False, "resolving referenced resource failed"
def test_resolve_referenced_resource(monkeypatch): boto3 = MagicMock() resource = { "StackResourceDetail": { "ResourceStatus": "CREATE_COMPLETE", "ResourceType": "AWS::EC2::Something", "PhysicalResourceId": "some-resource", } } boto3.describe_stack_resource.return_value = resource stack = {"StackStatus": "CREATE_COMPLETE", "Outputs": [{"OutputKey": "DatabaseHost", "OutputValue": "localhost"}]} boto3.describe_stacks.return_value = {"Stacks": [stack]} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3)) ref = {"Fn::GetAtt": ["RefSecGroup", "GroupId"]} assert ref == resolve_referenced_resource(ref, "region") ref = {"Stack": "stack", "LogicalId": "id"} assert "some-resource" == resolve_referenced_resource(ref, "region") resource["StackResourceDetail"]["ResourceStatus"] = "CREATE_IN_PROGRESS" try: resolve_referenced_resource(ref, "region") except ValueError: pass else: assert False, "resolving referenced resource failed" ref = {"Stack": "stack", "Output": "DatabaseHost"} assert "localhost" == resolve_referenced_resource(ref, "region") stack["StackStatus"] = "CREATE_IN_PROGRESS" try: resolve_referenced_resource(ref, "region") except ValueError: pass else: assert False, "resolving referenced resource failed" stack["StackStatus"] = "CREATE_COMPLETE" del stack["Outputs"] assert resolve_referenced_resource(ref, "region") is None stack["Outputs"] = [] assert resolve_referenced_resource(ref, "region") is None
def test_resolve_referenced_resource(monkeypatch): boto3 = MagicMock() resource = {'StackResourceDetail': {'ResourceStatus':'CREATE_COMPLETE', 'ResourceType': 'AWS::EC2::Something', 'PhysicalResourceId':'some-resource'}} boto3.describe_stack_resource.return_value = resource monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) ref = {'Fn::GetAtt': ['RefSecGroup', 'GroupId']} assert ref == resolve_referenced_resource(ref, 'region') ref = {'Stack': 'stack', 'LogicalId': 'id'} assert 'some-resource' == resolve_referenced_resource(ref, 'region') resource['StackResourceDetail']['ResourceStatus'] = 'CREATE_IN_PROGRESS' try: resolve_referenced_resource(ref, 'region') except ValueError: pass else: assert False, "resolving referenced resource failed"
def test_resolve_referenced_output_when_stack_is_in_update_complete_status(monkeypatch): output_value = 'some-resource' output_key = 'some-key' boto3 = MagicMock() boto3.describe_stacks.return_value = { 'Stacks': [ {'StackStatus': 'UPDATE_COMPLETE', 'Outputs': [{'OutputKey': output_key, 'OutputValue': output_value}]} ] } monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) ref = {'Stack': 'stack', 'Output': output_key} assert output_value == resolve_referenced_resource(ref, 'any-region')
def test_resolve_referenced_output_when_stack_is_in_update_complete_status( monkeypatch): output_value = 'some-resource' output_key = 'some-key' boto3 = MagicMock() boto3.describe_stacks.return_value = { 'Stacks': [ {'StackStatus': 'UPDATE_COMPLETE', 'Outputs': [ {'OutputKey': output_key, 'OutputValue': output_value}]} ] } monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) ref = {'Stack': 'stack', 'Output': output_key} assert output_value == resolve_referenced_resource(ref, 'any-region')
def test_resolve_referenced_resource_with_update_complete_status(monkeypatch): resource_id = 'some-resource' boto3 = MagicMock() boto3.describe_stack_resource.return_value = { 'StackResourceDetail': { 'ResourceStatus': 'UPDATE_COMPLETE', 'ResourceType': 'AWS::EC2::Something', 'PhysicalResourceId': resource_id } } boto3.describe_stacks.return_value = {'Stacks': [{'StackStatus': 'CREATE_COMPLETE'}]} monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) ref = {'Stack': 'stack', 'LogicalId': 'id'} assert resource_id == resolve_referenced_resource(ref, 'any-region')
def test_resolve_referenced_resource_with_update_complete_status(monkeypatch): resource_id = 'some-resource' boto3 = MagicMock() boto3.describe_stack_resource.return_value = { 'StackResourceDetail': { 'ResourceStatus': 'UPDATE_COMPLETE', 'ResourceType': 'AWS::EC2::Something', 'PhysicalResourceId': resource_id } } boto3.describe_stacks.return_value = { 'Stacks': [{'StackStatus': 'CREATE_COMPLETE'}]} monkeypatch.setattr('boto3.client', MagicMock(return_value=boto3)) ref = {'Stack': 'stack', 'LogicalId': 'id'} assert resource_id == resolve_referenced_resource(ref, 'any-region')
def handle_iam_roles(definition, configuration, args): """ This function resolves Senza's IAMRoles attribute and creates the CF InstanceProfile resources """ logical_id = configuration["Name"] + "InstanceProfile" roles = configuration.pop("IamRoles") if len(roles) > 1: for role in roles: if isinstance(role, dict): raise click.UsageError( 'Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' + 'You can use at most one IAM role with "Ref".') logical_role_id = configuration["Name"] + "Role" definition["Resources"][logical_role_id] = { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"], }], }, "Path": "/", "Policies": get_merged_policies(roles), }, } instance_profile_roles = [{"Ref": logical_role_id}] elif isinstance(roles[0], dict): instance_profile_roles = [ resolve_referenced_resource(roles[0], args.region) ] else: instance_profile_roles = roles definition["Resources"][logical_id] = { "Type": "AWS::IAM::InstanceProfile", "Properties": { "Path": "/", "Roles": instance_profile_roles }, } return logical_id
def handle_iam_roles(definition, configuration, args): """ This function resolves Senza's IAMRoles attribute and creates the CF InstanceProfile resources """ logical_id = configuration['Name'] + 'InstanceProfile' roles = configuration.pop("IamRoles") if len(roles) > 1: for role in roles: if isinstance(role, dict): raise click.UsageError('Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' + 'You can use at most one IAM role with "Ref".') logical_role_id = configuration['Name'] + 'Role' definition['Resources'][logical_role_id] = { 'Type': 'AWS::IAM::Role', 'Properties': { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"] } ] }, 'Path': '/', 'Policies': get_merged_policies(roles) } } instance_profile_roles = [{'Ref': logical_role_id}] elif isinstance(roles[0], dict): instance_profile_roles = [resolve_referenced_resource(roles[0], args.region)] else: instance_profile_roles = roles definition['Resources'][logical_id] = { 'Type': 'AWS::IAM::InstanceProfile', 'Properties': { 'Path': '/', 'Roles': instance_profile_roles } } return logical_id
def transform(node): """Transform AWS functions and refs into an string representation for later split and substitution""" if isinstance(node, dict): num_keys = len(node) if 'Stack' in node and 'Output' in node: return resolve_referenced_resource(node, region) if num_keys > 0: key = next(iter(node.keys())) if num_keys == 1 and is_aws_fn(key): return "".join(["{{ ", json.dumps(node), " }}"]) else: return {key: transform(value) for key, value in node.items()} else: return node elif isinstance(node, list): return [transform(subnode) for subnode in node] else: return node
def component_auto_scaling_group(definition, configuration, args, info, force, account_info): definition = ensure_keys(definition, "Resources") # launch configuration config_name = configuration["Name"] + "Config" definition["Resources"][config_name] = { "Type": "AWS::AutoScaling::LaunchConfiguration", "Properties": { "InstanceType": configuration["InstanceType"], "ImageId": {"Fn::FindInMap": ["Images", {"Ref": "AWS::Region"}, configuration["Image"]]}, "AssociatePublicIpAddress": configuration.get('AssociatePublicIpAddress', False), "EbsOptimized": configuration.get('EbsOptimized', False) } } if 'BlockDeviceMappings' in configuration: definition['Resources'][config_name]['Properties']['BlockDeviceMappings'] = configuration['BlockDeviceMappings'] if "IamInstanceProfile" in configuration: definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = configuration["IamInstanceProfile"] if 'IamRoles' in configuration: logical_id = configuration['Name'] + 'InstanceProfile' roles = configuration['IamRoles'] if len(roles) > 1: for role in roles: if isinstance(role, dict): raise click.UsageError('Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' + 'You can use at most one IAM role with "Ref".') logical_role_id = configuration['Name'] + 'Role' definition['Resources'][logical_role_id] = { 'Type': 'AWS::IAM::Role', 'Properties': { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"] } ] }, 'Path': '/', 'Policies': get_merged_policies(roles) } } instance_profile_roles = [{'Ref': logical_role_id}] elif isinstance(roles[0], dict): instance_profile_roles = [resolve_referenced_resource(roles[0], args.region)] else: instance_profile_roles = roles definition['Resources'][logical_id] = { 'Type': 'AWS::IAM::InstanceProfile', 'Properties': { 'Path': '/', 'Roles': instance_profile_roles } } definition["Resources"][config_name]["Properties"]["IamInstanceProfile"] = {'Ref': logical_id} if "SecurityGroups" in configuration: definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \ resolve_security_groups(configuration["SecurityGroups"], args.region) if "UserData" in configuration: definition["Resources"][config_name]["Properties"]["UserData"] = { "Fn::Base64": configuration["UserData"] } # auto scaling group asg_name = configuration["Name"] asg_success = ["1", "PT15M"] if "AutoScaling" in configuration: if "SuccessRequires" in configuration["AutoScaling"]: asg_success = normalize_asg_success(configuration["AutoScaling"]["SuccessRequires"]) definition["Resources"][asg_name] = { "Type": "AWS::AutoScaling::AutoScalingGroup", # wait to get a signal from an amount of servers to signal that it booted "CreationPolicy": { "ResourceSignal": { "Count": asg_success[0], "Timeout": asg_success[1] } }, "Properties": { # for our operator some notifications "LaunchConfigurationName": {"Ref": config_name}, "VPCZoneIdentifier": {"Fn::FindInMap": ["ServerSubnets", {"Ref": "AWS::Region"}, "Subnets"]}, "Tags": [ # Tag "Name" { "Key": "Name", "PropagateAtLaunch": True, "Value": "{0}-{1}".format(info["StackName"], info["StackVersion"]) }, # Tag "StackName" { "Key": "StackName", "PropagateAtLaunch": True, "Value": info["StackName"], }, # Tag "StackVersion" { "Key": "StackVersion", "PropagateAtLaunch": True, "Value": info["StackVersion"] } ] } } asg_properties = definition["Resources"][asg_name]["Properties"] if "OperatorTopicId" in info: asg_properties["NotificationConfiguration"] = { "NotificationTypes": [ "autoscaling:EC2_INSTANCE_LAUNCH", "autoscaling:EC2_INSTANCE_LAUNCH_ERROR", "autoscaling:EC2_INSTANCE_TERMINATE", "autoscaling:EC2_INSTANCE_TERMINATE_ERROR" ], "TopicARN": resolve_topic_arn(args.region, info["OperatorTopicId"]) } default_health_check_type = 'EC2' if "ElasticLoadBalancer" in configuration: if isinstance(configuration["ElasticLoadBalancer"], str): asg_properties["LoadBalancerNames"] = [{"Ref": configuration["ElasticLoadBalancer"]}] elif isinstance(configuration["ElasticLoadBalancer"], list): asg_properties["LoadBalancerNames"] = [{'Ref': ref} for ref in configuration["ElasticLoadBalancer"]] # use ELB health check by default default_health_check_type = 'ELB' asg_properties['HealthCheckType'] = configuration.get('HealthCheckType', default_health_check_type) asg_properties['HealthCheckGracePeriod'] = configuration.get('HealthCheckGracePeriod', 300) if "AutoScaling" in configuration: as_conf = configuration["AutoScaling"] asg_properties["MaxSize"] = as_conf["Maximum"] asg_properties["MinSize"] = as_conf["Minimum"] asg_properties["DesiredCapacity"] = max(int(as_conf["Minimum"]), int(as_conf.get('DesiredCapacity', 1))) scaling_adjustment = int(as_conf.get("ScalingAdjustment", 1)) # ScaleUp policy definition["Resources"][asg_name + "ScaleUp"] = { "Type": "AWS::AutoScaling::ScalingPolicy", "Properties": { "AdjustmentType": "ChangeInCapacity", "ScalingAdjustment": str(scaling_adjustment), "Cooldown": str(as_conf.get("Cooldown", "60")), "AutoScalingGroupName": { "Ref": asg_name } } } # ScaleDown policy definition["Resources"][asg_name + "ScaleDown"] = { "Type": "AWS::AutoScaling::ScalingPolicy", "Properties": { "AdjustmentType": "ChangeInCapacity", "ScalingAdjustment": str((-1) * scaling_adjustment), "Cooldown": str(as_conf.get("Cooldown", "60")), "AutoScalingGroupName": { "Ref": asg_name } } } if "MetricType" in as_conf: metric_type = as_conf["MetricType"] metricfns = { "CPU": metric_cpu, "NetworkIn": metric_network, "NetworkOut": metric_network } # lowercase cpu is an acceptable metric, be compatible if metric_type.lower() not in map(lambda t: t.lower(), metricfns.keys()): raise click.UsageError('Auto scaling MetricType "{}" not supported.'.format(metric_type)) metricfn = metricfns[metric_type] definition = metricfn(asg_name, definition, as_conf, args, info, force) else: asg_properties["MaxSize"] = 1 asg_properties["MinSize"] = 1 return definition
def component_auto_scaling_group(definition, configuration, args, info, force, account_info): definition = ensure_keys(definition, "Resources") # launch configuration config_name = configuration["Name"] + "Config" definition["Resources"][config_name] = { "Type": "AWS::AutoScaling::LaunchConfiguration", "Properties": { "InstanceType": configuration["InstanceType"], "ImageId": { "Fn::FindInMap": ["Images", { "Ref": "AWS::Region" }, configuration["Image"]] }, "AssociatePublicIpAddress": configuration.get('AssociatePublicIpAddress', False), "EbsOptimized": configuration.get('EbsOptimized', False) } } if 'IamRoles' in configuration: logical_id = configuration['Name'] + 'InstanceProfile' roles = configuration['IamRoles'] if len(roles) > 1: for role in roles: if isinstance(role, dict): raise click.UsageError( 'Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' + 'You can use at most one IAM role with "Ref".') logical_role_id = configuration['Name'] + 'Role' definition['Resources'][logical_role_id] = { 'Type': 'AWS::IAM::Role', 'Properties': { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"] }] }, 'Path': '/', 'Policies': get_merged_policies(roles) } } instance_profile_roles = [{'Ref': logical_role_id}] elif isinstance(roles[0], dict): instance_profile_roles = [ resolve_referenced_resource(roles[0], args.region) ] else: instance_profile_roles = roles definition['Resources'][logical_id] = { 'Type': 'AWS::IAM::InstanceProfile', 'Properties': { 'Path': '/', 'Roles': instance_profile_roles } } definition["Resources"][config_name]["Properties"][ "IamInstanceProfile"] = { 'Ref': logical_id } if "SecurityGroups" in configuration: definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \ resolve_security_groups(configuration["SecurityGroups"], args.region) if "UserData" in configuration: definition["Resources"][config_name]["Properties"]["UserData"] = { "Fn::Base64": configuration["UserData"] } # auto scaling group asg_name = configuration["Name"] asg_success = ["1", "PT15M"] if "AutoScaling" in configuration: if "SuccessRequires" in configuration["AutoScaling"]: asg_success = normalize_asg_success( configuration["AutoScaling"]["SuccessRequires"]) tags = [ # Tag "Name" { "Key": "Name", "PropagateAtLaunch": True, "Value": "{0}-{1}".format(info["StackName"], info["StackVersion"]) }, # Tag "StackName" { "Key": "StackName", "PropagateAtLaunch": True, "Value": info["StackName"], }, # Tag "StackVersion" { "Key": "StackVersion", "PropagateAtLaunch": True, "Value": info["StackVersion"] } ] if "Tags" in configuration: for tag in configuration["Tags"]: tags.append({ "Key": tag["Key"], "PropagateAtLaunch": True, "Value": tag["Value"] }) definition["Resources"][asg_name] = { "Type": "AWS::AutoScaling::AutoScalingGroup", # wait to get a signal from an amount of servers to signal that it booted "CreationPolicy": { "ResourceSignal": { "Count": asg_success[0], "Timeout": asg_success[1] } }, "Properties": { # for our operator some notifications "LaunchConfigurationName": { "Ref": config_name }, "VPCZoneIdentifier": { "Fn::FindInMap": ["ServerSubnets", { "Ref": "AWS::Region" }, "Subnets"] }, "Tags": tags } } asg_properties = definition["Resources"][asg_name]["Properties"] if "OperatorTopicId" in info: asg_properties["NotificationConfiguration"] = { "NotificationTypes": [ "autoscaling:EC2_INSTANCE_LAUNCH", "autoscaling:EC2_INSTANCE_LAUNCH_ERROR", "autoscaling:EC2_INSTANCE_TERMINATE", "autoscaling:EC2_INSTANCE_TERMINATE_ERROR" ], "TopicARN": resolve_topic_arn(args.region, info["OperatorTopicId"]) } default_health_check_type = 'EC2' if "ElasticLoadBalancer" in configuration: if isinstance(configuration["ElasticLoadBalancer"], str): asg_properties["LoadBalancerNames"] = [{ "Ref": configuration["ElasticLoadBalancer"] }] elif isinstance(configuration["ElasticLoadBalancer"], list): asg_properties["LoadBalancerNames"] = [{ 'Ref': ref } for ref in configuration["ElasticLoadBalancer"]] # use ELB health check by default default_health_check_type = 'ELB' if "ElasticLoadBalancerV2" in configuration: if isinstance(configuration["ElasticLoadBalancerV2"], str): asg_properties["TargetGroupARNs"] = [{ "Ref": configuration["ElasticLoadBalancerV2"] + 'TargetGroup' }] elif isinstance(configuration["ElasticLoadBalancerV2"], list): asg_properties["TargetGroupARNs"] = [{ 'Ref': ref + 'TargetGroup' } for ref in configuration["ElasticLoadBalancerV2"]] # use ELB health check by default default_health_check_type = 'ELB' asg_properties['HealthCheckType'] = configuration.get( 'HealthCheckType', default_health_check_type) asg_properties['HealthCheckGracePeriod'] = configuration.get( 'HealthCheckGracePeriod', 300) if "AutoScaling" in configuration: as_conf = configuration["AutoScaling"] asg_properties["MaxSize"] = as_conf["Maximum"] asg_properties["MinSize"] = as_conf["Minimum"] asg_properties["DesiredCapacity"] = max( int(as_conf["Minimum"]), int(as_conf.get('DesiredCapacity', 1))) default_scaling_adjustment = as_conf.get("ScalingAdjustment", 1) default_cooldown = as_conf.get("Cooldown", "60") # ScaleUp policy scale_up_name = asg_name + "ScaleUp" scale_up_adjustment = int( as_conf.get("ScaleUpAdjustment", default_scaling_adjustment)) scale_up_cooldown = as_conf.get("ScaleUpCooldown", default_cooldown) definition["Resources"][scale_up_name] = create_autoscaling_policy( asg_name, scale_up_name, scale_up_adjustment, scale_up_cooldown, definition) # ScaleDown policy scale_down_name = asg_name + "ScaleDown" scale_down_adjustment = (-1) * int( as_conf.get("ScaleDownAdjustment", default_scaling_adjustment)) scale_down_cooldown = as_conf.get("ScaleDownCooldown", default_cooldown) definition["Resources"][scale_down_name] = create_autoscaling_policy( asg_name, scale_down_name, scale_down_adjustment, scale_down_cooldown, definition) if "MetricType" in as_conf: metric_type = as_conf["MetricType"] metricfns = { "CPU": metric_cpu, "NetworkIn": metric_network, "NetworkOut": metric_network } # lowercase cpu is an acceptable metric, be compatible if metric_type.lower() not in map(lambda t: t.lower(), metricfns.keys()): raise click.UsageError( 'Auto scaling MetricType "{}" not supported.'.format( metric_type)) metricfn = metricfns[metric_type] definition = metricfn(asg_name, definition, as_conf, args, info, force) else: asg_properties["MaxSize"] = 1 asg_properties["MinSize"] = 1 for res in (config_name, asg_name): props = definition['Resources'][res]['Properties'] additional_cf_properties = ADDITIONAL_PROPERTIES.get( definition['Resources'][res]['Type']) properties_allowed_to_overwrite = ( set(props.keys()) - SENZA_PROPERTIES) | additional_cf_properties for key in properties_allowed_to_overwrite: if key in configuration: props[key] = configuration[key] return definition