Beispiel #1
0
def create_efs_stack(settings, new_resources):
    """
    Function to create the root stack and add EFS FS.

    :param ecs_composex.common.settings.ComposeXSettings settings:
    :param list new_resources:
    :return: Root template for EFS
    :rtype: troposphere.Template
    """
    template = build_template("Root for EFS built by ECS Compose-X", [FS_PORT])
    for res in new_resources:
        res_cfn_props = import_record_properties(res.properties, FileSystem)
        res.cfn_resource = FileSystem(res.logical_name, **res_cfn_props)
        res.db_sg = SecurityGroup(
            f"{res.logical_name}SecurityGroup",
            GroupName=Sub(f"{res.logical_name}EfsSg"),
            GroupDescription=Sub(f"SG for EFS {res.cfn_resource.title}"),
            VpcId=Ref(VPC_ID),
        )
        template.add_resource(res.cfn_resource)
        template.add_resource(res.db_sg)
        res.init_outputs()
        res.generate_outputs()
        template.add_output(res.outputs)
    return template
Beispiel #2
0
def efs_stack(cfn_stacks_factory, request, region):
    """EFS stack contains a single efs resource."""
    efs_template = Template()
    efs_template.set_version("2010-09-09")
    efs_template.set_description("EFS stack created for testing existing EFS")
    efs_template.add_resource(FileSystem("FileSystemResource"))
    stack = CfnStack(
        name=generate_stack_name("integ-tests-efs",
                                 request.config.getoption("stackname_suffix")),
        region=region,
        template=efs_template.to_json(),
    )
    cfn_stacks_factory.create_stack(stack)

    yield stack

    if not request.config.getoption("no_delete"):
        cfn_stacks_factory.delete_stack(stack.name, region)
Beispiel #3
0
    FromPort='2049',
    ToPort='2049',
    SourceSecurityGroupId=Ref(efs_host_security_group))

# Security group that's applied to the Mount Targets.
efs_security_group = SecurityGroup(
    "SecurityGroup",
    SecurityGroupIngress=[efs_security_group_rule],
    VpcId=Ref(vpcid_param),
    GroupDescription="Allow NFS over TCP")
template.add_resource(efs_security_group)

# Create FileSystem. This is the actual filesystem, which has one or more
# mount targets. Give it some tags so we can identify it later.
tags = Tags(Name='MyEFSFileSystem')
efs_file_system = FileSystem("MyEFSFileSystem", FileSystemTags=tags)
template.add_resource(efs_file_system)

# create MountTarget. You really want a mount target in each subnet where
# it's required, but for the purpose of this example we'll
# put it in just one.
efs_mount_target = MountTarget("MyEFSMountTarget",
                               FileSystemId=Ref(efs_file_system),
                               SecurityGroups=[Ref(efs_security_group)],
                               SubnetId=Ref(subnetid_param))
template.add_resource(efs_mount_target)

# Create the policy that allows the instance to describe file systems and tags,
# so it can lookup the file system using AWS tags. An alternative would be to
# pass in the FileSystem name as UserData.
efs_host_role = Role(
Beispiel #4
0
)

# Security group that's applied to the Mount Targets.
efs_security_group = SecurityGroup(
    "SecurityGroup",
    SecurityGroupIngress=[Ref(efs_security_group_rule)],
    VpcId=Ref(vpcid_param),
    GroupDescription="Allow NFS over TCP"
)
template.add_resource(efs_security_group)

# Create FileSystem. This is the actual filesystem, which has one or more
# mount targets. Give it some tags so we can identify it later.
tags = Tags(Name='MyEFSFileSystem')
efs_file_system = FileSystem(
    title='MyEFSFileSystem',
    FileSystemTags=tags
)
template.add_resource(efs_file_system)

# create MountTarget. You really want a mount target in each subnet where
# it's required, but for the purpose of this example we'll
# put it in just one.
efs_mount_target = MountTarget(
    title='MyEFSMountTarget',
    FileSystemId=Ref(efs_file_system),
    SecurityGroups=[Ref(efs_security_group)],
    SubnetId=Ref(subnetid_param)
)
template.add_resource(efs_mount_target)

# Create the policy that allows the instance to describe file systems and tags,
Beispiel #5
0
def main(args):
    t = Template()

    # [0 shared_dir, 1 efs_fs_id, 2 performance_mode, 3 efs_kms_key_id,
    # 4 provisioned_throughput, 5 encrypted, 6 throughput_mode, 7 exists_valid_head_node_mt, 8 exists_valid_compute_mt]
    efs_options = t.add_parameter(
        Parameter(
            "EFSOptions",
            Type="CommaDelimitedList",
            Description="Comma separated list of efs related options, 9 parameters in total",
        )
    )
    compute_security_group = t.add_parameter(
        Parameter("ComputeSecurityGroup", Type="String", Description="Security Group for Mount Target")
    )
    head_node_subnet_id = t.add_parameter(
        Parameter("MasterSubnetId", Type="String", Description="Head node subnet id for head node mount target")
    )
    compute_subnet_id = t.add_parameter(
        Parameter(
            "ComputeSubnetId",
            Type="String",
            Description="User provided compute subnet id. Will be use to create compute mount target if needed.",
        )
    )

    create_efs = t.add_condition(
        "CreateEFS",
        And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")), Equals(Select(str(1), Ref(efs_options)), "NONE")),
    )
    create_head_node_mt = t.add_condition(
        "CreateMasterMT",
        And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")), Equals(Select(str(7), Ref(efs_options)), "NONE")),
    )
    no_mt_in_compute_az = t.add_condition("NoMTInComputeAZ", Equals(Select(str(8), Ref(efs_options)), "NONE"))
    use_user_provided_compute_subnet = t.add_condition(
        "UseUserProvidedComputeSubnet", Not(Equals(Ref(compute_subnet_id), "NONE"))
    )
    # Need to create compute mount target if:
    # user is providing a compute subnet and
    # there is no existing MT in compute subnet's AZ(includes case where head node AZ == compute AZ).
    #
    # If user is not providing a compute subnet, either we are using the head node subnet as compute subnet,
    # or we will be creating a compute subnet that is in the same AZ as head node subnet,
    # see ComputeSubnet resource in the main stack.
    # In both cases no compute MT is needed.
    create_compute_mt = t.add_condition(
        "CreateComputeMT", And(Condition(use_user_provided_compute_subnet), Condition(no_mt_in_compute_az))
    )

    use_performance_mode = t.add_condition("UsePerformanceMode", Not(Equals(Select(str(2), Ref(efs_options)), "NONE")))
    use_efs_encryption = t.add_condition("UseEFSEncryption", Equals(Select(str(5), Ref(efs_options)), "true"))
    use_efs_kms_key = t.add_condition(
        "UseEFSKMSKey", And(Condition(use_efs_encryption), Not(Equals(Select(str(3), Ref(efs_options)), "NONE")))
    )
    use_throughput_mode = t.add_condition("UseThroughputMode", Not(Equals(Select(str(6), Ref(efs_options)), "NONE")))
    use_provisioned = t.add_condition("UseProvisioned", Equals(Select(str(6), Ref(efs_options)), "provisioned"))
    use_provisioned_throughput = t.add_condition(
        "UseProvisionedThroughput",
        And(Condition(use_provisioned), Not(Equals(Select(str(4), Ref(efs_options)), "NONE"))),
    )

    fs = t.add_resource(
        FileSystem(
            "EFSFS",
            PerformanceMode=If(use_performance_mode, Select(str(2), Ref(efs_options)), NoValue),
            ProvisionedThroughputInMibps=If(use_provisioned_throughput, Select(str(4), Ref(efs_options)), NoValue),
            ThroughputMode=If(use_throughput_mode, Select(str(6), Ref(efs_options)), NoValue),
            Encrypted=If(use_efs_encryption, Select(str(5), Ref(efs_options)), NoValue),
            KmsKeyId=If(use_efs_kms_key, Select(str(3), Ref(efs_options)), NoValue),
            Condition=create_efs,
        )
    )

    t.add_resource(
        MountTarget(
            "MasterSubnetEFSMT",
            FileSystemId=If(create_efs, Ref(fs), Select(str(1), Ref(efs_options))),
            SecurityGroups=[Ref(compute_security_group)],
            SubnetId=Ref(head_node_subnet_id),
            Condition=create_head_node_mt,
        )
    )

    t.add_resource(
        MountTarget(
            "ComputeSubnetEFSMT",
            FileSystemId=If(create_efs, Ref(fs), Select(str(1), Ref(efs_options))),
            SecurityGroups=[Ref(compute_security_group)],
            SubnetId=Ref(compute_subnet_id),
            Condition=create_compute_mt,
        )
    )

    t.add_output(
        Output(
            "FileSystemId",
            Description="ID of the FileSystem",
            Value=If(create_efs, Ref(fs), Select("1", Ref(efs_options))),
        )
    )

    # Specify output file path
    json_file_path = args.target_path
    output_file = open(json_file_path, "w")
    output_file.write(t.to_json())
    output_file.close()
def main(args):
    t = Template()

    # [0 shared_dir, 1 efs_fs_id, 2 performance_mode, 3 efs_kms_key_id,
    # 4 provisioned_throughput, 5 encrypted, 6 throughput_mode, 7 exists_valid_mt]
    efs_options = t.add_parameter(
        Parameter(
            "EFSOptions",
            Type="CommaDelimitedList",
            Description="Comma separated list of efs related options, "
            "8 parameters in total",
        ))
    compute_security_group = t.add_parameter(
        Parameter("ComputeSecurityGroup",
                  Type="String",
                  Description="SecurityGroup for Mount Target"))
    subnet_id = t.add_parameter(
        Parameter("SubnetId",
                  Type="String",
                  Description="SubnetId for Mount Target"))
    create_efs = t.add_condition(
        "CreateEFS",
        And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")),
            Equals(Select(str(1), Ref(efs_options)), "NONE")),
    )
    create_mt = t.add_condition(
        "CreateMT",
        And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")),
            Equals(Select(str(7), Ref(efs_options)), "NONE")),
    )
    use_performance_mode = t.add_condition(
        "UsePerformanceMode",
        Not(Equals(Select(str(2), Ref(efs_options)), "NONE")))
    use_efs_encryption = t.add_condition(
        "UseEFSEncryption", Equals(Select(str(5), Ref(efs_options)), "true"))
    use_efs_kms_key = t.add_condition(
        "UseEFSKMSKey",
        And(Condition(use_efs_encryption),
            Not(Equals(Select(str(3), Ref(efs_options)), "NONE"))))
    use_throughput_mode = t.add_condition(
        "UseThroughputMode",
        Not(Equals(Select(str(6), Ref(efs_options)), "NONE")))
    use_provisioned = t.add_condition(
        "UseProvisioned",
        Equals(Select(str(6), Ref(efs_options)), "provisioned"))
    use_provisioned_throughput = t.add_condition(
        "UseProvisionedThroughput",
        And(Condition(use_provisioned),
            Not(Equals(Select(str(4), Ref(efs_options)), "NONE"))),
    )

    fs = t.add_resource(
        FileSystem(
            "EFSFS",
            PerformanceMode=If(use_performance_mode,
                               Select(str(2), Ref(efs_options)), NoValue),
            ProvisionedThroughputInMibps=If(use_provisioned_throughput,
                                            Select(str(4), Ref(efs_options)),
                                            NoValue),
            ThroughputMode=If(use_throughput_mode,
                              Select(str(6), Ref(efs_options)), NoValue),
            Encrypted=If(use_efs_encryption, Select(str(5), Ref(efs_options)),
                         NoValue),
            KmsKeyId=If(use_efs_kms_key, Select(str(3), Ref(efs_options)),
                        NoValue),
            Condition=create_efs,
        ))

    mt = t.add_resource(
        MountTarget(
            "EFSMT",
            FileSystemId=If(create_efs, Ref(fs),
                            Select(str(1), Ref(efs_options))),
            SecurityGroups=[Ref(compute_security_group)],
            SubnetId=Ref(subnet_id),
            Condition=create_mt,
        ))

    t.add_output(
        Output(
            "FileSystemId",
            Description="ID of the FileSystem",
            Value=If(create_efs, Ref(fs), Select("1", Ref(efs_options))),
        ))

    # Specify output file path
    json_file_path = args.target_path
    output_file = open(json_file_path, "w")
    output_file.write(t.to_json())
    output_file.close()
Beispiel #7
0
    def test_efs(self):
        test_stack_name = 'TestNFSUsingEFS'
        init_cf_env(test_stack_name)
        ###
        t = Template()
        fs = t.add_resource(FileSystem("MyFileSystem"))

        client_sg = ts_add_security_group(t, name="ClientSecurityGroup")
        efs_client_sg = t.add_resource(
            SecurityGroup(
                "EFSClientSecurityGroup",  # This security group is just used to mark traffic to Mount Target
                GroupDescription="EFS Mount target client",
                VpcId=get_default_vpc()))
        mt_security_group = t.add_resource(
            SecurityGroup(
                "MountTargetSecurityGroup",
                GroupDescription='EFS Mount target',
                VpcId=get_default_vpc(),
                SecurityGroupIngress=[
                    SecurityGroupRule(
                        IpProtocol='tcp',
                        SourceSecurityGroupId=Ref(
                            efs_client_sg
                        ),  # Only allow traffic from the EFS client security group.
                        FromPort=2049,
                        ToPort=2049),
                ],
            ))

        mount_target_a = t.add_resource(
            MountTarget("MountTargetA",
                        FileSystemId=Ref(fs),
                        SecurityGroups=[Ref(mt_security_group)],
                        SubnetId=get_first_subnet()))
        mount_target_b = t.add_resource(
            MountTarget("MountTargetB",
                        FileSystemId=Ref(fs),
                        SecurityGroups=[Ref(mt_security_group)],
                        SubnetId=get_subnet(index=1)))

        instance_a = ts_add_instance_with_public_ip(
            t, [Ref(client_sg), Ref(efs_client_sg)],
            name='InstanceA',
            subnet_id=get_first_subnet())
        instance_a.DependsOn = "MountTargetA"
        instance_b = ts_add_instance_with_public_ip(
            t, [Ref(client_sg), Ref(efs_client_sg)],
            name='InstanceB',
            subnet_id=get_subnet(index=1))
        instance_b.DependsOn = "MountTargetB"

        t.add_output([
            Output(
                "PublicIPA",
                Value=GetAtt(instance_a, "PublicIp"),
            ),
            Output(
                "PublicIPB",
                Value=GetAtt(instance_b, "PublicIp"),
            ),
            Output("Region", Value=Ref("AWS::Region")),
            Output("FileSystemId", Value=Ref(fs)),
        ])

        dump_template(t, True)
        create_stack(test_stack_name, t)
        outputs = get_stack_outputs(test_stack_name)
        public_ip_a = get_output_value(outputs, 'PublicIPA')
        public_ip_b = get_output_value(outputs, 'PublicIPB')
        region = get_output_value(outputs, 'Region')
        fs_id = get_output_value(outputs, 'FileSystemId')

        nfs_url = f'{fs_id}.efs.{region}.amazonaws.com'  # use mount target IP if across VPC or on-premise
        share_path = '/var/share'
        run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo mkdir {share_path}'
            )
        run(f"""ssh {SSH_OPTIONS} ec2-user@{public_ip_a} 'echo "{nfs_url}:/ {share_path} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0" | sudo tee -a /etc/fstab'"""
            )
        run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo mount -a')
        run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo touch {share_path}/hello'
            )

        run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo mkdir {share_path}'
            )
        run(f"""ssh {SSH_OPTIONS} ec2-user@{public_ip_b} 'echo "{nfs_url}:/ {share_path} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0" | sudo tee -a /etc/fstab'"""
            )
        run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo mount -a')
        stdout = run(
            f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo ls {share_path}')
        self.assertIn('hello', stdout)