def create_efs_stack(settings, new_resources):
"""
Function to create the root stack and add EFS FS.
:param ecs_composex.common.settings.ComposeXSettings settings:
:param list new_resources:
:return: Root template for EFS
:rtype: troposphere.Template
"""
template = build_template("Root for EFS built by ECS Compose-X", [FS_PORT])
for res in new_resources:
res_cfn_props = import_record_properties(res.properties, FileSystem)
res.cfn_resource = FileSystem(res.logical_name, **res_cfn_props)
res.db_sg = SecurityGroup(
f"{res.logical_name}SecurityGroup",
GroupName=Sub(f"{res.logical_name}EfsSg"),
GroupDescription=Sub(f"SG for EFS {res.cfn_resource.title}"),
VpcId=Ref(VPC_ID),
)
template.add_resource(res.cfn_resource)
template.add_resource(res.db_sg)
res.init_outputs()
res.generate_outputs()
template.add_output(res.outputs)
return template
def efs_stack(cfn_stacks_factory, request, region):
"""EFS stack contains a single efs resource."""
efs_template = Template()
efs_template.set_version("2010-09-09")
efs_template.set_description("EFS stack created for testing existing EFS")
efs_template.add_resource(FileSystem("FileSystemResource"))
stack = CfnStack(
name=generate_stack_name("integ-tests-efs",
request.config.getoption("stackname_suffix")),
region=region,
template=efs_template.to_json(),
)
cfn_stacks_factory.create_stack(stack)
yield stack
if not request.config.getoption("no_delete"):
cfn_stacks_factory.delete_stack(stack.name, region)
FromPort='2049',
ToPort='2049',
SourceSecurityGroupId=Ref(efs_host_security_group))
# Security group that's applied to the Mount Targets.
efs_security_group = SecurityGroup(
"SecurityGroup",
SecurityGroupIngress=[efs_security_group_rule],
VpcId=Ref(vpcid_param),
GroupDescription="Allow NFS over TCP")
template.add_resource(efs_security_group)
# Create FileSystem. This is the actual filesystem, which has one or more
# mount targets. Give it some tags so we can identify it later.
tags = Tags(Name='MyEFSFileSystem')
efs_file_system = FileSystem("MyEFSFileSystem", FileSystemTags=tags)
template.add_resource(efs_file_system)
# create MountTarget. You really want a mount target in each subnet where
# it's required, but for the purpose of this example we'll
# put it in just one.
efs_mount_target = MountTarget("MyEFSMountTarget",
FileSystemId=Ref(efs_file_system),
SecurityGroups=[Ref(efs_security_group)],
SubnetId=Ref(subnetid_param))
template.add_resource(efs_mount_target)
# Create the policy that allows the instance to describe file systems and tags,
# so it can lookup the file system using AWS tags. An alternative would be to
# pass in the FileSystem name as UserData.
efs_host_role = Role(
)
# Security group that's applied to the Mount Targets.
efs_security_group = SecurityGroup(
"SecurityGroup",
SecurityGroupIngress=[Ref(efs_security_group_rule)],
VpcId=Ref(vpcid_param),
GroupDescription="Allow NFS over TCP"
)
template.add_resource(efs_security_group)
# Create FileSystem. This is the actual filesystem, which has one or more
# mount targets. Give it some tags so we can identify it later.
tags = Tags(Name='MyEFSFileSystem')
efs_file_system = FileSystem(
title='MyEFSFileSystem',
FileSystemTags=tags
)
template.add_resource(efs_file_system)
# create MountTarget. You really want a mount target in each subnet where
# it's required, but for the purpose of this example we'll
# put it in just one.
efs_mount_target = MountTarget(
title='MyEFSMountTarget',
FileSystemId=Ref(efs_file_system),
SecurityGroups=[Ref(efs_security_group)],
SubnetId=Ref(subnetid_param)
)
template.add_resource(efs_mount_target)
# Create the policy that allows the instance to describe file systems and tags,
def main(args):
t = Template()
# [0 shared_dir, 1 efs_fs_id, 2 performance_mode, 3 efs_kms_key_id,
# 4 provisioned_throughput, 5 encrypted, 6 throughput_mode, 7 exists_valid_head_node_mt, 8 exists_valid_compute_mt]
efs_options = t.add_parameter(
Parameter(
"EFSOptions",
Type="CommaDelimitedList",
Description="Comma separated list of efs related options, 9 parameters in total",
)
)
compute_security_group = t.add_parameter(
Parameter("ComputeSecurityGroup", Type="String", Description="Security Group for Mount Target")
)
head_node_subnet_id = t.add_parameter(
Parameter("MasterSubnetId", Type="String", Description="Head node subnet id for head node mount target")
)
compute_subnet_id = t.add_parameter(
Parameter(
"ComputeSubnetId",
Type="String",
Description="User provided compute subnet id. Will be use to create compute mount target if needed.",
)
)
create_efs = t.add_condition(
"CreateEFS",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")), Equals(Select(str(1), Ref(efs_options)), "NONE")),
)
create_head_node_mt = t.add_condition(
"CreateMasterMT",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")), Equals(Select(str(7), Ref(efs_options)), "NONE")),
)
no_mt_in_compute_az = t.add_condition("NoMTInComputeAZ", Equals(Select(str(8), Ref(efs_options)), "NONE"))
use_user_provided_compute_subnet = t.add_condition(
"UseUserProvidedComputeSubnet", Not(Equals(Ref(compute_subnet_id), "NONE"))
)
# Need to create compute mount target if:
# user is providing a compute subnet and
# there is no existing MT in compute subnet's AZ(includes case where head node AZ == compute AZ).
#
# If user is not providing a compute subnet, either we are using the head node subnet as compute subnet,
# or we will be creating a compute subnet that is in the same AZ as head node subnet,
# see ComputeSubnet resource in the main stack.
# In both cases no compute MT is needed.
create_compute_mt = t.add_condition(
"CreateComputeMT", And(Condition(use_user_provided_compute_subnet), Condition(no_mt_in_compute_az))
)
use_performance_mode = t.add_condition("UsePerformanceMode", Not(Equals(Select(str(2), Ref(efs_options)), "NONE")))
use_efs_encryption = t.add_condition("UseEFSEncryption", Equals(Select(str(5), Ref(efs_options)), "true"))
use_efs_kms_key = t.add_condition(
"UseEFSKMSKey", And(Condition(use_efs_encryption), Not(Equals(Select(str(3), Ref(efs_options)), "NONE")))
)
use_throughput_mode = t.add_condition("UseThroughputMode", Not(Equals(Select(str(6), Ref(efs_options)), "NONE")))
use_provisioned = t.add_condition("UseProvisioned", Equals(Select(str(6), Ref(efs_options)), "provisioned"))
use_provisioned_throughput = t.add_condition(
"UseProvisionedThroughput",
And(Condition(use_provisioned), Not(Equals(Select(str(4), Ref(efs_options)), "NONE"))),
)
fs = t.add_resource(
FileSystem(
"EFSFS",
PerformanceMode=If(use_performance_mode, Select(str(2), Ref(efs_options)), NoValue),
ProvisionedThroughputInMibps=If(use_provisioned_throughput, Select(str(4), Ref(efs_options)), NoValue),
ThroughputMode=If(use_throughput_mode, Select(str(6), Ref(efs_options)), NoValue),
Encrypted=If(use_efs_encryption, Select(str(5), Ref(efs_options)), NoValue),
KmsKeyId=If(use_efs_kms_key, Select(str(3), Ref(efs_options)), NoValue),
Condition=create_efs,
)
)
t.add_resource(
MountTarget(
"MasterSubnetEFSMT",
FileSystemId=If(create_efs, Ref(fs), Select(str(1), Ref(efs_options))),
SecurityGroups=[Ref(compute_security_group)],
SubnetId=Ref(head_node_subnet_id),
Condition=create_head_node_mt,
)
)
t.add_resource(
MountTarget(
"ComputeSubnetEFSMT",
FileSystemId=If(create_efs, Ref(fs), Select(str(1), Ref(efs_options))),
SecurityGroups=[Ref(compute_security_group)],
SubnetId=Ref(compute_subnet_id),
Condition=create_compute_mt,
)
)
t.add_output(
Output(
"FileSystemId",
Description="ID of the FileSystem",
Value=If(create_efs, Ref(fs), Select("1", Ref(efs_options))),
)
)
# Specify output file path
json_file_path = args.target_path
output_file = open(json_file_path, "w")
output_file.write(t.to_json())
output_file.close()
def main(args):
t = Template()
# [0 shared_dir, 1 efs_fs_id, 2 performance_mode, 3 efs_kms_key_id,
# 4 provisioned_throughput, 5 encrypted, 6 throughput_mode, 7 exists_valid_mt]
efs_options = t.add_parameter(
Parameter(
"EFSOptions",
Type="CommaDelimitedList",
Description="Comma separated list of efs related options, "
"8 parameters in total",
))
compute_security_group = t.add_parameter(
Parameter("ComputeSecurityGroup",
Type="String",
Description="SecurityGroup for Mount Target"))
subnet_id = t.add_parameter(
Parameter("SubnetId",
Type="String",
Description="SubnetId for Mount Target"))
create_efs = t.add_condition(
"CreateEFS",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")),
Equals(Select(str(1), Ref(efs_options)), "NONE")),
)
create_mt = t.add_condition(
"CreateMT",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")),
Equals(Select(str(7), Ref(efs_options)), "NONE")),
)
use_performance_mode = t.add_condition(
"UsePerformanceMode",
Not(Equals(Select(str(2), Ref(efs_options)), "NONE")))
use_efs_encryption = t.add_condition(
"UseEFSEncryption", Equals(Select(str(5), Ref(efs_options)), "true"))
use_efs_kms_key = t.add_condition(
"UseEFSKMSKey",
And(Condition(use_efs_encryption),
Not(Equals(Select(str(3), Ref(efs_options)), "NONE"))))
use_throughput_mode = t.add_condition(
"UseThroughputMode",
Not(Equals(Select(str(6), Ref(efs_options)), "NONE")))
use_provisioned = t.add_condition(
"UseProvisioned",
Equals(Select(str(6), Ref(efs_options)), "provisioned"))
use_provisioned_throughput = t.add_condition(
"UseProvisionedThroughput",
And(Condition(use_provisioned),
Not(Equals(Select(str(4), Ref(efs_options)), "NONE"))),
)
fs = t.add_resource(
FileSystem(
"EFSFS",
PerformanceMode=If(use_performance_mode,
Select(str(2), Ref(efs_options)), NoValue),
ProvisionedThroughputInMibps=If(use_provisioned_throughput,
Select(str(4), Ref(efs_options)),
NoValue),
ThroughputMode=If(use_throughput_mode,
Select(str(6), Ref(efs_options)), NoValue),
Encrypted=If(use_efs_encryption, Select(str(5), Ref(efs_options)),
NoValue),
KmsKeyId=If(use_efs_kms_key, Select(str(3), Ref(efs_options)),
NoValue),
Condition=create_efs,
))
mt = t.add_resource(
MountTarget(
"EFSMT",
FileSystemId=If(create_efs, Ref(fs),
Select(str(1), Ref(efs_options))),
SecurityGroups=[Ref(compute_security_group)],
SubnetId=Ref(subnet_id),
Condition=create_mt,
))
t.add_output(
Output(
"FileSystemId",
Description="ID of the FileSystem",
Value=If(create_efs, Ref(fs), Select("1", Ref(efs_options))),
))
# Specify output file path
json_file_path = args.target_path
output_file = open(json_file_path, "w")
output_file.write(t.to_json())
output_file.close()
def test_efs(self):
test_stack_name = 'TestNFSUsingEFS'
init_cf_env(test_stack_name)
###
t = Template()
fs = t.add_resource(FileSystem("MyFileSystem"))
client_sg = ts_add_security_group(t, name="ClientSecurityGroup")
efs_client_sg = t.add_resource(
SecurityGroup(
"EFSClientSecurityGroup", # This security group is just used to mark traffic to Mount Target
GroupDescription="EFS Mount target client",
VpcId=get_default_vpc()))
mt_security_group = t.add_resource(
SecurityGroup(
"MountTargetSecurityGroup",
GroupDescription='EFS Mount target',
VpcId=get_default_vpc(),
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
SourceSecurityGroupId=Ref(
efs_client_sg
), # Only allow traffic from the EFS client security group.
FromPort=2049,
ToPort=2049),
],
))
mount_target_a = t.add_resource(
MountTarget("MountTargetA",
FileSystemId=Ref(fs),
SecurityGroups=[Ref(mt_security_group)],
SubnetId=get_first_subnet()))
mount_target_b = t.add_resource(
MountTarget("MountTargetB",
FileSystemId=Ref(fs),
SecurityGroups=[Ref(mt_security_group)],
SubnetId=get_subnet(index=1)))
instance_a = ts_add_instance_with_public_ip(
t, [Ref(client_sg), Ref(efs_client_sg)],
name='InstanceA',
subnet_id=get_first_subnet())
instance_a.DependsOn = "MountTargetA"
instance_b = ts_add_instance_with_public_ip(
t, [Ref(client_sg), Ref(efs_client_sg)],
name='InstanceB',
subnet_id=get_subnet(index=1))
instance_b.DependsOn = "MountTargetB"
t.add_output([
Output(
"PublicIPA",
Value=GetAtt(instance_a, "PublicIp"),
),
Output(
"PublicIPB",
Value=GetAtt(instance_b, "PublicIp"),
),
Output("Region", Value=Ref("AWS::Region")),
Output("FileSystemId", Value=Ref(fs)),
])
dump_template(t, True)
create_stack(test_stack_name, t)
outputs = get_stack_outputs(test_stack_name)
public_ip_a = get_output_value(outputs, 'PublicIPA')
public_ip_b = get_output_value(outputs, 'PublicIPB')
region = get_output_value(outputs, 'Region')
fs_id = get_output_value(outputs, 'FileSystemId')
nfs_url = f'{fs_id}.efs.{region}.amazonaws.com' # use mount target IP if across VPC or on-premise
share_path = '/var/share'
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo mkdir {share_path}'
)
run(f"""ssh {SSH_OPTIONS} ec2-user@{public_ip_a} 'echo "{nfs_url}:/ {share_path} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0" | sudo tee -a /etc/fstab'"""
)
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo mount -a')
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo touch {share_path}/hello'
)
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo mkdir {share_path}'
)
run(f"""ssh {SSH_OPTIONS} ec2-user@{public_ip_b} 'echo "{nfs_url}:/ {share_path} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0" | sudo tee -a /etc/fstab'"""
)
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo mount -a')
stdout = run(
f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo ls {share_path}')
self.assertIn('hello', stdout)