def update_from_vpc(self, vpc_stack, settings=None):
"""
Override for EFS to update settings from VPC Stack
:param ecs_composex.vpc.vpc_stack.XStack vpc_stack:
:param ecs_composex.common.settings.ComposeXSettings settings:
:return:
"""
subnets_params = self.subnets_param
if self.subnets_override:
for subnet_az in vpc_stack.vpc_resource.azs:
if subnet_az.title == self.subnets_override:
subnets_params = subnet_az
break
else:
raise KeyError(
f"{self.module.res_key}.{self.name} - "
f"Override subnet name {self.subnets_override} is not defined in x-vpc",
list(vpc_stack.vpc_resource.azs.keys()),
)
for count, az in enumerate(vpc_stack.vpc_resource.azs[subnets_params]):
self.stack.stack_template.add_resource(
MountTarget(
f"{self.logical_name}MountPoint{az.title().strip().split('-')[-1]}",
FileSystemId=Ref(self.cfn_resource),
SecurityGroups=[GetAtt(self.db_sg, "GroupId")],
SubnetId=Select(count, Ref(STORAGE_SUBNETS)),
))
def _write_file_into_efs(region, vpc_stack, efs_stack, request, key_name,
cfn_stacks_factory):
"""Write file stack contains a mount target and a instance to write a empty file with random name into the efs."""
write_file_template = Template()
write_file_template.set_version("2010-09-09")
write_file_template.set_description(
"Stack to write a file to the existing EFS")
default_security_group_id = get_default_vpc_security_group(
vpc_stack.cfn_outputs["VpcId"], region)
write_file_template.add_resource(
MountTarget(
"MountTargetResource",
FileSystemId=efs_stack.cfn_resources["FileSystemResource"],
SubnetId=vpc_stack.cfn_outputs["PublicSubnetId"],
SecurityGroups=[default_security_group_id],
))
random_file_name = random_alphanumeric()
user_data = (
"""
#cloud-config
package_update: true
package_upgrade: true
runcmd:
- yum install -y nfs-utils
- file_system_id_1=""" +
efs_stack.cfn_resources["FileSystemResource"] + """
- efs_mount_point_1=/mnt/efs/fs1
- mkdir -p "${!efs_mount_point_1}"
- mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport,_netdev """
+
""""${!file_system_id_1}.efs.${AWS::Region}.${AWS::URLSuffix}:/" "${!efs_mount_point_1}"
- touch ${!efs_mount_point_1}/""" + random_file_name + """
- umount ${!efs_mount_point_1}
- opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource InstanceToWriteEFS --region ${AWS::Region}
""")
write_file_template.add_resource(
Instance(
"InstanceToWriteEFS",
CreationPolicy={"ResourceSignal": {
"Timeout": "PT10M"
}},
ImageId=retrieve_latest_ami(region, "alinux2"),
InstanceType="c5.xlarge",
SubnetId=vpc_stack.cfn_outputs["PublicSubnetId"],
UserData=Base64(Sub(user_data)),
KeyName=key_name,
DependsOn=["MountTargetResource"],
))
write_file_stack = CfnStack(
name=generate_stack_name("integ-tests-efs-write-file",
request.config.getoption("stackname_suffix")),
region=region,
template=write_file_template.to_json(),
)
cfn_stacks_factory.create_stack(write_file_stack)
cfn_stacks_factory.delete_stack(write_file_stack.name, region)
return random_file_name
SecurityGroupIngress=[efs_security_group_rule],
VpcId=Ref(vpcid_param),
GroupDescription="Allow NFS over TCP")
template.add_resource(efs_security_group)
# Create FileSystem. This is the actual filesystem, which has one or more
# mount targets. Give it some tags so we can identify it later.
tags = Tags(Name='MyEFSFileSystem')
efs_file_system = FileSystem("MyEFSFileSystem", FileSystemTags=tags)
template.add_resource(efs_file_system)
# create MountTarget. You really want a mount target in each subnet where
# it's required, but for the purpose of this example we'll
# put it in just one.
efs_mount_target = MountTarget("MyEFSMountTarget",
FileSystemId=Ref(efs_file_system),
SecurityGroups=[Ref(efs_security_group)],
SubnetId=Ref(subnetid_param))
template.add_resource(efs_mount_target)
# Create the policy that allows the instance to describe file systems and tags,
# so it can lookup the file system using AWS tags. An alternative would be to
# pass in the FileSystem name as UserData.
efs_host_role = Role(
"EFSHostRole",
AssumeRolePolicyDocument=PolicyDocument(Statement=[
Statement(Effect=Allow,
Action=[
Action('elasticfilesystem', 'DescribeFileSystems'),
Action('elasticfilesystem', 'DescribeTags')
],
Resource=["*"])
def main(args):
t = Template()
# [0 shared_dir, 1 efs_fs_id, 2 performance_mode, 3 efs_kms_key_id,
# 4 provisioned_throughput, 5 encrypted, 6 throughput_mode, 7 exists_valid_head_node_mt, 8 exists_valid_compute_mt]
efs_options = t.add_parameter(
Parameter(
"EFSOptions",
Type="CommaDelimitedList",
Description="Comma separated list of efs related options, 9 parameters in total",
)
)
compute_security_group = t.add_parameter(
Parameter("ComputeSecurityGroup", Type="String", Description="Security Group for Mount Target")
)
head_node_subnet_id = t.add_parameter(
Parameter("MasterSubnetId", Type="String", Description="Head node subnet id for head node mount target")
)
compute_subnet_id = t.add_parameter(
Parameter(
"ComputeSubnetId",
Type="String",
Description="User provided compute subnet id. Will be use to create compute mount target if needed.",
)
)
create_efs = t.add_condition(
"CreateEFS",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")), Equals(Select(str(1), Ref(efs_options)), "NONE")),
)
create_head_node_mt = t.add_condition(
"CreateMasterMT",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")), Equals(Select(str(7), Ref(efs_options)), "NONE")),
)
no_mt_in_compute_az = t.add_condition("NoMTInComputeAZ", Equals(Select(str(8), Ref(efs_options)), "NONE"))
use_user_provided_compute_subnet = t.add_condition(
"UseUserProvidedComputeSubnet", Not(Equals(Ref(compute_subnet_id), "NONE"))
)
# Need to create compute mount target if:
# user is providing a compute subnet and
# there is no existing MT in compute subnet's AZ(includes case where head node AZ == compute AZ).
#
# If user is not providing a compute subnet, either we are using the head node subnet as compute subnet,
# or we will be creating a compute subnet that is in the same AZ as head node subnet,
# see ComputeSubnet resource in the main stack.
# In both cases no compute MT is needed.
create_compute_mt = t.add_condition(
"CreateComputeMT", And(Condition(use_user_provided_compute_subnet), Condition(no_mt_in_compute_az))
)
use_performance_mode = t.add_condition("UsePerformanceMode", Not(Equals(Select(str(2), Ref(efs_options)), "NONE")))
use_efs_encryption = t.add_condition("UseEFSEncryption", Equals(Select(str(5), Ref(efs_options)), "true"))
use_efs_kms_key = t.add_condition(
"UseEFSKMSKey", And(Condition(use_efs_encryption), Not(Equals(Select(str(3), Ref(efs_options)), "NONE")))
)
use_throughput_mode = t.add_condition("UseThroughputMode", Not(Equals(Select(str(6), Ref(efs_options)), "NONE")))
use_provisioned = t.add_condition("UseProvisioned", Equals(Select(str(6), Ref(efs_options)), "provisioned"))
use_provisioned_throughput = t.add_condition(
"UseProvisionedThroughput",
And(Condition(use_provisioned), Not(Equals(Select(str(4), Ref(efs_options)), "NONE"))),
)
fs = t.add_resource(
FileSystem(
"EFSFS",
PerformanceMode=If(use_performance_mode, Select(str(2), Ref(efs_options)), NoValue),
ProvisionedThroughputInMibps=If(use_provisioned_throughput, Select(str(4), Ref(efs_options)), NoValue),
ThroughputMode=If(use_throughput_mode, Select(str(6), Ref(efs_options)), NoValue),
Encrypted=If(use_efs_encryption, Select(str(5), Ref(efs_options)), NoValue),
KmsKeyId=If(use_efs_kms_key, Select(str(3), Ref(efs_options)), NoValue),
Condition=create_efs,
)
)
t.add_resource(
MountTarget(
"MasterSubnetEFSMT",
FileSystemId=If(create_efs, Ref(fs), Select(str(1), Ref(efs_options))),
SecurityGroups=[Ref(compute_security_group)],
SubnetId=Ref(head_node_subnet_id),
Condition=create_head_node_mt,
)
)
t.add_resource(
MountTarget(
"ComputeSubnetEFSMT",
FileSystemId=If(create_efs, Ref(fs), Select(str(1), Ref(efs_options))),
SecurityGroups=[Ref(compute_security_group)],
SubnetId=Ref(compute_subnet_id),
Condition=create_compute_mt,
)
)
t.add_output(
Output(
"FileSystemId",
Description="ID of the FileSystem",
Value=If(create_efs, Ref(fs), Select("1", Ref(efs_options))),
)
)
# Specify output file path
json_file_path = args.target_path
output_file = open(json_file_path, "w")
output_file.write(t.to_json())
output_file.close()
def main(args):
t = Template()
# [0 shared_dir, 1 efs_fs_id, 2 performance_mode, 3 efs_kms_key_id,
# 4 provisioned_throughput, 5 encrypted, 6 throughput_mode, 7 exists_valid_mt]
efs_options = t.add_parameter(
Parameter(
"EFSOptions",
Type="CommaDelimitedList",
Description="Comma separated list of efs related options, "
"8 parameters in total",
))
compute_security_group = t.add_parameter(
Parameter("ComputeSecurityGroup",
Type="String",
Description="SecurityGroup for Mount Target"))
subnet_id = t.add_parameter(
Parameter("SubnetId",
Type="String",
Description="SubnetId for Mount Target"))
create_efs = t.add_condition(
"CreateEFS",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")),
Equals(Select(str(1), Ref(efs_options)), "NONE")),
)
create_mt = t.add_condition(
"CreateMT",
And(Not(Equals(Select(str(0), Ref(efs_options)), "NONE")),
Equals(Select(str(7), Ref(efs_options)), "NONE")),
)
use_performance_mode = t.add_condition(
"UsePerformanceMode",
Not(Equals(Select(str(2), Ref(efs_options)), "NONE")))
use_efs_encryption = t.add_condition(
"UseEFSEncryption", Equals(Select(str(5), Ref(efs_options)), "true"))
use_efs_kms_key = t.add_condition(
"UseEFSKMSKey",
And(Condition(use_efs_encryption),
Not(Equals(Select(str(3), Ref(efs_options)), "NONE"))))
use_throughput_mode = t.add_condition(
"UseThroughputMode",
Not(Equals(Select(str(6), Ref(efs_options)), "NONE")))
use_provisioned = t.add_condition(
"UseProvisioned",
Equals(Select(str(6), Ref(efs_options)), "provisioned"))
use_provisioned_throughput = t.add_condition(
"UseProvisionedThroughput",
And(Condition(use_provisioned),
Not(Equals(Select(str(4), Ref(efs_options)), "NONE"))),
)
fs = t.add_resource(
FileSystem(
"EFSFS",
PerformanceMode=If(use_performance_mode,
Select(str(2), Ref(efs_options)), NoValue),
ProvisionedThroughputInMibps=If(use_provisioned_throughput,
Select(str(4), Ref(efs_options)),
NoValue),
ThroughputMode=If(use_throughput_mode,
Select(str(6), Ref(efs_options)), NoValue),
Encrypted=If(use_efs_encryption, Select(str(5), Ref(efs_options)),
NoValue),
KmsKeyId=If(use_efs_kms_key, Select(str(3), Ref(efs_options)),
NoValue),
Condition=create_efs,
))
mt = t.add_resource(
MountTarget(
"EFSMT",
FileSystemId=If(create_efs, Ref(fs),
Select(str(1), Ref(efs_options))),
SecurityGroups=[Ref(compute_security_group)],
SubnetId=Ref(subnet_id),
Condition=create_mt,
))
t.add_output(
Output(
"FileSystemId",
Description="ID of the FileSystem",
Value=If(create_efs, Ref(fs), Select("1", Ref(efs_options))),
))
# Specify output file path
json_file_path = args.target_path
output_file = open(json_file_path, "w")
output_file.write(t.to_json())
output_file.close()
def test_efs(self):
test_stack_name = 'TestNFSUsingEFS'
init_cf_env(test_stack_name)
###
t = Template()
fs = t.add_resource(FileSystem("MyFileSystem"))
client_sg = ts_add_security_group(t, name="ClientSecurityGroup")
efs_client_sg = t.add_resource(
SecurityGroup(
"EFSClientSecurityGroup", # This security group is just used to mark traffic to Mount Target
GroupDescription="EFS Mount target client",
VpcId=get_default_vpc()))
mt_security_group = t.add_resource(
SecurityGroup(
"MountTargetSecurityGroup",
GroupDescription='EFS Mount target',
VpcId=get_default_vpc(),
SecurityGroupIngress=[
SecurityGroupRule(
IpProtocol='tcp',
SourceSecurityGroupId=Ref(
efs_client_sg
), # Only allow traffic from the EFS client security group.
FromPort=2049,
ToPort=2049),
],
))
mount_target_a = t.add_resource(
MountTarget("MountTargetA",
FileSystemId=Ref(fs),
SecurityGroups=[Ref(mt_security_group)],
SubnetId=get_first_subnet()))
mount_target_b = t.add_resource(
MountTarget("MountTargetB",
FileSystemId=Ref(fs),
SecurityGroups=[Ref(mt_security_group)],
SubnetId=get_subnet(index=1)))
instance_a = ts_add_instance_with_public_ip(
t, [Ref(client_sg), Ref(efs_client_sg)],
name='InstanceA',
subnet_id=get_first_subnet())
instance_a.DependsOn = "MountTargetA"
instance_b = ts_add_instance_with_public_ip(
t, [Ref(client_sg), Ref(efs_client_sg)],
name='InstanceB',
subnet_id=get_subnet(index=1))
instance_b.DependsOn = "MountTargetB"
t.add_output([
Output(
"PublicIPA",
Value=GetAtt(instance_a, "PublicIp"),
),
Output(
"PublicIPB",
Value=GetAtt(instance_b, "PublicIp"),
),
Output("Region", Value=Ref("AWS::Region")),
Output("FileSystemId", Value=Ref(fs)),
])
dump_template(t, True)
create_stack(test_stack_name, t)
outputs = get_stack_outputs(test_stack_name)
public_ip_a = get_output_value(outputs, 'PublicIPA')
public_ip_b = get_output_value(outputs, 'PublicIPB')
region = get_output_value(outputs, 'Region')
fs_id = get_output_value(outputs, 'FileSystemId')
nfs_url = f'{fs_id}.efs.{region}.amazonaws.com' # use mount target IP if across VPC or on-premise
share_path = '/var/share'
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo mkdir {share_path}'
)
run(f"""ssh {SSH_OPTIONS} ec2-user@{public_ip_a} 'echo "{nfs_url}:/ {share_path} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0" | sudo tee -a /etc/fstab'"""
)
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo mount -a')
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_a} sudo touch {share_path}/hello'
)
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo mkdir {share_path}'
)
run(f"""ssh {SSH_OPTIONS} ec2-user@{public_ip_b} 'echo "{nfs_url}:/ {share_path} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,_netdev 0 0" | sudo tee -a /etc/fstab'"""
)
run(f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo mount -a')
stdout = run(
f'ssh {SSH_OPTIONS} ec2-user@{public_ip_b} sudo ls {share_path}')
self.assertIn('hello', stdout)