def default_ldap_user_password():
# Manage primary LDAP user/password
restricted_chars = ["'", "\"", "=", ";", ",", "$", "`", "~", "%", " "]
password_regex = "^(?:(?=.*[a-z])(?:(?=.*[A-Z])(?=.*[\d\W])|(?=.*\W)(?=.*\d))|(?=.*\W)(?=.*[A-Z])(?=.*\d)).{8,}$"
if args.ldap_password_file is None:
install_parameters["ldap_password"] = get_input(f"[Step 4/{total_install_phases}] {install_phases[4]}", args.ldap_password, None, str, hide=True)
while not re.match(password_regex, install_parameters["ldap_password"]) \
or install_parameters["ldap_user"].lower() in install_parameters["ldap_password"].lower() \
or [element for element in restricted_chars if (element in install_parameters["ldap_password"])]:
print(f"{fg('red')} LDAP password must have 1 uppercase, 1 lowercase, 1 digit and be 8 chars min.\n LDAP password cannot contain your username, white space, or any of the following special characters {''.join(restricted_chars)}{attr('reset')}")
install_parameters["ldap_password"] = get_input(f"[Step 4/{total_install_phases}] {install_phases[4]}", None, None, str, hide=True)
if not args.ldap_password:
# when pw is entered interactively via getpass, we ask for a verification
ldap_password_verify = get_input("[Verification] Please re-enter the password of your first LDAP account", args.ldap_password, None, str, hide=True)
if install_parameters["ldap_password"] != ldap_password_verify:
print(f"{fg('red')} You entered two different passwords. Please try again and make sure password and password (verification) are the same.{attr('reset')}")
return False
else:
return True
else:
# Retrieve password from file
print(f"{fg('yellow')}Retrieving password from {args.ldap_password_file}{attr('reset')}")
try:
with open(args.ldap_password_file) as f:
install_parameters["ldap_password"] = f.read().replace("\n", "").replace("\r", "")
if not re.match(password_regex, install_parameters["ldap_password"]):
print(f"{fg('red')} LDAP password must have 1 uppercase, 1 lowercase, 1 digit and be 8 char min{attr('reset')}")
sys.exit(1)
except FileNotFoundError:
print(f"{fg('red')}Unable to found {args.ldap_password_file}. Please specify absolute path {attr('reset')}")
sys.exit(1)
def get_subnets(self, vpc_id, environment, selected_subnets=[]):
try:
if environment == "private":
print(f"\n====== Select {fg('misty_rose_3')}3 subnets to use for your compute nodes (private subnets preferably) {attr('reset')} ======\n")
else:
print(f"\n====== Select {fg('misty_rose_3')}3 subnets to use for the main Scheduler and Load Balancer (public subnets preferably) {attr('reset')} ======\n")
subnets_by_name = {}
token = True
next_token = None
max_results = 50
while token is True:
if not next_token:
all_subnets = self.ec2.describe_subnets(MaxResults=max_results, Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}, {'Name': 'ipv6-native', 'Values': ['false']}])
else:
all_subnets = self.ec2.describe_subnets(Filters=[{'Name': 'vpc-id', 'Values': [vpc_id]}, {'Name': 'ipv6-native', 'Values': ['false']}], MaxResults=max_results, NextToken=next_token)
try:
next_token = all_subnets['Token']
except KeyError:
token = False
for subnet in all_subnets['Subnets']:
resource_name = False
if "Tags" in subnet.keys():
for tag in subnet["Tags"]:
if tag["Key"] == "Name":
resource_name = tag["Value"]
if not resource_name:
continue
subnets_by_name[resource_name] = subnet
subnets = {}
count = 1
for resource_name in sorted(subnets_by_name):
subnet = subnets_by_name[resource_name]
if f"{subnet['SubnetId']},{subnet['AvailabilityZone']}" not in selected_subnets:
subnets[count] = {"id": subnet['SubnetId'],
"availability_zone": subnet['AvailabilityZone'],
"description": f"{resource_name if resource_name is not False else ''} {subnet['CidrBlock']} in {subnet['AvailabilityZone']}"}
count += 1
[print(" {:2} > {}".format(key, value["description"])) for key, value in subnets.items()]
selected_subnets_count = get_input(f"How many of these subnets do you want to use?", None, list(range(1, count)), int)
while selected_subnets_count < 2:
print(f"{fg('red')} You must use at least 2 subnets for high availability {attr('reset')}")
selected_subnets_count = get_input(f"How many of these subnets do you want to use?", None, list(range(1, count)), int)
selected_subnets = []
while len(selected_subnets) != selected_subnets_count:
allowed_choices = list(subnets.keys())
if len(allowed_choices) == 0:
return {"success": False, "message": "Not enough subnets available"}
choice = get_input(f"Choose your subnet #{len(selected_subnets) + 1} ?", None, allowed_choices, int)
selected_subnets.append(f"{subnets[choice]['id']},{subnets[choice]['availability_zone']}")
del subnets[choice]
return {"success": True, "message": selected_subnets}
except Exception as err:
return {"success": False, "message": str(err)}
def get_iam_roles(self, environment, selected_roles=[]):
try:
print(f"\n====== Choose the {fg('misty_rose_3')}IAM role to use for the {environment.upper()}{attr('reset')} ======\n")
roles = {}
count = 1
marker = True
next_marker = None
max_items = 50
while marker is True:
if not next_marker:
all_roles = self.iam.list_roles(MaxItems=max_items)
else:
all_roles = self.iam.list_roles(MaxItems=max_items, Marker=next_marker)
try:
next_marker = all_roles['Marker']
except KeyError:
marker = False
for role in all_roles["Roles"]:
if role['RoleName'] not in selected_roles:
roles[count] = {"arn": f"{role['Arn']}", "name": role["RoleName"], "description": f"{role['RoleName']} - {role['Description'] if 'Description' in role.keys() else ''}"}
count += 1
[print(" {} > {}".format(key, value["description"])) for key, value in roles.items()]
allowed_choices = list(roles.keys())
choice = get_input(f"What IAM Role for you want to use for {environment.upper()}", None, allowed_choices, int)
return {'success': True, 'message': roles[choice]}
except Exception as err:
print(err)
return {'success': False, 'message': str(err)}
def find_directory_services(self, vpc_id):
try:
print(f"\n====== What {fg('misty_rose_3')}Directory Services (Microsoft AD){attr('reset')} do you want to use? [region: {self.region}, vpc: {vpc_id}] ======\n")
ds = {}
count = 1
token = True
next_token = None
max_results = 50
while token is True:
if not next_token:
all_ds = self.ds.describe_directories(MaxResults=max_results)
else:
all_ds = self.ds.describe_directories(MaxResults=max_results, NextToken=next_token)
try:
next_token = all_ds['Token']
except KeyError:
token = False
for directory in all_ds["DirectoryDescriptions"]:
if directory["VpcSettings"]["VpcId"] == vpc_id:
ds[count] = {"id": directory["DirectoryId"],
"name": directory["Name"],
"netbios": directory["ShortName"],
"dns": directory["DnsIpAddrs"],
"description": f"{directory['Name']} (Domain: {directory['ShortName']}, Id: {directory['DirectoryId']})"}
count += 1
[print(" {:2} > {}".format(key, value["description"])) for key, value in ds.items()]
allowed_choices = list(ds.keys())
choice = get_input(f"Choose the directory you want to use?", None, allowed_choices, int)
return {"success": True, "message": ds[choice]}
except Exception as err:
return {"success": False, "message": str(err)}
def find_vpc(self):
try:
print(f"\n====== What {fg('misty_rose_3')}VPC{attr('reset')} in {self.region} do you want to use? ======\n")
vpcs_by_name = {}
token = True
next_token = None
max_results = 50
while token is True:
if not next_token:
all_vpcs = self.ec2.describe_vpcs()
else:
all_vpcs = self.ec2.describe_vpcs(MaxResults=max_results, NextToken=next_token)
try:
next_token = all_vpcs['Token']
except KeyError:
token = False
for vpc in all_vpcs["Vpcs"]:
resource_name = False
if "Tags" in vpc.keys():
for tag in vpc["Tags"]:
if tag["Key"] == "Name":
resource_name = tag["Value"]
if not resource_name:
continue
vpcs_by_name[resource_name] = vpc
vpcs = {}
count = 1
for resource_name in sorted(vpcs_by_name):
vpc = vpcs_by_name[resource_name]
vpcs[count] = {"id": vpc["VpcId"],
"description": f"{resource_name if resource_name is not False else ''} {vpc['VpcId']} {vpc['CidrBlock']}",
"cidr": vpc['CidrBlock']}
count += 1
[print(" {:2} > {}".format(key, value["description"])) for key, value in vpcs.items()]
allowed_choices = list(vpcs.keys())
choice = get_input(f"Choose the VPC you want to use?", None, allowed_choices, int)
return {"success": True, "message": vpcs[choice]}
except Exception as err:
return {"success": False, "message": str(err)}
def get_security_groups(self, vpc_id, environment, scheduler_sg=[]):
try:
print(f"\n====== Choose the {fg('misty_rose_3')}security group to use for the {environment.upper()}{attr('reset')} [region: {self.region}, vpc: {vpc_id}] ======\n")
sgs_by_name = {}
token = True
next_token = None
max_results = 50
while token is True:
if not next_token:
all_sgs = self.ec2.describe_security_groups(MaxResults=max_results, Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
else:
all_sgs = self.ec2.describe_security_groups(MaxResults=max_results, NextToken=next_token, Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
try:
next_token = all_sgs['Token']
except KeyError:
token = False
for sg in all_sgs['SecurityGroups']:
resource_name = False
if "Tags" in sg.keys():
for tag in sg["Tags"]:
if tag["Key"] == "Name":
resource_name = tag["Value"]
if not resource_name:
continue
sgs_by_name[resource_name] = sg
sgs = {}
count = 1
for resource_name in sorted(sgs_by_name):
sg = sgs_by_name[resource_name]
if sg['GroupId'] not in scheduler_sg:
sgs[count] = {"id": f"{sg['GroupId']}",
"description": f"{resource_name if resource_name is not False else ''} {sg['GroupId']} {sg['GroupName']}"}
count += 1
[print(" {:2} > {}".format(key, sgs[key]["description"])) for key in sorted(sgs) ]
allowed_choices = list(sgs.keys())
choice = get_input(f"What security group for you want to use for {environment.upper()}", None, allowed_choices, int)
return {'success': True, 'message': sgs[choice]["id"]}
except Exception as err:
return {'success': False, 'message': str(err)}
def find_elasticsearch(self, vpc_id):
try:
print(f"\n====== What {fg('misty_rose_3')}ElasticSearch cluster{attr('reset')} do you want to use? [region: {self.region}, vpc: {vpc_id}] ======\n")
es = {}
count = 1
# note: list_domain_names() does not seems to support pagination
for es_cluster in self.es.list_domain_names()["DomainNames"]:
es[count] = {"name": es_cluster["DomainName"]}
count += 1
[print(" {} > {}".format(key, value["name"])) for key, value in es.items()]
allowed_choices = list(es.keys())
choice = get_input(f"Choose the ElasticSearch Cluster you want to use?", None, allowed_choices, int)
# note: describe_elasticsearch_domain() does not seems to support pagination
domain_info = self.es.describe_elasticsearch_domain(DomainName=es[choice]["name"])
if domain_info["DomainStatus"]["VPCOptions"]["VPCId"] == vpc_id:
for scope, endpoint in domain_info["DomainStatus"]["Endpoints"].items():
es[choice]["endpoint"] = endpoint
return {"success": True, "message": es[choice]}
except Exception as err:
return {"success": False, "message": str(err)}
def validate_sg_rules(self, cfn_params, check_fs=True):
try:
# Begin Verify Security Group Rules
print(f"\n====== Please wait a little as we {fg('misty_rose_3')}validate your security group rules {attr('reset')} ======\n")
security_groups = [cfn_params["scheduler_sg"], cfn_params["compute_node_sg"]]
if "vpc_endpoint_sg" in cfn_params:
security_groups.append(cfn_params["vpc_endpoint_sg"])
sg_rules = self.get_rules_for_security_group(security_groups)
if check_fs is True:
fs_sg = self.get_fs_security_groups(cfn_params)
if sg_rules["success"] is True:
scheduler_sg_rules = sg_rules["message"][cfn_params["scheduler_sg"]]
compute_node_sg_rules = sg_rules["message"][cfn_params["compute_node_sg"]]
vpc_endpoint_sg_rules = sg_rules["message"].get(cfn_params.get("vpc_endpoint_sg", None), None)
else:
print(f"{fg('red')}Error: {sg_rules['message']} {attr('reset')}")
sys.exit(1)
errors = {}
# status == True means that the check passed
errors["SCHEDULER_SG_IN_COMPUTE"] = {
"status": False,
"error": f"Compute Node SG must allow all TCP traffic from Scheduler SG",
"resolution": f"Add new rule on {cfn_params['compute_node_sg']} that allow TCP ports '0-65535' for {cfn_params['scheduler_sg']}"}
errors["COMPUTE_SG_IN_SCHEDULER"] = {
"status": False,
"error": f"Scheduler SG must allow all TCP traffic from Compute Node SG",
"resolution": f"Add a new rule on {cfn_params['scheduler_sg']} that allow TCP ports '0-65535' for {cfn_params['compute_node_sg']}"}
errors["CLIENT_IP_HTTPS_IN_SCHEDULER"] = {
"status": False,
"error": f"Client IP must be allowed for port 443 (80 optional) on Scheduler SG",
"resolution": f"Add two rules on {cfn_params['scheduler_sg']} that allow TCP ports 80 and 443 for {self.client_ip}"}
errors["CLIENT_IP_SSH_IN_SCHEDULER"] = {
"status": False,
"error": f"Client IP must be allowed for port 22 (SSH) on Scheduler SG",
"resolution": f"Add one rule on {cfn_params['scheduler_sg']} that allow TCP port 22 for {self.client_ip}"}
errors["SCHEDULER_SG_EQUAL_COMPUTE"] = {
"status": False,
"error": "Scheduler SG and Compute SG must be different",
"resolution": "You must choose two different security groups"}
errors["COMPUTE_SG_EGRESS_EFA"] = {
"status": False,
"error": "Compute SG must reference egress traffic to itself for EFA",
"resolution": f"Add a new (EGRESS) rule on {cfn_params['compute_node_sg']} that allow TCP ports '0-65535' for {cfn_params['compute_node_sg']}. Make sure you configure EGRESS rule and not INGRESS"}
if 'vpc_endpoint_sg' in cfn_params:
errors["COMPUTE_EGRESS_TO_VPC_ENDPOINTS"] = {
"status": False,
"error": "Compute SG must allow port 443 egress to the vpc endpoints security group",
"resolution": f"Add a new (EGRESS) rule on {cfn_params['compute_node_sg']} that allows TCP port '443' for {cfn_params['vpc_endpoint_sg']}. Make sure you configure EGRESS rule and not INGRESS"}
errors["VPC_ENDPOINTS_INGRESS_FROM_COMPUTE"] = {
"status": False,
"error": "vpc Endpoints SG must allow port 443 ingress from the Compute SG",
"resolution": f"Add a new (INGRESS) rule on {cfn_params['vpc_endpoint_sg']} that allows TCP port '443' from {cfn_params['compute_node_sg']}. Make sure you configure INGRESS rule and not EGRESS"}
errors["SCHEDULER_EGRESS_TO_VPC_ENDPOINTS"] = {
"status": False,
"error": "Scheduler SG must allow port 443 egress to the vpc endpoints security group",
"resolution": f"Add a new (EGRESS) rule on {cfn_params['scheduler_sg']} that allows TCP port '443' for {cfn_params['vpc_endpoint_sg']}. Make sure you configure EGRESS rule and not INGRESS"}
errors["VPC_ENDPOINTS_INGRESS_FROM_SCHEDULER"] = {
"status": False,
"error": "vpc Endpoints SG must allow port 443 ingress from the Scheduler SG",
"resolution": f"Add a new (INGRESS) rule on {cfn_params['vpc_endpoint_sg']} that allows TCP port '443' from {cfn_params['scheduler_sg']}. Make sure you configure INGRESS rule and not EGRESS"}
if check_fs is True:
errors["FS_APP_SG"] = {
"status": False,
"error": f"SG assigned to EFS App {cfn_params['fs_apps']} must allow Scheduler SG and Compute SG",
"resolution": f"Add {cfn_params['scheduler_sg']} and {cfn_params['compute_node_sg']} on your EFS Apps {cfn_params['fs_apps']}"}
errors["FS_DATA_SG"] = {
"status": False,
"error": f"SG assigned to EFS App {cfn_params['fs_data']} must allow Scheduler SG and Compute SG",
"resolution": f"Add {cfn_params['scheduler_sg']} and {cfn_params['compute_node_sg']} on your EFS Data {cfn_params['fs_data']}"}
# Verify Scheduler Rules
for rules in scheduler_sg_rules:
if rules["from_port"] == 0 and rules["to_port"] == 65535:
for rule in rules["approved_ips"]:
if cfn_params['compute_node_sg'] in rule:
errors["COMPUTE_SG_IN_SCHEDULER"]["status"] = True
if rules["from_port"] == 443 or rules["from_port"] == 22:
for rule in rules["approved_ips"]:
client_ip_netmask = 32
if client_ip_netmask == '32':
if ipaddress.IPv4Address(self.client_ip) in ipaddress.IPv4Network(rule):
if rules["from_port"] == 443:
errors["CLIENT_IP_HTTPS_IN_SCHEDULER"]["status"] = True
if rules["from_port"] == 22:
errors["CLIENT_IP_SSH_IN_SCHEDULER"]["status"] = True
else:
if self.client_ip in rule:
if rules["from_port"] == 443:
errors["CLIENT_IP_HTTPS_IN_SCHEDULER"]["status"] = True
if rules["from_port"] == 22:
errors["CLIENT_IP_SSH_IN_SCHEDULER"]["status"] = True
# Verify Compute Node Rules
for rules in compute_node_sg_rules:
if rules["from_port"] == 0 and rules["to_port"] == 65535:
for rule in rules["approved_ips"]:
if cfn_params['scheduler_sg'] in rule:
errors["SCHEDULER_SG_IN_COMPUTE"]["status"] = True
if rules["type"] == "egress":
if cfn_params['compute_node_sg'] in rule:
errors["COMPUTE_SG_EGRESS_EFA"]["status"] = True
# Verify VPC Endpoint Rules
if 'vpc_endpoint_sg' in cfn_params:
for rule in compute_node_sg_rules:
# Make sure compute node allows egress to vpc endpoints
if rule["type"] != "egress":
continue
for approved_ip in rule["approved_ips"]:
if rule["from_port"] <= 443 and rule["to_port"] >= 443:
if cfn_params['vpc_endpoint_sg'] in approved_ip:
errors["COMPUTE_EGRESS_TO_VPC_ENDPOINTS"]["status"] = True
for rule in scheduler_sg_rules:
# Make sure scheduler allows egress to vpc endpoints
if rule["type"] != "egress":
continue
for approved_ip in rule["approved_ips"]:
if rule["from_port"] <= 443 and rule["to_port"] >= 443:
if cfn_params['vpc_endpoint_sg'] in approved_ip:
errors["SCHEDULER_EGRESS_TO_VPC_ENDPOINTS"]["status"] = True
for rule in vpc_endpoint_sg_rules:
# Make sure endpoints allow ingress from compute nodes and scheduler
if rule["type"] != "ingress":
continue
for approved_ip in rule["approved_ips"]:
if rule["from_port"] <= 443 and rule["to_port"] >= 443:
if cfn_params['scheduler_sg'] in approved_ip:
errors["VPC_ENDPOINTS_INGRESS_FROM_SCHEDULER"]["status"] = True
if cfn_params['compute_node_sg'] in approved_ip:
errors["VPC_ENDPOINTS_INGRESS_FROM_COMPUTE"]["status"] = True
if check_fs is True:
if cfn_params['scheduler_sg'] in fs_sg["message"][cfn_params['fs_apps']] and cfn_params['compute_node_sg'] in fs_sg["message"][cfn_params['fs_apps']]:
errors["FS_APP_SG"]["status"] = True
if cfn_params['scheduler_sg'] in fs_sg["message"][cfn_params['fs_data']] and cfn_params['compute_node_sg'] in fs_sg["message"][cfn_params['fs_data']]:
errors["FS_DATA_SG"]["status"] = True
if cfn_params["scheduler_sg"] != cfn_params["compute_node_sg"]:
errors["SCHEDULER_SG_EQUAL_COMPUTE"]["status"] = True
sg_errors = {}
confirm_sg_settings = False
for error_id, error_info in errors.items():
if error_info["status"] is False:
if check_fs is False and "EFS" in error_id:
pass
else:
print(f"{fg('yellow')}ATTENTION!! {error_info['error']} {attr('reset')}\nHow to solve: {error_info['resolution']}\n")
sg_errors[error_info["error"]] = error_info["resolution"]
confirm_sg_settings = True
if confirm_sg_settings:
choice = get_input("Your security groups may not be configured correctly. Verify them and determine if the warnings listed above are false-positive.\n Do you still want to continue with the installation?",
None, ["yes", "no"], str)
if choice.lower() == "no":
sys.exit(1)
else:
print(f"{fg('green')} Security Groups seem to be configured correctly{attr('reset')}")
return {"success": True,
"message": ""}
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(f"{exc_type} {fname} {exc_tb.tb_lineno}")
return {"success": False, "message": f"{exc_type} {fname} {exc_tb.tb_lineno}"}
def get_fs(self, environment, vpc_id, selected_fs=[]):
try:
print(f"\n====== What {fg('misty_rose_3')}EFS/FSx for Lustre{attr('reset')} do you want to use for {fg('misty_rose_3')}{environment}{attr('reset')}? [region: {self.region}, vpc: {vpc_id}] ======\n")
filesystems = {}
count = 1
marker = True
next_marker = None
max_items = 50
while marker is True:
if not next_marker:
all_efs = self.efs.describe_file_systems(MaxItems=max_items)
else:
all_efs = self.efs.describe_file_systems(MaxItems=max_items, Marker=next_marker)
try:
next_marker = all_efs['Marker']
except KeyError:
marker = False
for filesystem in all_efs["FileSystems"]:
verified_vpc = False
for mount_target in self.efs.describe_mount_targets(FileSystemId=filesystem["FileSystemId"])["MountTargets"]:
if mount_target["VpcId"] == vpc_id:
verified_vpc = True
if verified_vpc is True:
if filesystem["FileSystemId"] not in selected_fs:
filesystems[count] = {"id": f"{filesystem['FileSystemId']}",
"description": f"EFS: {filesystem['Name'] if 'Name' in filesystem.keys() else 'EFS: '} {filesystem['FileSystemId']}.efs.{self.region}.amazonaws.com"}
count += 1
efs_count = count - 1
token = True
next_token = None
max_results = 50
while token is True:
if not next_token:
all_fsxl = self.fsx.describe_file_systems(MaxResults=max_results)
else:
all_fsxl = self.fsx.describe_file_systems(MaxResults=max_results, NextToken=next_token)
try:
next_token = all_fsxl['Token']
except KeyError:
token = False
for filesystem in all_fsxl["FileSystems"]:
resource_name = False
if filesystem["VpcId"] == vpc_id:
if filesystem["FileSystemId"] not in selected_fs:
for tag in filesystem['Tags']:
if tag["Key"] == 'Name':
resource_name = tag["Value"]
filesystems[count] = {"id": f"{filesystem['FileSystemId']}",
"description": f"FSX for Lustre: {resource_name if resource_name is not False else 'FSx for Lustre: '} {filesystem['FileSystemId']}.fsx.{self.region}.amazonaws.com"}
count += 1
[print(" {} > {}".format(key, value["description"])) for key, value in filesystems.items()]
allowed_choices = list(filesystems.keys())
choice = get_input(f"Choose the filesystem to use for {environment}?", None, allowed_choices, int)
if choice <= efs_count:
return {"success": True, "message": filesystems[choice]["id"], "provider": "efs"}
else:
return {"success": True, "message": filesystems[choice]["id"], "provider": "fsx_lustre"}
except Exception as err:
return {"success": False, "message": str(err)}
def get_install_parameters():
# Retrieve User Specified Variables
print("\n====== Validating SOCA Parameters ======\n")
install_parameters["cluster_name"] = get_input(f"[Step 1/{total_install_phases}] {install_phases[1]}", args.name,None, str)
while len(install_parameters["cluster_name"]) < 3 or len(install_parameters["cluster_name"]) > 11:
print(f"{fg('red')}SOCA cluster name must greater than 3 chars and shorter than 11 characters (soca- is automatically added as a prefix) {attr('reset')}")
install_parameters["cluster_name"] = get_input(f"[Step 1/{total_install_phases}] {install_phases[1]}", None, None, str)
# Sanitize cluster name (remove any non alphanumerical character) or generate random cluster identifier
sanitized_cluster_id = re.sub(r"\W+", "-", install_parameters["cluster_name"])
sanitized_cluster_id = re.sub(r"soca-", "", sanitized_cluster_id) # remove soca- if specified by the user
install_parameters["cluster_id"] = f"soca-{sanitized_cluster_id.lower()}" # do not remove soca- prefix or DCV IAM permission will not be working.
install_parameters["bucket"] = get_input(f"[Step 2/{total_install_phases}] {install_phases[2]}", args.bucket, None,
str)
while check_bucket_permission(install_parameters["bucket"]) is False:
install_parameters["bucket"] = get_input(f"[Step 2/{total_install_phases}] {install_phases[2]}", None, None, str)
install_parameters["ldap_user"] = get_input(f"[Step 3/{total_install_phases}] {install_phases[3]}", args.ldap_user, None, str)
while len(install_parameters["ldap_user"]) < 5 or not install_parameters["ldap_user"].isalnum():
print(f"{fg('red')}LDAP user must be 5 characters mins and can only contains alphanumeric.{attr('reset')}")
install_parameters["ldap_user"] = get_input(f"[Step 3/{total_install_phases}] {install_phases[3]}", None, None, str)
if install_props.Config.directoryservice.provider == "activedirectory":
while install_parameters["ldap_user"].lower() == "admin":
print(f"{fg('yellow')} To prevent conflict with Directory Service, the first SOCA user cannot be named admin. Please pick a different name.{attr('reset')}")
install_parameters["ldap_user"] = get_input(f"[Step 3/{total_install_phases}] {install_phases[3]}", None, None, str)
create_ldap_user = default_ldap_user_password()
while create_ldap_user is False:
create_ldap_user = default_ldap_user_password()
# Encode password to avoid any special char error while running bash CDK
install_parameters["ldap_password"] = (base64.b64encode(install_parameters["ldap_password"].encode("utf-8"))).decode("utf-8")
install_parameters["base_os"] = get_input(f"[Step 5/{total_install_phases}] {install_phases[5]}", args.base_os,["amazonlinux2", "centos7", "rhel7"], str)
install_parameters["ssh_keypair"] = get_input(f"[Step 6/{total_install_phases}] {install_phases[6]}", args.ssh_keypair, accepted_aws_values["accepted_keypairs"], str)
# Validate the prefix list id
if args.prefix_list_id:
try:
found_prefix_list_id = ec2.describe_managed_prefix_lists(PrefixListIds=[args.prefix_list_id])['PrefixLists'][0]['PrefixListId']
if found_prefix_list_id != args.prefix_list_id:
raise RuntimeError(f"Found prefix list {found_prefix_list_id} does not match {args.prefix_list_id}. This is a programming error; please create an issue.")
else:
install_parameters["prefix_list_id"] = args.prefix_list_id
except Exception as e:
print(f"{fg('red')}Error. {args.prefix_list_id} not found. Check that it exists and starts with pl-.\nException:\n{e} {attr('reset')}")
sys.exit(1)
install_parameters["custom_ami"] = args.custom_ami if args.custom_ami else None
# Network Configuration
cidr_regex = r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$'
if not args.vpc_cidr:
choice_vpc = get_input(f"[Step 7/{total_install_phases}] {install_phases[7]}", None, ["new", "existing"], str)
if choice_vpc == "new":
install_parameters["vpc_cidr"] = get_input("What CIDR do you want to use your your VPC? We recommend 10.0.0.0/16", args.vpc_cidr, None, str)
while not re.match(cidr_regex, install_parameters["vpc_cidr"]):
print(f"{fg('red')} Invalid CIDR {install_parameters['vpc_cidr']}. Format must be x.x.x.x/x (eg: 10.0.0.0/16){attr('reset')}")
install_parameters["vpc_cidr"] = get_input("What CIDR do you want to use your your VPC? We recommend 10.0.0.0/16", None, None, str)
else:
# List all VPCs running on AWS account
existing_vpc = FindExistingResource(install_parameters["region"], install_parameters["client_ip"]).find_vpc()
if existing_vpc["success"] is True:
install_parameters["vpc_id"] = existing_vpc["message"]["id"]
install_parameters["vpc_cidr"] = existing_vpc["message"]["cidr"]
else:
sys.exit(1)
# List all Subnets
public_subnets = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_subnets(install_parameters["vpc_id"], "public", [])
if public_subnets["success"] is True:
install_parameters["public_subnets"] = base64.b64encode(str(public_subnets["message"]).encode("utf-8")).decode("utf-8")
else:
print(f"{fg('red')}Error: {public_subnets['message']} {attr('reset')}")
sys.exit(1)
private_subnets = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_subnets(install_parameters["vpc_id"], "private", public_subnets["message"])
if private_subnets["success"] is True:
install_parameters["private_subnets"] = base64.b64encode(str(private_subnets["message"]).encode("utf-8")).decode("utf-8")
else:
print(f"{fg('red')}Error: {private_subnets['message']} {attr('reset')}")
sys.exit(1)
vpc_azs = []
for subnet in public_subnets["message"] + private_subnets["message"]:
az = subnet.split(",")[1]
if az not in vpc_azs:
vpc_azs.append(az)
install_parameters["vpc_azs"] = ",".join(vpc_azs)
else:
install_parameters["vpc_cidr"] = args.vpc_cidr
while not re.match(cidr_regex, install_parameters["vpc_cidr"]):
print(f"{fg('red')} Invalid CIDR {install_parameters['vpc_cidr']}. Format must be x.x.x.x/x (eg: 10.0.0.0/16){attr('reset')}")
install_parameters["vpc_cidr"] = get_input("What CIDR do you want to use your your VPC? We recommend 10.0.0.0/16", None, None, str)
# Security Groups Configuration (only possible if user use an existing VPC)
if install_parameters["vpc_id"]:
choice_security_groups = get_input(f"[Step 8/{total_install_phases}] {install_phases[8]}", None, ["new", "existing"], str)
if choice_security_groups == "existing":
scheduler_sg = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_security_groups(install_parameters["vpc_id"], "scheduler", [])
if scheduler_sg["success"] is True:
install_parameters["scheduler_sg"] = scheduler_sg["message"]
else:
print(f"{fg('red')}Error: {scheduler_sg['message']} {attr('reset')}")
sys.exit(1)
compute_node_sg = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_security_groups(install_parameters["vpc_id"], "compute nodes", install_parameters["scheduler_sg"])
if compute_node_sg["success"] is True:
install_parameters["compute_node_sg"] = compute_node_sg["message"]
else:
print(f"{fg('red')}Error: {compute_node_sg['message']} {attr('reset')}")
sys.exit(1)
if install_props.Config.network.vpc_interface_endpoints:
vpc_endpoint_sg = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_security_groups(install_parameters["vpc_id"], "vpc endpoints", install_parameters["scheduler_sg"])
if vpc_endpoint_sg["success"] is True:
install_parameters["vpc_endpoint_sg"] = vpc_endpoint_sg["message"]
else:
print(f"{fg('red')}Error: {vpc_endpoint_sg['message']} {attr('reset')}")
sys.exit(1)
else:
vpc_endpoint_sg = None
# Filesystem Configuration (only possible if user use an existing VPC)
if install_parameters["vpc_id"]:
choice_filesystem = get_input(f"[Step 9/{total_install_phases}] {install_phases[9]}", None, ["new", "existing"],str)
if choice_filesystem == "existing":
# List FS
fs_apps = FindExistingResource(install_parameters["region"], install_parameters["client_ip"]).get_fs("/apps", install_parameters["vpc_id"])
if fs_apps["success"] is True:
install_parameters["fs_apps_provider"] = fs_apps["provider"]
install_parameters["fs_apps"] = fs_apps["message"]
else:
print(f"{fg('red')}Error: {fs_apps['message']} {attr('reset')}")
sys.exit(1)
fs_data = FindExistingResource(install_parameters["region"], install_parameters["client_ip"]).get_fs("/data", install_parameters["vpc_id"], selected_fs=install_parameters["fs_apps"])
if fs_data["success"] is True:
install_parameters["fs_data_provider"] = fs_data["provider"]
install_parameters["fs_data"] = fs_data["message"]
if install_parameters["fs_data"] == install_parameters["fs_apps"]:
print(f"{fg('red')}Error: EFS/FSx for Lustre /apps and /data must be different {attr('reset')}")
sys.exit(1)
else:
print(f"{fg('red')}Error: {fs_data['message']} {attr('reset')}")
sys.exit(1)
# Verify SG permissions
if install_parameters["fs_apps"] or install_parameters["scheduler_sg"]:
FindExistingResource(install_parameters["region"], install_parameters["client_ip"]).validate_sg_rules(install_parameters, check_fs=True if install_parameters["fs_apps"] else False)
# AWS Directory Service Managed Active Directory configuration (only possible when using existing VPC)
if install_props.Config.directoryservice.provider == "activedirectory":
if install_parameters["vpc_id"]:
choice_mad = get_input(f"[Step 10/{total_install_phases}] {install_phases[10]}", None, ["new", "existing"],str)
if choice_mad == "existing":
directory_service = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).find_directory_services(install_parameters["vpc_id"])
if directory_service["success"] is True:
install_parameters["directory_service_ds_user"] = get_input(f"Username of a domain user with admin permissions?", None, None, str)
install_parameters["directory_service_ds_user_password"] = get_input(f"Password of the domain user with admin permissions", None, None, str)
install_parameters["directory_service"] = directory_service["message"]["id"]
install_parameters["directory_service_shortname"] = directory_service["message"]["netbios"]
install_parameters["directory_service_name"] = directory_service["message"]["name"]
install_parameters["directory_service_dns"] = directory_service["message"]["dns"]
else:
print(f"{fg('red')}Error: {directory_service['message']} {attr('reset')}")
sys.exit(1)
# ElasticSearch Configuration (only possible when using existing VPC)
if install_parameters["vpc_id"]:
choice_es = get_input(f"[Step 11/{total_install_phases}] {install_phases[11]}", None, ["new", "existing"], str)
if choice_es == "existing":
elasticsearch_cluster = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).find_elasticsearch(install_parameters["vpc_id"])
if elasticsearch_cluster["success"] is True:
install_parameters["es_endpoint"] = elasticsearch_cluster["message"]["endpoint"]
else:
print(f"{fg('red')}Error: {elasticsearch_cluster['message']} {attr('reset')}")
sys.exit(1)
else:
install_parameters["es_endpoint"] = None
# IAM Roles configuration (only possible when using existing VPC)
if install_parameters["vpc_id"]:
choice_iam_roles = get_input(f"[Step 12/{total_install_phases}] {install_phases[12]}", None, ["new", "existing"], str)
if choice_iam_roles == "existing":
scheduler_role = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_iam_roles("scheduler")
if scheduler_role["success"] is True:
install_parameters["scheduler_role_name"] = scheduler_role["message"]["name"]
install_parameters["scheduler_role_arn"] = scheduler_role["message"]["arn"]
else:
print(f"{fg('red')}Error: {scheduler_role['message']} {attr('reset')}")
sys.exit(1)
if get_input(f"Was this role generated by a previous SOCA deployment? If yes, are you also using the same S3 bucket?", None, ["yes", "no"], str) == "yes":
install_parameters["scheduler_role_from_previous_soca_deployment"] = True
else:
get_input(f"[IMPORTANT] Make sure this role is assumed by 'ec2.amazon.com' and 'ssm.amazonaws.com'\n Type ok to continue ...", None, ["ok"], str, color="yellow")
compute_node_role = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_iam_roles("compute nodes",
selected_roles=[install_parameters["scheduler_role_name"]])
if compute_node_role["success"] is True:
install_parameters["compute_node_role_name"] = compute_node_role["message"]["name"]
install_parameters["compute_node_role_arn"] = compute_node_role["message"]["arn"]
else:
print(f"{fg('red')}Error: {compute_node_role['message']} {attr('reset')}")
sys.exit(1)
if get_input(f"Was this role generated by a previous SOCA deployment?", None, ["yes", "no"], str) == "yes":
install_parameters["compute_node_role_from_previous_soca_deployment"] = True
else:
get_input(f"[IMPORTANT] Make sure this role is assumed by 'ec2.amazon.com' and 'ssm.amazonaws.com'\n Type ok to continue ...", None, ["ok"], str, color="yellow")
spotfleet_role = FindExistingResource(install_parameters["region"],
install_parameters["client_ip"]).get_iam_roles("spot fleet",
selected_roles=[install_parameters["scheduler_role_name"], install_parameters["compute_node_role_name"]])
if spotfleet_role["success"] is True:
install_parameters["spotfleet_role_name"] = spotfleet_role["message"]["name"]
install_parameters["spotfleet_role_arn"] = spotfleet_role["message"]["arn"]
else:
print(f"{fg('red')}Error: {spotfleet_role['message']} {attr('reset')}")
sys.exit(1)
if get_input(f"Was this role generated by a previous SOCA deployment?", None, ["yes", "no"], str) == "yes":
install_parameters["spotfleet_role_from_previous_soca_deployment"] = True
else:
get_input(f"[IMPORTANT] Make sure this role is assumed by 'spotfleet.amazonaws.com'\n Type ok to continue ...",None, ["ok"], str, color="yellow")
"compute_node_role_arn": None,
"computenode_role_from_previous_soca_deployment": None,
"scheduler_role_name": None,
"scheduler_role_arn": None,
"scheduler_role_from_previous_soca_deployment": None,
"spotfleet_role_name": None,
"spotfleet_role_arn": None,
"spotfleet_role_from_previous_soca_deployment": None,
# ElasticSearch
"es_domain": None,
}
print("\n====== Validating Default SOCA Configuration ======\n")
install_props = json.loads(json.dumps(get_install_properties(args.config)), object_hook=lambda d: SimpleNamespace(**d))
if not args.skip_config_message:
if get_input(f"SOCA will create AWS resources using the default parameters specified on installer/default_config.yml. \n Make sure you have read, reviewed and updated them (if needed). Enter 'yes' to continue ...", None, ["yes", "no"], str) != "yes":
sys.exit(1)
print("\n====== Validating AWS Environment ======\n")
# Load AWS custom profile if specified
if args.profile:
try:
session = boto3.session.Session(profile_name=args.profile)
except ProfileNotFound:
print(f"{fg('red')} Profile {args.profile} not found. Check ~/.aws/credentials file{attr('reset')}")
sys.exit(1)
else:
session = boto3.session.Session()
# Determine all AWS regions available on the account. We do not display opt-out region
default_region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")