def create_ebs_volumes(
worker_pattern,
size=100,
region_name=None,
):
"""
Create volumes on workers
Args:
worker_pattern (string): Worker name pattern e.g.:
cluster-55jx2-worker*
size (int): Size in GB (default: 100)
region_name (str): Region name (default: config.ENV_DATA['region'])
"""
aws = AWS(region_name)
region_name = region_name or config.ENV_DATA['region']
worker_instances = aws.get_instances_by_name_pattern(worker_pattern)
with parallel() as p:
for worker in worker_instances:
log.info(
f"Creating and attaching {size} GB volume to {worker['name']}")
p.spawn(
aws.create_volume_and_attach,
availability_zone=worker['avz'],
instance_id=worker['id'],
name=f"{worker['name']}_extra_volume",
size=size,
)
class AWSBase(Deployment):
def __init__(self):
"""
This would be base for both IPI and UPI deployment
"""
super(AWSBase, self).__init__()
self.region = config.ENV_DATA['region']
self.aws = AWSUtil(self.region)
def create_ebs_volumes(self, worker_pattern, size=100):
"""
Add new ebs volumes to the workers
Args:
worker_pattern (str): Worker name pattern e.g.:
cluster-55jx2-worker*
size (int): Size in GB (default: 100)
"""
worker_instances = self.aws.get_instances_by_name_pattern(
worker_pattern
)
with parallel() as p:
for worker in worker_instances:
logger.info(
f"Creating and attaching {size} GB "
f"volume to {worker['name']}"
)
p.spawn(
self.aws.create_volume_and_attach,
availability_zone=worker['avz'],
instance_id=worker['id'],
name=f"{worker['name']}_extra_volume",
size=size,
)
def add_volume(self, size=100):
"""
Add a new volume to all the workers
Args:
size (int): Size of volume in GB (default: 100)
"""
tfvars_file = "terraform.tfvars.json"
with open(os.path.join(self.cluster_path, tfvars_file)) as f:
tfvars = json.load(f)
cluster_id = tfvars['cluster_id']
worker_pattern = f'{cluster_id}-worker*'
logger.info(f'Worker pattern: {worker_pattern}')
self.create_ebs_volumes(worker_pattern, size)
def add_node(self):
# TODO: Implement later
super(AWSBase, self).add_node()
class AWSBase(CloudDeploymentBase):
# default storage class for StorageCluster CRD on AWS platform
DEFAULT_STORAGECLASS = "gp2"
def __init__(self):
"""
This would be base for both IPI and UPI deployment
"""
super(AWSBase, self).__init__()
self.aws = AWSUtil(self.region)
# dict of cluster prefixes with special handling rules (for existence
# check or during a cluster cleanup)
self.cluster_prefixes_special_rules = CLUSTER_PREFIXES_SPECIAL_RULES
def host_network_update(self):
"""
Update security group rules for HostNetwork
"""
cluster_id = get_infra_id(self.cluster_path)
worker_pattern = f"{cluster_id}-worker*"
worker_instances = self.aws.get_instances_by_name_pattern(
worker_pattern)
security_groups = worker_instances[0]["security_groups"]
sg_id = security_groups[0]["GroupId"]
security_group = self.aws.ec2_resource.SecurityGroup(sg_id)
# The ports are not 100 % clear yet. Taken from doc:
# https://docs.google.com/document/d/1c23ooTkW7cdbHNRbCTztprVU6leDqJxcvFZ1ZvK2qtU/edit#
security_group.authorize_ingress(
DryRun=False,
IpPermissions=[
{
"FromPort":
6800,
"ToPort":
7300,
"IpProtocol":
"tcp",
"UserIdGroupPairs": [
{
"Description": "Ceph OSDs",
"GroupId": sg_id,
},
],
},
{
"FromPort":
3300,
"ToPort":
3300,
"IpProtocol":
"tcp",
"UserIdGroupPairs": [
{
"Description": "Ceph MONs rule1",
"GroupId": sg_id,
},
],
},
{
"FromPort":
6789,
"ToPort":
6789,
"IpProtocol":
"tcp",
"UserIdGroupPairs": [
{
"Description": "Ceph MONs rule2",
"GroupId": sg_id,
},
],
},
{
"FromPort":
8443,
"ToPort":
8443,
"IpProtocol":
"tcp",
"UserIdGroupPairs": [
{
"Description": "Ceph Dashboard rule1",
"GroupId": sg_id,
},
],
},
{
"FromPort":
8080,
"ToPort":
8080,
"IpProtocol":
"tcp",
"UserIdGroupPairs": [
{
"Description": "Ceph Dashboard rule2",
"GroupId": sg_id,
},
],
},
],
)
def add_node(self):
# TODO: Implement later
super(AWSBase, self).add_node()
def check_cluster_existence(self, cluster_name_prefix):
"""
Check cluster existence according to cluster name prefix
Returns:
bool: True if a cluster with the same name prefix already exists,
False otherwise
"""
cluster_name_pattern = cluster_name_prefix + "*"
instances = self.aws.get_instances_by_name_pattern(
cluster_name_pattern)
instance_objs = [
self.aws.get_ec2_instance(ins.get("id")) for ins in instances
]
non_terminated_instances = [
ins for ins in instance_objs
if ins.state.get("Code") != constants.INSTANCE_TERMINATED
]
if non_terminated_instances:
logger.error(
f"Non terminated EC2 instances with the same name prefix were"
f" found: {[ins.id for ins in non_terminated_instances]}")
return True
return False
class AWSBase(Deployment):
def __init__(self):
"""
This would be base for both IPI and UPI deployment
"""
super(AWSBase, self).__init__()
self.region = config.ENV_DATA['region']
self.aws = AWSUtil(self.region)
if config.ENV_DATA.get('cluster_name'):
self.cluster_name = config.ENV_DATA['cluster_name']
else:
self.cluster_name = get_cluster_name(self.cluster_path)
def create_ebs_volumes(self, worker_pattern, size=100):
"""
Add new ebs volumes to the workers
Args:
worker_pattern (str): Worker name pattern e.g.:
cluster-55jx2-worker*
size (int): Size in GB (default: 100)
"""
worker_instances = self.aws.get_instances_by_name_pattern(
worker_pattern)
with parallel() as p:
for worker in worker_instances:
logger.info(f"Creating and attaching {size} GB "
f"volume to {worker['name']}")
p.spawn(
self.aws.create_volume_and_attach,
availability_zone=worker['avz'],
instance_id=worker['id'],
name=f"{worker['name']}_extra_volume",
size=size,
)
def add_volume(self, size=100):
"""
Add a new volume to all the workers
Args:
size (int): Size of volume in GB (default: 100)
"""
cluster_id = get_infra_id(self.cluster_path)
worker_pattern = f'{cluster_id}-worker*'
logger.info(f'Worker pattern: {worker_pattern}')
self.create_ebs_volumes(worker_pattern, size)
def host_network_update(self):
"""
Update security group rules for HostNetwork
"""
cluster_id = get_infra_id(self.cluster_path)
worker_pattern = f'{cluster_id}-worker*'
worker_instances = self.aws.get_instances_by_name_pattern(
worker_pattern)
security_groups = worker_instances[0]['security_groups']
sg_id = security_groups[0]['GroupId']
security_group = self.aws.ec2_resource.SecurityGroup(sg_id)
# The ports are not 100 % clear yet. Taken from doc:
# https://docs.google.com/document/d/1c23ooTkW7cdbHNRbCTztprVU6leDqJxcvFZ1ZvK2qtU/edit#
security_group.authorize_ingress(
DryRun=False,
IpPermissions=[
{
'FromPort':
6800,
'ToPort':
7300,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph OSDs',
'GroupId': sg_id,
},
],
},
{
'FromPort':
3300,
'ToPort':
3300,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph MONs rule1',
'GroupId': sg_id,
},
],
},
{
'FromPort':
6789,
'ToPort':
6789,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph MONs rule2',
'GroupId': sg_id,
},
],
},
{
'FromPort':
8443,
'ToPort':
8443,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph Dashboard rule1',
'GroupId': sg_id,
},
],
},
{
'FromPort':
8080,
'ToPort':
8080,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph Dashboard rule2',
'GroupId': sg_id,
},
],
},
])
def add_node(self):
# TODO: Implement later
super(AWSBase, self).add_node()
def check_cluster_existence(self, cluster_name_prefix):
"""
Check cluster existence according to cluster name prefix
Returns:
bool: True if a cluster with the same name prefix already exists,
False otherwise
"""
instances = self.aws.get_instances_by_name_pattern(cluster_name_prefix)
instance_objs = [
self.aws.get_ec2_instance(ins.get('id')) for ins in instances
]
non_terminated_instances = [
ins for ins in instance_objs
if ins.state.get('Code') != constants.INSTANCE_TERMINATED
]
if non_terminated_instances:
logger.error(
f"Non terminated EC2 instances with the same name prefix were"
f" found: {[ins.id for ins in non_terminated_instances]}")
return True
return False
def get_clusters(time_to_delete, region_name, prefixes_hours_to_spare):
"""
Get all cluster names that their EC2 instances running time is greater
than the specified time to delete
Args:
time_to_delete (int): The maximum time in seconds that is allowed
for clusters to continue running
region_name (str): The name of the AWS region to delete the resources from
prefixes_hours_to_spare (dict): Dictionaries of the cluster prefixes to spare
along with the maximum time in hours that is allowed for spared
clusters to continue running
Returns:
tuple: List of the cluster names (e.g ebenahar-cluster-gqtd4) to be provided to the
ci-cleanup script, a list of VPCs that are part of cloudformation,
and a list of remaining clusters
"""
def determine_cluster_deletion(ec2_instances, cluster_name):
for instance in ec2_instances:
allowed_running_time = time_to_delete
do_not_delete = False
if instance.state["Name"] == "running":
for prefix, hours in prefixes_hours_to_spare.items():
# case insensitive 'startswith'
if bool(re.match(prefix, cluster_name, re.I)):
if hours == 'never':
do_not_delete = True
else:
allowed_running_time = int(hours) * 60 * 60
break
if do_not_delete:
logger.info(
"%s marked as 'do not delete' and will not be "
"destroyed", cluster_name)
return False
else:
launch_time = instance.launch_time
current_time = datetime.datetime.now(launch_time.tzinfo)
running_time = current_time - launch_time
logger.info(
f"Instance {[tag['Value'] for tag in instance.tags if tag['Key'] == 'Name'][0]} "
f"(id: {instance.id}) running time is {running_time} hours while the allowed"
f" running time for it is {allowed_running_time/3600} hours"
)
if running_time.total_seconds() > allowed_running_time:
return True
return False
aws = AWS(region_name=region_name)
clusters_to_delete = list()
remaining_clusters = list()
cloudformation_vpc_names = list()
vpcs = aws.ec2_client.describe_vpcs()['Vpcs']
vpc_ids = [vpc['VpcId'] for vpc in vpcs]
vpc_objs = [aws.ec2_resource.Vpc(vpc_id) for vpc_id in vpc_ids]
for vpc_obj in vpc_objs:
vpc_tags = vpc_obj.tags
if vpc_tags:
cloudformation_vpc_name = [
tag['Value'] for tag in vpc_tags
if tag['Key'] == defaults.AWS_CLOUDFORMATION_TAG
]
if cloudformation_vpc_name:
cloudformation_vpc_names.append(cloudformation_vpc_name[0])
continue
vpc_name = [
tag['Value'] for tag in vpc_tags if tag['Key'] == 'Name'
][0]
cluster_name = vpc_name.replace('-vpc', '')
vpc_instances = vpc_obj.instances.all()
if not vpc_instances:
clusters_to_delete.append(cluster_name)
continue
# Append to clusters_to_delete if cluster should be deleted
if determine_cluster_deletion(vpc_instances, cluster_name):
clusters_to_delete.append(cluster_name)
else:
remaining_clusters.append(cluster_name)
else:
logger.info("No tags found for VPC")
# Get all cloudformation based clusters to delete
cf_clusters_to_delete = list()
for vpc_name in cloudformation_vpc_names:
instance_dicts = aws.get_instances_by_name_pattern(
f"{vpc_name.replace('-vpc', '')}*")
ec2_instances = [
aws.get_ec2_instance(instance_dict['id'])
for instance_dict in instance_dicts
]
if not ec2_instances:
continue
cluster_io_tag = None
for instance in ec2_instances:
cluster_io_tag = [
tag['Key'] for tag in instance.tags
if 'kubernetes.io/cluster' in tag['Key']
]
if cluster_io_tag:
break
if not cluster_io_tag:
logger.warning(
"Unable to find valid cluster IO tag from ec2 instance tags "
"for VPC %s. This is probably not an OCS cluster VPC!",
vpc_name)
continue
cluster_name = cluster_io_tag[0].replace('kubernetes.io/cluster/', '')
if determine_cluster_deletion(ec2_instances, cluster_name):
cf_clusters_to_delete.append(cluster_name)
else:
remaining_clusters.append(cluster_name)
return clusters_to_delete, cf_clusters_to_delete, remaining_clusters
class AWSBase(CloudDeploymentBase):
# default storage class for StorageCluster CRD on AWS platform
DEFAULT_STORAGECLASS = "gp2"
def __init__(self):
"""
This would be base for both IPI and UPI deployment
"""
super(AWSBase, self).__init__()
self.aws = AWSUtil(self.region)
def host_network_update(self):
"""
Update security group rules for HostNetwork
"""
cluster_id = get_infra_id(self.cluster_path)
worker_pattern = f'{cluster_id}-worker*'
worker_instances = self.aws.get_instances_by_name_pattern(
worker_pattern)
security_groups = worker_instances[0]['security_groups']
sg_id = security_groups[0]['GroupId']
security_group = self.aws.ec2_resource.SecurityGroup(sg_id)
# The ports are not 100 % clear yet. Taken from doc:
# https://docs.google.com/document/d/1c23ooTkW7cdbHNRbCTztprVU6leDqJxcvFZ1ZvK2qtU/edit#
security_group.authorize_ingress(
DryRun=False,
IpPermissions=[
{
'FromPort':
6800,
'ToPort':
7300,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph OSDs',
'GroupId': sg_id,
},
],
},
{
'FromPort':
3300,
'ToPort':
3300,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph MONs rule1',
'GroupId': sg_id,
},
],
},
{
'FromPort':
6789,
'ToPort':
6789,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph MONs rule2',
'GroupId': sg_id,
},
],
},
{
'FromPort':
8443,
'ToPort':
8443,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph Dashboard rule1',
'GroupId': sg_id,
},
],
},
{
'FromPort':
8080,
'ToPort':
8080,
'IpProtocol':
'tcp',
'UserIdGroupPairs': [
{
'Description': 'Ceph Dashboard rule2',
'GroupId': sg_id,
},
],
},
])
def add_node(self):
# TODO: Implement later
super(AWSBase, self).add_node()
def check_cluster_existence(self, cluster_name_prefix):
"""
Check cluster existence according to cluster name prefix
Returns:
bool: True if a cluster with the same name prefix already exists,
False otherwise
"""
cluster_name_pattern = cluster_name_prefix + "*"
instances = self.aws.get_instances_by_name_pattern(
cluster_name_pattern)
instance_objs = [
self.aws.get_ec2_instance(ins.get('id')) for ins in instances
]
non_terminated_instances = [
ins for ins in instance_objs
if ins.state.get('Code') != constants.INSTANCE_TERMINATED
]
if non_terminated_instances:
logger.error(
f"Non terminated EC2 instances with the same name prefix were"
f" found: {[ins.id for ins in non_terminated_instances]}")
return True
return False
class ROSA(CloudDeploymentBase):
"""
Deployment class for ROSA.
"""
OCPDeployment = ROSAOCP
def __init__(self):
self.name = self.__class__.__name__
super(ROSA, self).__init__()
ocm.download_ocm_cli()
rosa.download_rosa_cli()
self.aws = AWSUtil(self.region)
def deploy_ocp(self, log_cli_level="DEBUG"):
"""
Deployment specific to OCP cluster on a cloud platform.
Args:
log_cli_level (str): openshift installer's log level
(default: "DEBUG")
"""
ocm.login()
super(ROSA, self).deploy_ocp(log_cli_level)
if config.DEPLOYMENT.get("host_network"):
self.host_network_update()
def check_cluster_existence(self, cluster_name_prefix):
"""
Check cluster existence based on a cluster name.
Args:
cluster_name_prefix (str): name prefix which identifies a cluster
Returns:
bool: True if a cluster with the same name prefix already exists,
False otherwise
"""
cluster_list = ocm.list_cluster()
for cluster in cluster_list:
name, state = cluster
if state != "uninstalling" and name.startswith(cluster_name_prefix):
return True
return False
def deploy_ocs(self):
"""
Deployment of ODF Managed Service addon on ROSA.
"""
ceph_cluster = ocp.OCP(kind="CephCluster", namespace=self.namespace)
try:
ceph_cluster.get().get("items")[0]
logger.warning("OCS cluster already exists")
return
except (IndexError, CommandFailed):
logger.info("Running OCS basic installation")
rosa.install_odf_addon(self.cluster_name)
pod = ocp.OCP(kind=constants.POD, namespace=self.namespace)
if config.ENV_DATA.get("cluster_type") != "consumer":
# Check for Ceph pods
assert pod.wait_for_resource(
condition="Running",
selector=constants.MON_APP_LABEL,
resource_count=3,
timeout=600,
)
assert pod.wait_for_resource(
condition="Running", selector=constants.MGR_APP_LABEL, timeout=600
)
assert pod.wait_for_resource(
condition="Running",
selector=constants.OSD_APP_LABEL,
resource_count=3,
timeout=600,
)
if config.DEPLOYMENT.get("pullsecret_workaround"):
update_pull_secret()
if config.ENV_DATA.get("cluster_type") == "consumer":
patch_consumer_toolbox()
# Verify health of ceph cluster
ceph_health_check(namespace=self.namespace, tries=60, delay=10)
def destroy_ocs(self):
"""
Uninstall ODF Managed Service addon via rosa cli.
"""
cluster_namespace = config.ENV_DATA["cluster_namespace"]
# Deleting PVCs
rbd_pvcs = [
p
for p in pvc.get_all_pvcs_in_storageclass(constants.CEPHBLOCKPOOL_SC)
if not (
p.data["metadata"]["namespace"] == cluster_namespace
and p.data["metadata"]["labels"]["app"] == "noobaa"
)
]
pvc.delete_pvcs(rbd_pvcs)
cephfs_pvcs = pvc.get_all_pvcs_in_storageclass(constants.CEPHFILESYSTEM_SC)
pvc.delete_pvcs(cephfs_pvcs)
rosa.delete_odf_addon(self.cluster_name)
def host_network_update(self):
"""
Update security group rules for HostNetwork
"""
infrastructure_id = ocp.OCP().exec_oc_cmd(
"get -o jsonpath='{.status.infrastructureName}{\"\\n\"}' infrastructure cluster"
)
worker_pattern = f"{infrastructure_id}-worker*"
worker_instances = self.aws.get_instances_by_name_pattern(worker_pattern)
security_groups = worker_instances[0]["security_groups"]
sg_id = security_groups[0]["GroupId"]
security_group = self.aws.ec2_resource.SecurityGroup(sg_id)
# The ports are not 100 % clear yet. Taken from doc:
# https://docs.google.com/document/d/1RM8tmMbvnJcOZFdsqbCl9RvHXBv5K2ZI6ziQ-YTloGk/edit#
security_group.authorize_ingress(
DryRun=False,
IpPermissions=[
{
"FromPort": 6800,
"ToPort": 7300,
"IpProtocol": "tcp",
"IpRanges": [
{"CidrIp": "10.0.0.0/16", "Description": "Ceph OSDs"},
],
},
{
"FromPort": 3300,
"ToPort": 3300,
"IpProtocol": "tcp",
"IpRanges": [
{"CidrIp": "10.0.0.0/16", "Description": "Ceph MONs rule1"}
],
},
{
"FromPort": 6789,
"ToPort": 6789,
"IpProtocol": "tcp",
"IpRanges": [
{"CidrIp": "10.0.0.0/16", "Description": "Ceph MONs rule2"},
],
},
{
"FromPort": 9283,
"ToPort": 9283,
"IpProtocol": "tcp",
"IpRanges": [
{"CidrIp": "10.0.0.0/16", "Description": "Ceph Manager"},
],
},
{
"FromPort": 31659,
"ToPort": 31659,
"IpProtocol": "tcp",
"IpRanges": [
{"CidrIp": "10.0.0.0/16", "Description": "API Server"},
],
},
],
)