def add_worker_based_on_pods_count_per_node(node_count,
expected_count,
role_type=None,
machineset_name=None):
"""
Function to evaluate number of pods up in node and add new node accordingly.
Args:
machineset_name (list): Machineset_names to add more nodes if required.
node_count (int): Additional nodes to be added
expected_count (int): Expected pod count in one node
role_type (str): To add type to the nodes getting added
Returns:
bool: True if Nodes gets added, else false.
"""
# Check for POD running count on each nodes
if (config.ENV_DATA["deployment_type"] == "ipi"
and config.ENV_DATA["platform"].lower() == "aws"):
app_nodes = node.get_nodes(node_type=role_type)
pod_count_dict = node.get_running_pod_count_from_node(
node_type=role_type)
high_count_nodes, less_count_nodes = ([] for i in range(2))
for node_obj in app_nodes:
count = pod_count_dict[f"{node_obj.name}"]
if count >= expected_count:
high_count_nodes.append(node_obj.name)
else:
less_count_nodes.append(node_obj.name)
if len(less_count_nodes) <= 1:
for name in machineset_name:
count = machine.get_replica_count(machine_set=name)
machine.add_node(machine_set=name, count=(count + node_count))
machine.wait_for_new_node_to_be_ready(name)
return True
else:
logging.info(
f"Enough pods can be created with available nodes {pod_count_dict}"
)
return False
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "vsphere"):
raise UnsupportedPlatformError("Unsupported Platform to add worker")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "baremetal"):
raise UnsupportedPlatformError("Unsupported Platform to add worker")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "azure"):
raise UnsupportedPlatformError("Unsupported Platform to add worker")
def add_worker_based_on_cpu_utilization(node_count,
expected_percent,
role_type=None,
machineset_name=None):
"""
Function to evaluate CPU utilization of nodes and add node if required.
Args:
machineset_name (list): Machineset_names to add more nodes if required.
node_count (int): Additional nodes to be added
expected_percent (int): Expected utilization precent
role_type (str): To add type to the nodes getting added
Returns:
bool: True if Nodes gets added, else false.
"""
# Check for CPU utilization on each nodes
if (config.ENV_DATA["deployment_type"] == "ipi"
and config.ENV_DATA["platform"].lower() == "aws"):
app_nodes = node.get_nodes(node_type=role_type)
uti_dict = node.get_node_resource_utilization_from_oc_describe(
node_type=role_type)
uti_high_nodes, uti_less_nodes = ([] for i in range(2))
for node_obj in app_nodes:
utilization_percent = uti_dict[f"{node_obj.name}"]["cpu"]
if utilization_percent > expected_percent:
uti_high_nodes.append(node_obj.name)
else:
uti_less_nodes.append(node_obj.name)
if len(uti_less_nodes) <= 1:
for name in machineset_name:
count = machine.get_replica_count(machine_set=name)
machine.add_node(machine_set=name, count=(count + node_count))
machine.wait_for_new_node_to_be_ready(name)
return True
else:
logging.info(
f"Enough resource available for more pod creation {uti_dict}")
return False
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "vsphere"):
raise UnsupportedPlatformError("Unsupported Platform to add worker")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "baremetal"):
raise UnsupportedPlatformError("Unsupported Platform to add worker")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "azure"):
raise UnsupportedPlatformError("Unsupported Platform to add worker")
def check_enough_resource_available_in_workers(ms_name=None,
pod_dict_path=None):
"""
Function to check if there is enough resource in worker, if not add worker
for automation supported platforms
Args:
ms_name (list): Require param in-case of aws platform to increase the worker
pod_dict_path (str): Pod dict path for nginx pod.
"""
# Check for enough worker nodes
if (config.ENV_DATA["deployment_type"] == "ipi"
and config.ENV_DATA["platform"].lower() == "aws"):
if pod_dict_path == constants.NGINX_POD_YAML:
# Below expected count value is kind of hardcoded based on the manual
# execution result i.e. With m5.4xlarge instance and nginx pod
# TODO: Revisit the expected_count value once there is support for
# TODO: more pod creation in one worker node
if add_worker_based_on_pods_count_per_node(
machineset_name=ms_name,
node_count=1,
expected_count=140,
role_type="app,worker",
):
logging.info("Nodes added for app pod creation")
else:
logging.info(
"Existing resource are enough to create more pods")
else:
if add_worker_based_on_cpu_utilization(
machineset_name=ms_name,
node_count=1,
expected_percent=59,
role_type="app,worker",
):
logging.info("Nodes added for app pod creation")
else:
logging.info(
"Existing resource are enough to create more pods")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "vsphere"):
raise UnsupportedPlatformError("Unsupported Platform")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "baremetal"):
raise UnsupportedPlatformError("Unsupported Platform")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "azure"):
raise UnsupportedPlatformError("Unsupported Platform")
def get_expected_worker_count(scale_count=1500):
"""
Function to get expected worker count based on platform to scale pods in cluster
Args:
scale_count (int): Scale count of the PVC+POD to be created
Returns:
expected_worker_count (int): Expected worker count to scale required number of pod
"""
# Get expected worker count based on dict in constants.py
worker_count_dict = constants.SCALE_WORKER_DICT
if scale_count in worker_count_dict:
if (config.ENV_DATA["deployment_type"] == "ipi"
and config.ENV_DATA["platform"].lower() == "aws"):
expected_worker_count = worker_count_dict[scale_count]["aws"]
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "vsphere"):
expected_worker_count = worker_count_dict[scale_count]["vmware"]
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "baremetal"):
expected_worker_count = worker_count_dict[scale_count]["bm"]
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "azure"):
expected_worker_count = worker_count_dict[scale_count]["azure"]
else:
raise UnsupportedPlatformError("Unsupported Platform")
return expected_worker_count
else:
raise UnexpectedBehaviour(
"Scale_count value is not matching the dict key")
def get_device_paths(worker_names):
"""
Retrieve a list of the device paths for each worker node
Args:
worker_names (list): worker node names
Returns:
list: device path ids
"""
device_paths = []
platform = config.ENV_DATA.get('platform').lower()
if platform == 'aws':
pattern = 'nvme-Amazon_EC2_NVMe_Instance_Storage'
# TODO: add patterns for vsphere and bare metal
else:
raise UnsupportedPlatformError(
'LSO deployment is not supported for platform: %s', platform)
for worker in worker_names:
logger.info("Retrieving device path for node: %s", worker)
cmd = (f"oc debug nodes/{worker} "
f"-- chroot /host ls -la /dev/disk/by-id/")
out = run_cmd(cmd)
out_lines = out.split('\n')
nvme_lines = [line for line in out_lines if pattern in line]
for nvme_line in nvme_lines:
device_path = [
part for part in nvme_line.split(' ') if pattern in part
][0]
logger.info("Adding %s to device paths", device_path)
device_paths.append(f'/dev/disk/by-id/{device_path}')
return device_paths
def get_device_paths(worker_names):
"""
Retrieve a list of the device paths for each worker node
Args:
worker_names (list): worker node names
Returns:
list: device path ids
"""
device_paths = []
platform = config.ENV_DATA.get("platform").lower()
if platform == constants.IBM_POWER_PLATFORM:
device_paths = config.ENV_DATA.get("disk_pattern").lower()
return [device_paths]
if platform == "aws":
pattern = "nvme-Amazon_EC2_NVMe_Instance_Storage"
elif platform == "vsphere":
pattern = "wwn"
elif platform == "baremetal":
pattern = config.ENV_DATA.get("disk_pattern")
elif platform == "baremetalpsi":
pattern = "virtio"
# TODO: add patterns bare metal
else:
raise UnsupportedPlatformError(
"LSO deployment is not supported for platform: %s", platform)
for worker in worker_names:
logger.info("Retrieving device path for node: %s", worker)
out = _get_disk_by_id(worker)
out_lines = out.split("\n")
nvme_lines = [
line for line in out_lines
if (pattern in line and constants.ROOT_DISK_NAME not in line)
]
for nvme_line in nvme_lines:
device_path = [
part for part in nvme_line.split(" ") if pattern in part
][0]
logger.info("Adding %s to device paths", device_path)
device_paths.append(f"/dev/disk/by-id/{device_path}")
return device_paths
def get_device_paths(worker_names):
"""
Retrieve a list of the device paths for each worker node
Args:
worker_names (list): worker node names
Returns:
list: device path ids
"""
device_paths = []
platform = config.ENV_DATA.get('platform').lower()
if platform == 'aws':
pattern = 'nvme-Amazon_EC2_NVMe_Instance_Storage'
elif platform == 'vsphere':
pattern = 'wwn'
elif platform == 'baremetal':
pattern = config.ENV_DATA.get('disk_pattern')
elif platform == 'baremetalpsi':
pattern = 'virtio'
# TODO: add patterns bare metal
else:
raise UnsupportedPlatformError(
'LSO deployment is not supported for platform: %s', platform
)
for worker in worker_names:
logger.info("Retrieving device path for node: %s", worker)
out = _get_disk_by_id(worker)
out_lines = out.split('\n')
nvme_lines = [
line for line in out_lines if (
pattern in line and constants.ROOT_DISK_NAME not in line
)
]
for nvme_line in nvme_lines:
device_path = [
part for part in nvme_line.split(' ') if pattern in part
][0]
logger.info("Adding %s to device paths", device_path)
device_paths.append(f'/dev/disk/by-id/{device_path}')
return device_paths
def get_gather_bootstrap_node_data():
"""
Retrieve node IPs required by the gather bootstrap command
Raises:
UnsupportedPlatformError: If we do not support gathering bootstrap
data for the configured provider
Returns:
dict: Public IP of the bootstrap node and Private IPs of master nodes
"""
logger.info("Retrieving bootstrap node data")
platform = config.ENV_DATA['platform'].lower()
if platform == constants.AWS_PLATFORM:
return get_node_data_aws()
elif platform == constants.VSPHERE_PLATFORM:
return get_node_data_vsphere()
else:
raise UnsupportedPlatformError(
"Platform '%s' is not supported, "
"unable to retrieve gather bootstrap node data", platform)
def check_and_add_enough_worker(worker_count):
"""
Function to check if there is enough workers available to scale pods.
IF there is no enough worker then worker will be added based on supported platforms
Function also adds scale label to the respective worker nodes.
Args:
worker_count (int): Expected worker count to be present in the setup
Returns:
book: True is there is enough worker count else raise exception.
"""
# Check either to use OCS workers for scaling app pods
# Further continue to label the worker with scale label else not
worker_list = node.get_worker_nodes()
ocs_worker_list = machine.get_labeled_nodes(constants.OPERATOR_NODE_LABEL)
scale_worker = machine.get_labeled_nodes(constants.SCALE_LABEL)
if config.RUN.get("use_ocs_worker_for_scale"):
if not scale_worker:
helpers.label_worker_node(node_list=worker_list,
label_key="scale-label",
label_value="app-scale")
else:
if not scale_worker:
for node_item in ocs_worker_list:
worker_list.remove(node_item)
if worker_list:
helpers.label_worker_node(
node_list=worker_list,
label_key="scale-label",
label_value="app-scale",
)
scale_worker_list = machine.get_labeled_nodes(constants.SCALE_LABEL)
logging.info(f"Print existing scale worker {scale_worker_list}")
# Check if there is enough nodes to continue scaling of app pods
if len(scale_worker_list) >= worker_count:
logging.info(f"Setup has expected worker count {worker_count} "
"to continue scale of pods")
return True
else:
logging.info(
"There is no enough worker in the setup, will add enough worker "
"for the automation supported platforms")
# Add enough worker for AWS
if (config.ENV_DATA["deployment_type"] == "ipi"
and config.ENV_DATA["platform"].lower() == "aws"):
# Create machineset for app worker nodes on each aws zone
# Each zone will have one app worker node
ms_name = list()
labels = [("node-role.kubernetes.io/app", "app-scale")]
for obj in machine.get_machineset_objs():
if "app" in obj.name:
ms_name.append(obj.name)
if not ms_name:
if len(machine.get_machineset_objs()) == 3:
for zone in ["a", "b", "c"]:
ms_name.append(
machine.create_custom_machineset(
instance_type="m5.4xlarge",
labels=labels,
zone=zone,
))
else:
ms_name.append(
machine.create_custom_machineset(
instance_type="m5.4xlarge",
labels=labels,
zone="a",
))
for ms in ms_name:
machine.wait_for_new_node_to_be_ready(ms)
if len(ms_name) == 3:
exp_count = int(worker_count / 3)
else:
exp_count = worker_count
for name in ms_name:
machine.add_node(machine_set=name, count=exp_count)
for ms in ms_name:
machine.wait_for_new_node_to_be_ready(ms)
worker_list = node.get_worker_nodes()
ocs_worker_list = machine.get_labeled_nodes(
constants.OPERATOR_NODE_LABEL)
scale_label_worker = machine.get_labeled_nodes(
constants.SCALE_LABEL)
ocs_worker_list.extend(scale_label_worker)
final_list = list(dict.fromkeys(ocs_worker_list))
for node_item in final_list:
if node_item in worker_list:
worker_list.remove(node_item)
if worker_list:
helpers.label_worker_node(
node_list=worker_list,
label_key="scale-label",
label_value="app-scale",
)
return True
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "vsphere"):
raise UnsupportedPlatformError(
"Unsupported Platform to add worker")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "baremetal"):
raise UnsupportedPlatformError(
"Unsupported Platform to add worker")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "azure"):
raise UnsupportedPlatformError(
"Unsupported Platform to add worker")
else:
raise UnavailableResourceException(
"There is no enough worker nodes to continue app pod scaling")
def create_custom_machineset(
role="app",
instance_type="m4.xlarge",
labels=None,
taints=None,
zone="a",
):
"""
Function to create custom machineset works only for AWS
i.e. Using this user can create nodes with different instance type and role.
https://docs.openshift.com/container-platform/4.1/machine_management/creating-machineset.html
Args:
role (str): Role type to be added for node eg: it will be app,worker
instance_type (str): Type of aws instance
label (list): List of Labels (key, val) to be added to the node
taints (list): List of taints to be applied
zone (str): Machineset zone for node creation.
Returns:
machineset (str): Created machineset name
Raise:
ResourceNotFoundError: Incase machineset creation failed
UnsupportedPlatformError: Incase of wrong platform
"""
# check for platform, since it's supported only for IPI
if config.ENV_DATA["deployment_type"] == "ipi":
machinesets_obj = OCP(
kind=constants.MACHINESETS,
namespace=constants.OPENSHIFT_MACHINE_API_NAMESPACE,
)
for machine in machinesets_obj.get()["items"]:
# Get inputs from existing machineset config.
region = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("placement").get("region"))
aws_zone = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("placement").get(
"availabilityZone"))
cls_id = (machine.get("spec").get("selector").get(
"matchLabels").get("machine.openshift.io/cluster-api-cluster"))
ami_id = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("ami").get("id"))
if aws_zone == f"{region}{zone}":
machineset_yaml = templating.load_yaml(
constants.MACHINESET_YAML)
# Update machineset_yaml with required values.
machineset_yaml["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["metadata"][
"name"] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-role"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-type"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["ami"]["id"] = ami_id
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["iamInstanceProfile"][
"id"] = f"{cls_id}-worker-profile"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["instanceType"] = instance_type
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["placement"]["availabilityZone"] = aws_zone
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["placement"]["region"] = region
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["securityGroups"][0]["filters"][0]["values"][
0] = f"{cls_id}-worker-sg"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["subnet"]["filters"][0]["values"][
0] = f"{cls_id}-private-{aws_zone}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["tags"][0][
"name"] = f"kubernetes.io/cluster/{cls_id}"
# Apply the labels
if labels:
for label in labels:
machineset_yaml["spec"]["template"]["spec"][
"metadata"]["labels"][label[0]] = label[1]
# Apply the Taints
# ex taint list looks like:
# [ {'effect': 'NoSchedule',
# 'key': 'node.ocs.openshift.io/storage',
# 'value': 'true',
# }, {'effect': 'Schedule', 'key': 'xyz', 'value': 'False'} ]
if taints:
machineset_yaml["spec"]["template"]["spec"].update(
{"taints": taints})
# Create new custom machineset
ms_obj = OCS(**machineset_yaml)
ms_obj.create()
if check_machineset_exists(f"{cls_id}-{role}-{aws_zone}"):
logging.info(
f"Machineset {cls_id}-{role}-{aws_zone} created")
return f"{cls_id}-{role}-{aws_zone}"
else:
raise ResourceNotFoundError(
"Machineset resource not found")
else:
raise UnsupportedPlatformError("Functionality not supported in UPI")
def create_connection(self, cld_mgr, platform, conn_name=None):
"""
Creates a new NooBaa connection to an AWS backend
Args:
cld_mgr (obj): A cloud manager instance
platform (str): Platform to use for new connection
conn_name (str): The connection name to be used
If None provided then the name will be generated
Returns:
bool: False if the connection creation failed
"""
if conn_name is None:
conn_name = create_unique_resource_name(f"{platform}-connection",
"mcgconn")
if platform == constants.AWS_PLATFORM:
params = {
"auth_method": "AWS_V4",
"endpoint": constants.MCG_NS_AWS_ENDPOINT,
"endpoint_type": "AWS",
"identity": cld_mgr.aws_client.access_key,
"name": conn_name,
"secret": cld_mgr.aws_client.secret_key,
}
elif platform == constants.AZURE_PLATFORM:
params = {
"endpoint": constants.MCG_NS_AZURE_ENDPOINT,
"endpoint_type": "AZURE",
"identity": cld_mgr.azure_client.account_name,
"name": conn_name,
"secret": cld_mgr.azure_client.credential,
}
elif platform == constants.RGW_PLATFORM:
params = {
"auth_method": "AWS_V4",
"endpoint": cld_mgr.rgw_client.endpoint,
"endpoint_type": "S3_COMPATIBLE",
"identity": cld_mgr.rgw_client.access_key,
"name": conn_name,
"secret": cld_mgr.rgw_client.secret_key,
}
else:
raise UnsupportedPlatformError(f"Unsupported Platform: {platform}")
try:
for resp in TimeoutSampler(
30,
3,
self.send_rpc_query,
"account_api",
"add_external_connection",
params,
):
if "error" not in resp.text:
logger.info(f"Connection {conn_name} created successfully")
return True
else:
logger.info(
f"{platform} IAM {conn_name} did not yet propagate: {resp.text}"
)
except TimeoutExpiredError:
logger.error(f"Could not create connection {conn_name}")
assert False
def create_custom_machineset(
role="app",
instance_type=None,
labels=None,
taints=None,
zone="a",
):
"""
Function to create custom machineset works only for AWS
i.e. Using this user can create nodes with different instance type and role.
https://docs.openshift.com/container-platform/4.1/machine_management/creating-machineset.html
Args:
role (str): Role type to be added for node eg: it will be app,worker
instance_type (str): Type of instance
labels (list): List of Labels (key, val) to be added to the node
taints (list): List of taints to be applied
zone (str): Machineset zone for node creation.
Returns:
machineset (str): Created machineset name
Raise:
ResourceNotFoundError: Incase machineset creation failed
UnsupportedPlatformError: Incase of wrong platform
"""
# check for aws and IPI platform
if config.ENV_DATA["platform"].lower() == "aws":
machinesets_obj = OCP(
kind=constants.MACHINESETS,
namespace=constants.OPENSHIFT_MACHINE_API_NAMESPACE,
)
m4_xlarge = "m4.xlarge"
aws_instance = instance_type if instance_type else m4_xlarge
for machine in machinesets_obj.get()["items"]:
# Get inputs from existing machineset config.
region = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("placement").get("region"))
aws_zone = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("placement").get(
"availabilityZone"))
cls_id = (machine.get("spec").get("selector").get(
"matchLabels").get("machine.openshift.io/cluster-api-cluster"))
ami_id = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("ami").get("id"))
if aws_zone == f"{region}{zone}":
machineset_yaml = templating.load_yaml(
constants.MACHINESET_YAML)
# Update machineset_yaml with required values.
machineset_yaml["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["metadata"][
"name"] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-role"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-type"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["ami"]["id"] = ami_id
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["iamInstanceProfile"][
"id"] = f"{cls_id}-worker-profile"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["instanceType"] = aws_instance
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["placement"]["availabilityZone"] = aws_zone
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["placement"]["region"] = region
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["securityGroups"][0]["filters"][0]["values"][
0] = f"{cls_id}-worker-sg"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["subnet"]["filters"][0]["values"][
0] = f"{cls_id}-private-{aws_zone}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["tags"][0][
"name"] = f"kubernetes.io/cluster/{cls_id}"
# Apply the labels
if labels:
for label in labels:
machineset_yaml["spec"]["template"]["spec"][
"metadata"]["labels"][label[0]] = label[1]
# Remove app label in case of infra nodes
if role == "infra":
machineset_yaml["spec"]["template"]["spec"][
"metadata"]["labels"].pop(constants.APP_LABEL,
None)
# Apply the Taints
# ex taint list looks like:
# [ {'effect': 'NoSchedule',
# 'key': 'node.ocs.openshift.io/storage',
# 'value': 'true',
# }, {'effect': 'Schedule', 'key': 'xyz', 'value': 'False'} ]
if taints:
machineset_yaml["spec"]["template"]["spec"].update(
{"taints": taints})
# Create new custom machineset
ms_obj = OCS(**machineset_yaml)
ms_obj.create()
if check_machineset_exists(f"{cls_id}-{role}-{aws_zone}"):
log.info(f"Machineset {cls_id}-{role}-{aws_zone} created")
return f"{cls_id}-{role}-{aws_zone}"
else:
raise ResourceNotFoundError(
"Machineset resource not found")
# check for azure and IPI platform
elif config.ENV_DATA["platform"] == "azure":
machinesets_obj = OCP(
kind=constants.MACHINESETS,
namespace=constants.OPENSHIFT_MACHINE_API_NAMESPACE,
)
vmsize = constants.AZURE_PRODUCTION_INSTANCE_TYPE
azure_instance = instance_type if instance_type else vmsize
for machine in machinesets_obj.get()["items"]:
# Get inputs from existing machineset config.
region = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("location"))
azure_zone = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("zone"))
cls_id = (machine.get("spec").get("selector").get(
"matchLabels").get("machine.openshift.io/cluster-api-cluster"))
if azure_zone == zone:
az_zone = f"{region}{zone}"
machineset_yaml = templating.load_yaml(
constants.MACHINESET_YAML_AZURE)
# Update machineset_yaml with required values.
machineset_yaml["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["metadata"][
"name"] = f"{cls_id}-{role}-{az_zone}"
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{az_zone}"
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-role"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-type"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{az_zone}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["image"][
"resourceID"] = f"/resourceGroups/{cls_id}-rg/providers/Microsoft.Compute/images/{cls_id}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["location"] = region
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["managedIdentity"] = f"{cls_id}-identity"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["resourceGroup"] = f"{cls_id}-rg"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["subnet"] = f"{cls_id}-worker-subnet"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["vmSize"] = azure_instance
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["vnet"] = f"{cls_id}-vnet"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["zone"] = zone
# Apply the labels
if labels:
for label in labels:
machineset_yaml["spec"]["template"]["spec"][
"metadata"]["labels"][label[0]] = label[1]
# Remove app label in case of infra nodes
if role == "infra":
machineset_yaml["spec"]["template"]["spec"][
"metadata"]["labels"].pop(constants.APP_LABEL,
None)
if taints:
machineset_yaml["spec"]["template"]["spec"].update(
{"taints": taints})
# Create new custom machineset
ms_obj = OCS(**machineset_yaml)
ms_obj.create()
if check_machineset_exists(f"{cls_id}-{role}-{az_zone}"):
log.info(f"Machineset {cls_id}-{role}-{az_zone} created")
return f"{cls_id}-{role}-{az_zone}"
else:
raise ResourceNotFoundError(
"Machineset resource not found")
# check for RHV and IPI platform
elif config.ENV_DATA["platform"] == "rhv":
machinesets_obj = OCP(
kind=constants.MACHINESETS,
namespace=constants.OPENSHIFT_MACHINE_API_NAMESPACE,
)
for machine in machinesets_obj.get()["items"]:
# Get inputs from existing machineset config.
cls_uuid = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("cluster_id"))
template_name = (machine.get("spec").get("template").get(
"spec").get("providerSpec").get("value").get("template_name"))
cls_id = (machine.get("spec").get("selector").get(
"matchLabels").get("machine.openshift.io/cluster-api-cluster"))
socket = (machine.get("spec").get("template").get("spec").get(
"providerSpec").get("value").get("cpu").get("sockets"))
machineset_yaml = templating.load_yaml(
constants.MACHINESET_YAML_RHV)
# Update machineset_yaml with required values.
machineset_yaml["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["metadata"]["name"] = f"{cls_id}-{role}-{zone}"
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{zone}"
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-role"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-type"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}-{zone}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["cluster_id"] = cls_uuid
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["template_name"] = template_name
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["cpu"]["sockets"] = socket
# Apply the labels
if labels:
for label in labels:
machineset_yaml["spec"]["template"]["spec"]["metadata"][
"labels"][label[0]] = label[1]
# Remove app label in case of infra nodes
if role == "infra":
machineset_yaml["spec"]["template"]["spec"]["metadata"][
"labels"].pop(constants.APP_LABEL, None)
if taints:
machineset_yaml["spec"]["template"]["spec"].update(
{"taints": taints})
# Create new custom machineset
ms_obj = OCS(**machineset_yaml)
ms_obj.create()
if check_machineset_exists(f"{cls_id}-{role}-{zone}"):
log.info(f"Machineset {cls_id}-{role}-{zone} created")
return f"{cls_id}-{role}-{zone}"
else:
raise ResourceNotFoundError("Machineset resource not found")
# check for vmware and IPI platform
elif config.ENV_DATA["platform"] == constants.VSPHERE_PLATFORM:
machinesets_obj = OCP(
kind=constants.MACHINESETS,
namespace=constants.OPENSHIFT_MACHINE_API_NAMESPACE,
)
for machine in machinesets_obj.get()["items"]:
# Get inputs from existing machineset config.
cls_id = machine.get("spec")["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-cluster"]
disk_size = machine.get(
"spec")["template"]["spec"]["providerSpec"]["value"]["diskGiB"]
memory = machine.get("spec")["template"]["spec"]["providerSpec"][
"value"]["memoryMiB"]
network_name = machine.get(
"spec")["template"]["spec"]["providerSpec"]["value"][
"network"]["devices"][0]["networkName"]
num_cpu = machine.get(
"spec")["template"]["spec"]["providerSpec"]["value"]["numCPUs"]
num_core = machine.get("spec")["template"]["spec"]["providerSpec"][
"value"]["numCoresPerSocket"]
vm_template = machine.get("spec")["template"]["spec"][
"providerSpec"]["value"]["template"]
datacenter = machine.get("spec")["template"]["spec"][
"providerSpec"]["value"]["workspace"]["datacenter"]
datastore = machine.get("spec")["template"]["spec"][
"providerSpec"]["value"]["workspace"]["datastore"]
ds_folder = machine.get("spec")["template"]["spec"][
"providerSpec"]["value"]["workspace"]["folder"]
ds_resourcepool = machine.get("spec")["template"]["spec"][
"providerSpec"]["value"]["workspace"]["resourcePool"]
ds_server = machine.get("spec")["template"]["spec"][
"providerSpec"]["value"]["workspace"]["server"]
machineset_yaml = templating.load_yaml(
constants.MACHINESET_YAML_VMWARE)
# Update machineset_yaml with required values.
machineset_yaml["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["metadata"]["name"] = f"{cls_id}-{role}"
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["selector"]["matchLabels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}"
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-cluster"] = cls_id
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-role"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machine-type"] = role
machineset_yaml["spec"]["template"]["metadata"]["labels"][
"machine.openshift.io/cluster-api-machineset"] = f"{cls_id}-{role}"
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["diskGiB"] = disk_size
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["memoryMiB"] = memory
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["network"]["devices"][0]["networkName"] = network_name
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["numCPUs"] = num_cpu
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["numCoresPerSocket"] = num_core
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["template"] = vm_template
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["workspace"]["datacenter"] = datacenter
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["workspace"]["datastore"] = datastore
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["workspace"]["folder"] = ds_folder
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["workspace"]["resourcepool"] = ds_resourcepool
machineset_yaml["spec"]["template"]["spec"]["providerSpec"][
"value"]["workspace"]["server"] = ds_server
# Apply the labels
if labels:
for label in labels:
machineset_yaml["spec"]["template"]["spec"]["metadata"][
"labels"][label[0]] = label[1]
# Remove app label in case of infra nodes
if role == "infra":
machineset_yaml["spec"]["template"]["spec"]["metadata"][
"labels"].pop(constants.APP_LABEL, None)
if taints:
machineset_yaml["spec"]["template"]["spec"].update(
{"taints": taints})
# Create new custom machineset
ms_obj = OCS(**machineset_yaml)
ms_obj.create()
if check_machineset_exists(f"{cls_id}-{role}"):
log.info(f"Machineset {cls_id}-{role} created")
return f"{cls_id}-{role}"
else:
raise ResourceNotFoundError("Machineset resource not found")
else:
raise UnsupportedPlatformError(
"Functionality not supported in this platform")
def create_custom_machineset(role='app',
instance_type='m4.xlarge',
label='app-scale',
zone='a'):
"""
Function to create custom machineset works only for AWS
i.e. Using this user can create nodes with different instance type and role.
https://docs.openshift.com/container-platform/4.1/machine_management/creating-machineset.html
Args:
role (str): Role type to be added for node eg: it will be app,worker
instance_type (str): Type of aws instance
label (str): Label to be added to the node
zone (str): Machineset zone for node creation.
Returns:
machineset (str): Created machineset name
Raise:
ResourceNotFoundError: Incase machineset creation failed
UnsupportedPlatformError: Incase of wrong platform
"""
# check for platform, since it's supported only for IPI
if config.ENV_DATA['deployment_type'] == 'ipi':
machinesets_obj = OCP(
kind=constants.MACHINESETS,
namespace=constants.OPENSHIFT_MACHINE_API_NAMESPACE)
for machine in machinesets_obj.get()['items']:
# Get inputs from existing machineset config.
region = machine.get('spec').get('template').get('spec').get(
'providerSpec').get('value').get('placement').get('region')
aws_zone = machine.get('spec').get('template').get('spec').get(
'providerSpec').get('value').get('placement').get(
'availabilityZone')
cls_id = machine.get('spec').get('selector').get(
'matchLabels').get('machine.openshift.io/cluster-api-cluster')
ami_id = machine.get('spec').get('template').get('spec').get(
'providerSpec').get('value').get('ami').get('id')
if aws_zone == f"{region}{zone}":
machineset_yaml = templating.load_yaml(
constants.MACHINESET_YAML)
# Update machineset_yaml with required values.
machineset_yaml['metadata']['labels'][
'machine.openshift.io/cluster-api-cluster'] = cls_id
machineset_yaml['metadata'][
'name'] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml['spec']['selector']['matchLabels'][
'machine.openshift.io/cluster-api-cluster'] = cls_id
machineset_yaml['spec']['selector']['matchLabels'][
'machine.openshift.io/cluster-api-machineset'] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml['spec']['template']['metadata']['labels'][
'machine.openshift.io/cluster-api-cluster'] = cls_id
machineset_yaml['spec']['template']['metadata']['labels'][
'machine.openshift.io/cluster-api-machine-role'] = role
machineset_yaml['spec']['template']['metadata']['labels'][
'machine.openshift.io/cluster-api-machine-type'] = role
machineset_yaml['spec']['template']['metadata']['labels'][
'machine.openshift.io/cluster-api-machineset'] = f"{cls_id}-{role}-{aws_zone}"
machineset_yaml['spec']['template']['spec']['metadata'][
'labels'][f"node-role.kubernetes.io/{role}"] = f"{label}"
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['ami']['id'] = ami_id
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['iamInstanceProfile'][
'id'] = f"{cls_id}-worker-profile"
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['instanceType'] = instance_type
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['placement']['availabilityZone'] = aws_zone
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['placement']['region'] = region
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['securityGroups'][0]['filters'][0]['values'][
0] = f"{cls_id}-worker-sg"
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['subnet']['filters'][0]['values'][
0] = f"{cls_id}-private-{aws_zone}"
machineset_yaml['spec']['template']['spec']['providerSpec'][
'value']['tags'][0][
'name'] = f"kubernetes.io/cluster/{cls_id}"
# Create new custom machineset
ms_obj = OCS(**machineset_yaml)
ms_obj.create()
if check_machineset_exists(f"{cls_id}-{role}-{aws_zone}"):
logging.info(
f"Machineset {cls_id}-{role}-{aws_zone} created")
return f"{cls_id}-{role}-{aws_zone}"
else:
raise ResourceNotFoundError(
f"Machineset resource not found")
else:
raise UnsupportedPlatformError("Functionality not supported in UPI")
def add_worker_node(instance_type=None):
global ms_name
ms_name = list()
worker_list = node.get_worker_nodes()
ocs_worker_list = machine.get_labeled_nodes(constants.OPERATOR_NODE_LABEL)
scale_worker = machine.get_labeled_nodes(constants.SCALE_LABEL)
if config.RUN.get("use_ocs_worker_for_scale"):
if not scale_worker:
helpers.label_worker_node(node_list=worker_list,
label_key="scale-label",
label_value="app-scale")
else:
if not scale_worker:
for node_item in ocs_worker_list:
worker_list.remove(node_item)
if worker_list:
helpers.label_worker_node(
node_list=worker_list,
label_key="scale-label",
label_value="app-scale",
)
scale_worker_list = machine.get_labeled_nodes(constants.SCALE_LABEL)
logging.info(f"Print existing scale worker {scale_worker_list}")
if (config.ENV_DATA["deployment_type"] == "ipi"
and config.ENV_DATA["platform"].lower() == "aws"):
log.info("Adding worker nodes on the current cluster")
labels = [("node-role.kubernetes.io/app", "app-scale")]
# Create machineset for app worker nodes on each zone
for obj in machine.get_machineset_objs():
if "app" in obj.name:
ms_name.append(obj.name)
if instance_type is not None:
instance_type = instance_type
else:
instance_type = "m5.4xlarge"
if not ms_name:
if len(machine.get_machineset_objs()) == 3:
for zone in ["a", "b", "c"]:
ms_name.append(
machine.create_custom_machineset(
instance_type=instance_type,
labels=labels,
zone=zone,
))
else:
ms_name.append(
machine.create_custom_machineset(
instance_type=instance_type,
labels=labels,
zone="a",
))
for ms in ms_name:
machine.wait_for_new_node_to_be_ready(ms)
worker_list = node.get_worker_nodes()
ocs_worker_list = machine.get_labeled_nodes(
constants.OPERATOR_NODE_LABEL)
scale_label_worker = machine.get_labeled_nodes(constants.SCALE_LABEL)
ocs_worker_list.extend(scale_label_worker)
final_list = list(dict.fromkeys(ocs_worker_list))
for node_item in final_list:
if node_item in worker_list:
worker_list.remove(node_item)
if worker_list:
helpers.label_worker_node(node_list=worker_list,
label_key="scale-label",
label_value="app-scale")
return True
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "vsphere"):
log.info("Running scale test on existing worker nodes.")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "baremetal"):
log.info("Running scale test on existing worker nodes.")
elif (config.ENV_DATA["deployment_type"] == "upi"
and config.ENV_DATA["platform"].lower() == "azure"):
raise UnsupportedPlatformError("Unsupported Platform")
def add_worker_node(instance_type=None):
global ms_name
ms_name = list()
worker_list = helpers.get_worker_nodes()
ocs_worker_list = machine.get_labeled_nodes(constants.OPERATOR_NODE_LABEL)
scale_worker = machine.get_labeled_nodes(constants.SCALE_LABEL)
if config.RUN.get('use_ocs_worker_for_scale'):
if not scale_worker:
helpers.label_worker_node(node_list=worker_list,
label_key='scale-label',
label_value='app-scale')
else:
if not scale_worker:
for node_item in ocs_worker_list:
worker_list.remove(node_item)
if worker_list:
helpers.label_worker_node(node_list=worker_list,
label_key='scale-label',
label_value='app-scale')
scale_worker_list = machine.get_labeled_nodes(constants.SCALE_LABEL)
logging.info(f"Print existing scale worker {scale_worker_list}")
if config.ENV_DATA['deployment_type'] == 'ipi' and config.ENV_DATA[
'platform'].lower() == 'aws':
log.info("Adding worker nodes on the current cluster")
# Create machineset for app worker nodes on each zone
for obj in machine.get_machineset_objs():
if 'app' in obj.name:
ms_name.append(obj.name)
if instance_type is not None:
instance_type = instance_type
else:
instance_type = 'm5.4xlarge'
if not ms_name:
if len(machine.get_machineset_objs()) == 3:
for zone in ['a', 'b', 'c']:
ms_name.append(
machine.create_custom_machineset(
instance_type=instance_type, zone=zone))
else:
ms_name.append(
machine.create_custom_machineset(
instance_type=instance_type, zone='a'))
for ms in ms_name:
machine.wait_for_new_node_to_be_ready(ms)
worker_list = helpers.get_worker_nodes()
ocs_worker_list = machine.get_labeled_nodes(
constants.OPERATOR_NODE_LABEL)
scale_label_worker = machine.get_labeled_nodes(constants.SCALE_LABEL)
ocs_worker_list.extend(scale_label_worker)
final_list = list(dict.fromkeys(ocs_worker_list))
for node_item in final_list:
if node_item in worker_list:
worker_list.remove(node_item)
if worker_list:
helpers.label_worker_node(node_list=worker_list,
label_key='scale-label',
label_value='app-scale')
return True
elif config.ENV_DATA['deployment_type'] == 'upi' and config.ENV_DATA[
'platform'].lower() == 'vsphere':
log.info('Running pgsql on existing worker nodes')
elif config.ENV_DATA['deployment_type'] == 'upi' and config.ENV_DATA[
'platform'].lower() == 'baremetal':
log.info('Running pgsql on existing worker nodes')
elif config.ENV_DATA['deployment_type'] == 'upi' and config.ENV_DATA[
'platform'].lower() == 'azure':
raise UnsupportedPlatformError("Unsupported Platform")