def clone_and_unlock_ocs_private_conf(self):
"""
Clone ocs_private_conf (flexy env and config) repo into
flexy_host_dir
"""
clone_repo(
self.flexy_private_conf_url,
self.flexy_host_private_conf_dir_path,
self.flexy_private_conf_branch,
)
# prepare flexy private repo keyfile (if it is base64 encoded)
keyfile = os.path.expanduser(constants.FLEXY_GIT_CRYPT_KEYFILE)
try:
with open(keyfile, "rb") as fd:
keyfile_content = base64.b64decode(fd.read())
logger.info(
"Private flexy repository git crypt keyfile is base64 encoded. "
f"Decoding it and saving to the same place ({keyfile})")
with open(keyfile, "wb") as fd:
fd.write(keyfile_content)
except binascii.Error:
logger.info(
f"Private flexy repository git crypt keyfile is already prepared ({keyfile})."
)
# git-crypt unlock /path/to/keyfile
cmd = f"git-crypt unlock {keyfile}"
exec_cmd(cmd, cwd=self.flexy_host_private_conf_dir_path)
logger.info("Unlocked the git repo")
def clone_openshift_installer():
"""
Clone the openshift installer repo
"""
# git clone repo from openshift installer
# installer ( https://github.com/openshift/installer ) master and
# other branches (greater than release-4.3) structure has been
# changed. Use appropriate branch when ocs-ci is ready
# with the changes.
# Note: Currently use release-4.3 branch for the ocp versions
# which is greater than 4.3
upi_repo_path = os.path.join(
constants.EXTERNAL_DIR,
'installer'
)
ocp_version = get_ocp_version()
if Version.coerce(ocp_version) >= Version.coerce('4.4'):
clone_repo(
constants.VSPHERE_INSTALLER_REPO, upi_repo_path,
constants.VSPHERE_INSTALLER_BRANCH
)
else:
clone_repo(
constants.VSPHERE_INSTALLER_REPO, upi_repo_path,
f'release-{ocp_version}'
)
def destroy_cluster(self, log_level="DEBUG"):
"""
Destroy OCP cluster specific to vSphere UPI
Args:
log_level (str): log level openshift-installer (default: DEBUG)
"""
previous_dir = os.getcwd()
terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR)
upi_repo_path = os.path.join(
constants.EXTERNAL_DIR, 'installer',
)
tfvars = os.path.join(
config.ENV_DATA.get('cluster_path'),
constants.TERRAFORM_DATA_DIR,
constants.TERRAFORM_VARS
)
clone_repo(
constants.VSPHERE_INSTALLER_REPO, upi_repo_path
)
if (
os.path.exists(f"{constants.VSPHERE_MAIN}.backup")
and os.path.exists(f"{constants.VSPHERE_MAIN}.json")
):
os.rename(f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup")
terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/"))
os.chdir(terraform_data_dir)
terraform.initialize(upgrade=True)
terraform.destroy(tfvars)
os.chdir(previous_dir)
def _deploy_prereqs(self):
"""
Perform prerequisites
"""
# Clone workload repo
clone_repo(self.workload_repo_url, self.target_clone_dir)
def destroy_scaleup_nodes(self, scale_up_terraform_data_dir, scale_up_terraform_var):
"""
Destroy the scale-up nodes
Args:
scale_up_terraform_data_dir (str): Path to scale-up terraform
data directory
scale_up_terraform_var (str): Path to scale-up
terraform.tfvars file
"""
clone_repo(
constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path
)
# modify scale-up repo
self.modify_scaleup_repo()
terraform_scale_up = Terraform(
os.path.join(
self.upi_scale_up_repo_path,
"v4-testing-misc/v4-scaleup/vsphere/"
)
)
os.chdir(scale_up_terraform_data_dir)
terraform_scale_up.initialize(upgrade=True)
terraform_scale_up.destroy(scale_up_terraform_var)
def destroy_scaleup_nodes(self, scale_up_terraform_data_dir,
scale_up_terraform_var):
"""
Destroy the scale-up nodes
Args:
scale_up_terraform_data_dir (str): Path to scale-up terraform
data directory
scale_up_terraform_var (str): Path to scale-up
terraform.tfvars file
"""
clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path)
# git clone repo from cluster-launcher
clone_repo(constants.VSPHERE_CLUSTER_LAUNCHER,
self.cluster_launcer_repo_path)
# modify scale-up repo
helpers = VSPHEREHELPERS()
helpers.modify_scaleup_repo()
vsphere_dir = constants.SCALEUP_VSPHERE_DIR
if Version.coerce(self.ocp_version) >= Version.coerce("4.5"):
vsphere_dir = os.path.join(
constants.CLUSTER_LAUNCHER_VSPHERE_DIR,
f"aos-{get_ocp_version('_')}",
"vsphere",
)
terraform_scale_up = Terraform(vsphere_dir)
os.chdir(scale_up_terraform_data_dir)
terraform_scale_up.initialize(upgrade=True)
terraform_scale_up.destroy(scale_up_terraform_var)
def get_new_device_paths(device_sets_required, osd_size_capacity_requested):
"""
Get new device paths to add capacity over Baremetal cluster
Args:
device_sets_required (int) : Count of device sets to be added
osd_size_capacity_requested (int) : Requested OSD size capacity
Returns:
cur_device_list (list) : List containing added device paths
"""
ocp_obj = OCP()
workers = get_typed_nodes(node_type="worker")
worker_names = [worker.name for worker in workers]
output = ocp_obj.exec_oc_cmd("get localvolume local-block -n local-storage -o yaml")
cur_device_list = output["spec"]["storageClassDevices"][0]["devicePaths"]
path = os.path.join(constants.EXTERNAL_DIR, "device-by-id-ocp")
utils.clone_repo(constants.OCP_QE_DEVICEPATH_REPO, path)
os.chdir(path)
utils.run_cmd("ansible-playbook devices_by_id.yml")
with open("local-storage-block.yaml", "r") as cloned_file:
with open("local-block.yaml", "w") as our_file:
device_from_worker1 = device_sets_required
device_from_worker2 = device_sets_required
device_from_worker3 = device_sets_required
cur_line = cloned_file.readline()
while "devicePaths:" not in cur_line:
our_file.write(cur_line)
cur_line = cloned_file.readline()
our_file.write(cur_line)
cur_line = cloned_file.readline()
# Add required number of device path from each node
while cur_line:
if str(osd_size_capacity_requested) in cur_line:
if device_from_worker1 and (str(worker_names[0]) in cur_line):
if not any(s in cur_line for s in cur_device_list):
our_file.write(cur_line)
device_from_worker1 = device_from_worker1 - 1
if device_from_worker2 and (str(worker_names[1]) in cur_line):
if not any(s in cur_line for s in cur_device_list):
our_file.write(cur_line)
device_from_worker2 = device_from_worker2 - 1
if device_from_worker3 and (str(worker_names[2]) in cur_line):
if not any(s in cur_line for s in cur_device_list):
our_file.write(cur_line)
device_from_worker3 = device_from_worker3 - 1
cur_line = cloned_file.readline()
local_block_yaml = open("local-block.yaml")
lvcr = yaml.load(local_block_yaml, Loader=yaml.FullLoader)
new_dev_paths = lvcr["spec"]["storageClassDevices"][0]["devicePaths"]
log.info(f"Newly added devices are: {new_dev_paths}")
assert len(new_dev_paths) == (len(worker_names) * device_sets_required), (
f"Current devices available = {len(new_dev_paths)}"
)
os.chdir(constants.TOP_DIR)
shutil.rmtree(path)
cur_device_list.extend(new_dev_paths)
return cur_device_list
def get_new_device_paths(device_sets_required, osd_size_capacity_requested):
"""
Get new device paths to add capacity over Baremetal cluster
Args:
device_sets_required (int) : Count of device sets to be added
osd_size_capacity_requested (int) : Requested OSD size capacity
Returns:
list : List containing added device paths
"""
ocp_obj = OCP(kind="localvolume",
namespace=config.ENV_DATA["local_storage_namespace"])
workers = get_nodes(node_type="worker")
worker_names = [worker.name for worker in workers]
config.ENV_DATA["worker_replicas"] = len(worker_names)
output = ocp_obj.get(resource_name="local-block")
# Fetch device paths present in the current LVCR
cur_device_list = output["spec"]["storageClassDevices"][0]["devicePaths"]
# Clone repo and run playbook to fetch all device paths from each node
path = os.path.join(constants.EXTERNAL_DIR, "device-by-id-ocp")
clone_repo(constants.OCP_QE_DEVICEPATH_REPO, path)
os.chdir(path)
run_cmd("ansible-playbook devices_by_id.yml")
# Filter unused/unallocated device paths
with open("local-storage-block.yaml", "r") as cloned_file:
with open("local-block.yaml", "w") as our_file:
device_from_worker = [1] * config.ENV_DATA["worker_replicas"]
cur_line = cloned_file.readline()
while "devicePaths:" not in cur_line:
our_file.write(cur_line)
cur_line = cloned_file.readline()
our_file.write(cur_line)
cur_line = cloned_file.readline()
# Add required number of device path from each worker node
while cur_line:
if str(osd_size_capacity_requested) in cur_line:
for i in range(len(worker_names)):
if device_from_worker[i] and (str(worker_names[i])
in cur_line):
if not any(s in cur_line for s in cur_device_list):
our_file.write(cur_line)
device_from_worker[
i] = device_from_worker[i] - 1
cur_line = cloned_file.readline()
local_block_yaml = open("local-block.yaml")
lvcr = yaml.load(local_block_yaml, Loader=yaml.FullLoader)
new_dev_paths = lvcr["spec"]["storageClassDevices"][0]["devicePaths"]
logger.info(f"Newly added devices are: {new_dev_paths}")
if new_dev_paths:
assert len(new_dev_paths) == (
len(worker_names) * device_sets_required
), f"Current devices available = {len(new_dev_paths)}"
os.chdir(constants.TOP_DIR)
shutil.rmtree(path)
# Return list of old device paths and newly added device paths
cur_device_list.extend(new_dev_paths)
return cur_device_list
def deploy_prereq(self):
"""
Overriding deploy_prereq from parent. Perform all necessary
prerequisites for AWSUPI here.
"""
super(AWSUPI.OCPDeployment, self).deploy_prereq()
# setup necessary env variables
upi_env_vars = {
'INSTANCE_NAME_PREFIX': config.ENV_DATA['cluster_name'],
'AWS_REGION': config.ENV_DATA['region'],
'rhcos_ami': config.ENV_DATA.get('rhcos_ami'),
'route53_domain_name': config.ENV_DATA['base_domain'],
'vm_type_masters': config.ENV_DATA['master_instance_type'],
'vm_type_workers': config.ENV_DATA['worker_instance_type'],
'num_workers': str(config.ENV_DATA['worker_replicas']),
'AVAILABILITY_ZONE_COUNT': str(config.ENV_DATA.get(
'availability_zone_count', ''
)),
'BASE_DOMAIN': config.ENV_DATA['base_domain'],
'remove_bootstrap': 'yes'
}
if config.DEPLOYMENT['preserve_bootstrap_node']:
logger.info("Setting ENV VAR to preserve bootstrap node")
upi_env_vars['remove_bootstrap'] = 'No'
for key, value in upi_env_vars.items():
if value:
os.environ[key] = value
# ensure environment variables have been set correctly
for key, value in upi_env_vars.items():
if value:
assert os.getenv(key) == value
# git clone repo from openshift-qe repo
clone_repo(
constants.OCP_QE_MISC_REPO, self.upi_repo_path
)
# Sym link install-dir to cluster_path
install_dir = os.path.join(self.upi_script_path, "install-dir")
absolute_cluster_path = os.path.abspath(self.cluster_path)
logger.info(
"Sym linking %s to %s", install_dir, absolute_cluster_path
)
os.symlink(absolute_cluster_path, install_dir)
# NOT A CLEAN APPROACH: copy openshift-install and oc binary to
# script path because upi script expects it to be present in
# script dir
bindir = os.path.abspath(os.path.expanduser(config.RUN['bin_dir']))
shutil.copy2(
os.path.join(bindir, 'openshift-install'),
self.upi_script_path,
)
shutil.copy2(
os.path.join(bindir, 'oc'), self.upi_script_path
)
def destroy_cluster(self, log_level="DEBUG"):
"""
Destroy OCP cluster specific to vSphere UPI
Args:
log_level (str): log level openshift-installer (default: DEBUG)
"""
previous_dir = os.getcwd()
# delete the extra disks
self.delete_disks()
# check whether cluster has scale-up nodes
scale_up_terraform_data_dir = os.path.join(
self.cluster_path,
constants.TERRAFORM_DATA_DIR,
constants.SCALEUP_TERRAFORM_DATA_DIR
)
scale_up_terraform_var = os.path.join(
scale_up_terraform_data_dir,
"scale_up_terraform.tfvars"
)
if os.path.exists(scale_up_terraform_var):
os.chdir(scale_up_terraform_data_dir)
self.destroy_scaleup_nodes(
scale_up_terraform_data_dir,
scale_up_terraform_var
)
os.chdir(previous_dir)
terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR)
upi_repo_path = os.path.join(
constants.EXTERNAL_DIR, 'installer',
)
tfvars = os.path.join(
config.ENV_DATA.get('cluster_path'),
constants.TERRAFORM_DATA_DIR,
constants.TERRAFORM_VARS
)
clone_repo(
constants.VSPHERE_INSTALLER_REPO, upi_repo_path,
f'release-{get_ocp_version()}'
)
if (
os.path.exists(f"{constants.VSPHERE_MAIN}.backup")
and os.path.exists(f"{constants.VSPHERE_MAIN}.json")
):
os.rename(f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup")
terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/"))
os.chdir(terraform_data_dir)
terraform.initialize(upgrade=True)
terraform.destroy(tfvars)
os.chdir(previous_dir)
# post destroy checks
self.post_destroy_checks()
def factory(destination_dir, git_url):
"""
Args:
destination_dir (str): Destination directory for cloned project.
git_url (str): Project's URL.
"""
bdi_dir.append(destination_dir)
log.info(f"Cloning chart from github into {destination_dir}")
clone_repo(url=git_url, location=destination_dir)
def add_nodes(self):
"""
Add new nodes to the cluster
"""
# create separate directory for scale-up terraform data
scaleup_terraform_data_dir = os.path.join(
self.cluster_path, constants.TERRAFORM_DATA_DIR,
constants.SCALEUP_TERRAFORM_DATA_DIR)
create_directory_path(scaleup_terraform_data_dir)
logger.info(
f"scale-up terraform data directory: {scaleup_terraform_data_dir}")
# git clone repo from openshift-misc
clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path)
# modify scale-up repo
self.modify_scaleup_repo()
config.ENV_DATA['vsphere_resource_pool'] = config.ENV_DATA.get(
"cluster_name")
# sync guest time with host
if config.ENV_DATA.get('sync_time_with_host'):
sync_time_with_host(constants.SCALEUP_VSPHERE_MACHINE_CONF, True)
# get the RHCOS worker list
self.rhcos_ips = get_node_ips()
logger.info(f"RHCOS IP's: {json.dumps(self.rhcos_ips)}")
# generate terraform variable for scaling nodes
self.generate_terraform_vars_for_scaleup()
# Add nodes using terraform
scaleup_terraform = Terraform(constants.SCALEUP_VSPHERE_DIR)
previous_dir = os.getcwd()
os.chdir(scaleup_terraform_data_dir)
scaleup_terraform.initialize()
scaleup_terraform.apply(self.scale_up_terraform_var)
scaleup_terraform_tfstate = os.path.join(scaleup_terraform_data_dir,
"terraform.tfstate")
out = scaleup_terraform.output(scaleup_terraform_tfstate,
"rhel_worker")
rhel_worker_nodes = json.loads(out)['value']
logger.info(f"RHEL worker nodes: {rhel_worker_nodes}")
os.chdir(previous_dir)
# Install OCP on rhel nodes
rhel_install = OCPINSTALLRHEL(rhel_worker_nodes)
rhel_install.prepare_rhel_nodes()
rhel_install.execute_ansible_playbook()
# Giving some time to settle down the new nodes
time.sleep(self.wait_time)
# wait for nodes to be in READY state
wait_for_nodes_status(timeout=300)
def clone_and_unlock_ocs_private_conf(self):
"""
Clone ocs_private_conf (flexy env and config) repo into
flexy_host_dir
"""
clone_repo(self.flexy_private_conf_url,
self.flexy_host_private_conf_dir_path,
self.flexy_private_conf_branch)
# git-crypt unlock /path/to/keyfile
cmd = (f'git-crypt unlock '
f'{os.path.expanduser(constants.FLEXY_GIT_CRYPT_KEYFILE)}')
run_cmd(cmd, cwd=self.flexy_host_private_conf_dir_path)
logger.info("Unlocked the git repo")
def deploy_prereq(self):
"""
Overriding deploy_prereq from parent. Perform all necessary
prerequisites for AWSUPI here.
"""
super(AWSUPI.OCPDeployment, self).deploy_prereq()
# setup necessary env variables
upi_env_vars = {
'INSTANCE_NAME_PREFIX': config.ENV_DATA['cluster_name'],
'AWS_REGION': config.ENV_DATA['region'],
'rhcos_ami': config.ENV_DATA['rhcos_ami'],
'route53_domain_name': config.ENV_DATA['base_domain'],
'vm_type_masters': config.ENV_DATA['master_instance_type'],
'vm_type_workers': config.ENV_DATA['worker_instance_type'],
'num_workers': str(config.ENV_DATA['worker_replicas'])
}
for key, value in upi_env_vars.items():
os.environ[key] = value
# ensure environment variables have been set correctly
for key, value in upi_env_vars.items():
assert os.getenv(key) == value, f"{os.getenv(key)} != {value}"
# git clone repo from openshift-qe repo
clone_repo(
constants.OCP_QE_MISC_REPO, self.upi_repo_path
)
# Sym link install-dir to cluster_path
install_dir = os.path.join(self.upi_script_path, "install-dir")
absolute_cluster_path = os.path.abspath(self.cluster_path)
logger.info(
"Sym linking %s to %s", install_dir, absolute_cluster_path
)
os.symlink(absolute_cluster_path, install_dir)
# NOT A CLEAN APPROACH: copy openshift-install and oc binary to
# script path because upi script expects it to be present in
# script dir
bindir = os.path.abspath(os.path.expanduser(config.RUN['bin_dir']))
shutil.copy2(
os.path.join(bindir, 'openshift-install'),
self.upi_script_path,
)
shutil.copy2(
os.path.join(bindir, 'oc'), self.upi_script_path
)
def test_ocs_monkey():
ocs_monkety_dir = "/tmp/ocs-monkey"
# ocs-monkey run time in seconds
run_time = 3200
clone_repo(constants.OCS_MONKEY_REPOSITORY, ocs_monkety_dir)
run_cmd(
f"pip install -r {os.path.join(ocs_monkety_dir, 'requirements.txt')}")
workload_run_cmd = f"python workload_runner.py -t {run_time}"
start_time = time.time()
log.info("Starting ocs-monkey")
popen_obj = subprocess.Popen(
shlex.split(workload_run_cmd),
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
encoding="utf-8",
cwd=ocs_monkety_dir,
)
while True:
output = popen_obj.stdout.readline()
# Check whether the process is completed
ret = popen_obj.poll()
if len(output) == 0 and ret is not None:
log.info("ocs-monkey run completed.")
assert ret == 0, (f"ocs-monkey exited with return value {ret}. "
f"Check logs for details.")
break
# Stream the output in console
if output:
log.info(output.strip())
# Terminate the process if it is not completed within the specified
# time. Give grace period of 600 seconds considering the time
# taken for setup
if time.time() - start_time > run_time + 600:
log.error(
f"ocs-monkey did not complete with in the specified run time"
f" {run_time} seconds. Killing the process now.")
popen_obj.terminate()
raise TimeoutError(
f"ocs-monkey did not complete with in the specified run time "
f"{run_time} seconds. Killed the process after providing "
f"grace period of 600 seconds.")
def fetch_all_device_paths():
"""
Return all device paths inside worker nodes
Returns:
list : List containing all device paths
"""
path = os.path.join(constants.EXTERNAL_DIR, "device-by-id-ocp")
clone_repo(constants.OCP_QE_DEVICEPATH_REPO, path)
os.chdir(path)
logger.info("Running script to fetch device paths...")
run_cmd("ansible-playbook devices_by_id.yml")
with open("local-storage-block.yaml") as local_storage_block:
local_block = yaml.load(local_storage_block, Loader=yaml.FullLoader)
dev_paths = local_block["spec"]["storageClassDevices"][0]["devicePaths"]
logger.info(f"All devices are {dev_paths}")
os.chdir(constants.TOP_DIR)
shutil.rmtree(path)
return dev_paths
def test_ocs_monkey():
ocs_monkety_dir = "/tmp/ocs-monkey"
# ocs-monkey run time in seconds
run_time = 3600
clone_repo(constants.OCS_MONKEY_REPOSITORY, ocs_monkety_dir)
run_cmd(
f"pip install -r {os.path.join(ocs_monkety_dir, 'requirements.txt')}")
workload_run_cmd = f"python workload_runner.py -t {run_time}"
chaos_runner_cmd = "python chaos_runner.py"
start_time = time.time()
log.info("Starting workload runner")
popen_workload = subprocess.Popen(
shlex.split(workload_run_cmd),
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
encoding="utf-8",
cwd=ocs_monkety_dir,
)
log.info("Starting chaos runner")
popen_chaos = subprocess.Popen(
shlex.split(chaos_runner_cmd),
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
encoding="utf-8",
cwd=ocs_monkety_dir,
)
while True:
output_workload = popen_workload.stdout.readline()
# Get the status of workload runner process
ret_workload = popen_workload.poll()
# Stream the workload runner output in console
if output_workload:
log.info(output_workload.strip())
if ret_workload is not None:
log.info("Workload runner completed.")
log.debug(popen_workload.stdout.read())
# Terminate chaos_runner if workload_runner is completed
log.info("Terminating chaos runner")
popen_chaos.terminate()
# Check return value of workload runner process
assert ret_workload == 0, (
f"Workload runner exited with return value {ret_workload}. "
f"Check logs for details.")
log.info("ocs-monkey run completed")
break
output_chaos = popen_chaos.stdout.readline()
# Get the status of chaos runner process
ret_chaos = popen_chaos.poll()
# Stream the chaos runner output in console
if output_chaos:
log.info(output_chaos.strip())
if ret_chaos is not None:
log.info("Chaos runner completed.")
log.debug(popen_chaos.stdout.read())
# Terminate workload_runner if chaos_runner is completed
log.info("Terminating workload runner")
popen_workload.terminate()
assert ret_chaos == 0, (
f"Chaos runner exited with return value {ret_chaos}. "
f"Check logs for details.")
log.info("ocs-monkey run completed")
break
# Terminate the process if it is not completed within the specified
# time. Give grace period of 900 seconds considering the time
# taken for setup
if time.time() - start_time > run_time + 900:
log.error(
f"ocs-monkey did not complete with in the specified run time"
f" {run_time} seconds. Killing the process now.")
popen_workload.terminate()
popen_chaos.terminate()
raise TimeoutError(
f"ocs-monkey did not complete with in the specified run time "
f"{run_time} seconds. Killed the process after providing "
f"grace period of 900 seconds.")
assert ceph_health_check(tries=40,
delay=30), "Ceph cluster health is not OK"
def deploy_prereq(self):
"""
Pre-Requisites for vSphere UPI Deployment
"""
super(VSPHEREUPI.OCPDeployment, self).deploy_prereq()
# create ignitions
self.create_ignitions()
self.kubeconfig = os.path.join(
self.cluster_path, config.RUN.get('kubeconfig_location'))
# git clone repo from openshift installer
clone_repo(constants.VSPHERE_INSTALLER_REPO, self.upi_repo_path)
# upload bootstrap ignition to public access server
bootstrap_path = os.path.join(config.ENV_DATA.get('cluster_path'),
constants.BOOTSTRAP_IGN)
remote_path = os.path.join(
config.ENV_DATA.get('path_to_upload'),
f"{config.RUN.get('run_id')}_{constants.BOOTSTRAP_IGN}")
upload_file(config.ENV_DATA.get('httpd_server'), bootstrap_path,
remote_path, config.ENV_DATA.get('httpd_server_user'),
config.ENV_DATA.get('httpd_server_password'))
# generate bootstrap ignition url
path_to_bootstrap_on_remote = remote_path.replace(
"/var/www/html/", "")
bootstrap_ignition_url = (
f"http://{config.ENV_DATA.get('httpd_server')}/"
f"{path_to_bootstrap_on_remote}")
logger.info(f"bootstrap_ignition_url: {bootstrap_ignition_url}")
config.ENV_DATA['bootstrap_ignition_url'] = bootstrap_ignition_url
# load master and worker ignitions to variables
master_ignition_path = os.path.join(
config.ENV_DATA.get('cluster_path'), constants.MASTER_IGN)
master_ignition = read_file_as_str(f"{master_ignition_path}")
config.ENV_DATA['control_plane_ignition'] = master_ignition
worker_ignition_path = os.path.join(
config.ENV_DATA.get('cluster_path'), constants.WORKER_IGN)
worker_ignition = read_file_as_str(f"{worker_ignition_path}")
config.ENV_DATA['compute_ignition'] = worker_ignition
cluster_domain = (f"{config.ENV_DATA.get('cluster_name')}."
f"{config.ENV_DATA.get('base_domain')}")
config.ENV_DATA['cluster_domain'] = cluster_domain
# generate terraform variables from template
logger.info("Generating terraform variables")
_templating = Templating()
terraform_var_template = "terraform.tfvars.j2"
terraform_var_template_path = os.path.join("ocp-deployment",
terraform_var_template)
terraform_config_str = _templating.render_template(
terraform_var_template_path, config.ENV_DATA)
terraform_var_yaml = os.path.join(self.cluster_path,
constants.TERRAFORM_DATA_DIR,
"terraform.tfvars.yaml")
with open(terraform_var_yaml, "w") as f:
f.write(terraform_config_str)
self.terraform_var = convert_yaml2tfvars(terraform_var_yaml)
# update gateway and DNS
if config.ENV_DATA.get('gateway'):
replace_content_in_file(constants.INSTALLER_IGNITION,
'${cidrhost(var.machine_cidr,1)}',
f"{config.ENV_DATA.get('gateway')}")
if config.ENV_DATA.get('dns'):
replace_content_in_file(constants.INSTALLER_IGNITION,
constants.INSTALLER_DEFAULT_DNS,
f"{config.ENV_DATA.get('dns')}")
# update the zone in route
if config.ENV_DATA.get('region'):
def_zone = 'provider "aws" { region = "%s" } \n' % config.ENV_DATA.get(
'region')
replace_content_in_file(constants.INSTALLER_ROUTE53, "xyz",
def_zone)
# increase memory
if config.ENV_DATA.get('memory'):
replace_content_in_file(constants.INSTALLER_MACHINE_CONF,
'${var.memory}',
config.ENV_DATA.get('memory'))
# increase CPUs
worker_num_cpus = config.ENV_DATA.get('worker_num_cpus')
master_num_cpus = config.ENV_DATA.get('master_num_cpus')
if worker_num_cpus or master_num_cpus:
with open(constants.VSPHERE_MAIN, 'r') as fd:
obj = hcl.load(fd)
if worker_num_cpus:
obj['module']['compute']['num_cpu'] = worker_num_cpus
if master_num_cpus:
obj['module']['control_plane'][
'num_cpu'] = master_num_cpus
# Dump data to json file since hcl module
# doesn't support dumping of data in HCL format
dump_data_to_json(obj, f"{constants.VSPHERE_MAIN}.json")
os.rename(constants.VSPHERE_MAIN,
f"{constants.VSPHERE_MAIN}.backup")
def svt_project_clone():
"""
This function clones the SVT project.
"""
clone_repo("https://github.com/openshift/svt.git", "/tmp/svt")
def deploy_prereq(self):
"""
Overriding deploy_prereq from parent. Perform all necessary
prerequisites for AWSUPI here.
"""
super(AWSUPI.OCPDeployment, self).deploy_prereq()
# setup necessary env variables
upi_env_vars = {
"INSTANCE_NAME_PREFIX":
config.ENV_DATA["cluster_name"],
"CLUSTER_NAME":
config.ENV_DATA["cluster_name"],
"AWS_REGION":
config.ENV_DATA["region"],
"rhcos_ami":
config.ENV_DATA.get("rhcos_ami"),
"route53_domain_name":
config.ENV_DATA["base_domain"],
"vm_type_masters":
config.ENV_DATA["master_instance_type"],
"vm_type_workers":
config.ENV_DATA["worker_instance_type"],
"num_workers":
str(config.ENV_DATA["worker_replicas"]),
"AVAILABILITY_ZONE_COUNT":
str(config.ENV_DATA.get("availability_zone_count", "")),
"BASE_DOMAIN":
config.ENV_DATA["base_domain"],
"remove_bootstrap":
"yes",
"IAAS_PLATFORM":
"aws",
"HOSTS_SCRIPT_DIR":
self.upi_script_path,
"OCP_INSTALL_DIR":
os.path.join(self.upi_script_path, "install-dir"),
"DISABLE_MASTER_MACHINESET":
"yes",
}
if config.DEPLOYMENT["preserve_bootstrap_node"]:
logger.info("Setting ENV VAR to preserve bootstrap node")
upi_env_vars["remove_bootstrap"] = "No"
logger.info(f"UPI ENV VARS = {upi_env_vars}")
for key, value in upi_env_vars.items():
if value:
os.environ[key] = value
# ensure environment variables have been set correctly
for key, value in upi_env_vars.items():
if value:
assert os.getenv(key) == value
# git clone repo from openshift-qe repo
clone_repo(constants.OCP_QE_MISC_REPO, self.upi_repo_path)
# Sym link install-dir to cluster_path
install_dir = os.path.join(self.upi_script_path, "install-dir")
absolute_cluster_path = os.path.abspath(self.cluster_path)
logger.info("Sym linking %s to %s", install_dir,
absolute_cluster_path)
os.symlink(absolute_cluster_path, install_dir)
# NOT A CLEAN APPROACH: copy openshift-install and oc binary to
# script path because upi script expects it to be present in
# script dir
bindir = os.path.abspath(os.path.expanduser(config.RUN["bin_dir"]))
shutil.copy2(
os.path.join(bindir, "openshift-install"),
self.upi_script_path,
)
shutil.copy2(os.path.join(bindir, "oc"), self.upi_script_path)
# and another UGLY WORKAROUND: copy openshift-install also to the
# absolute_cluster_path (for more details, see
# https://github.com/red-hat-storage/ocs-ci/pull/4650)
shutil.copy2(
os.path.join(bindir, "openshift-install"),
os.path.abspath(os.path.join(self.cluster_path, "..")),
)
def add_nodes(self):
"""
Add new nodes to the cluster
"""
# create separate directory for scale-up terraform data
scaleup_terraform_data_dir = os.path.join(
self.cluster_path,
constants.TERRAFORM_DATA_DIR,
constants.SCALEUP_TERRAFORM_DATA_DIR,
)
create_directory_path(scaleup_terraform_data_dir)
logger.info(
f"scale-up terraform data directory: {scaleup_terraform_data_dir}")
# git clone repo from openshift-misc
clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path)
# git clone repo from cluster-launcher
clone_repo(constants.VSPHERE_CLUSTER_LAUNCHER,
self.cluster_launcer_repo_path)
helpers = VSPHEREHELPERS()
helpers.modify_scaleup_repo()
config.ENV_DATA["vsphere_resource_pool"] = config.ENV_DATA.get(
"cluster_name")
# sync guest time with host
sync_time_with_host_file = constants.SCALEUP_VSPHERE_MACHINE_CONF
if config.ENV_DATA["folder_structure"]:
sync_time_with_host_file = os.path.join(
constants.CLUSTER_LAUNCHER_VSPHERE_DIR,
f"aos-{get_ocp_version(seperator='_')}",
constants.CLUSTER_LAUNCHER_MACHINE_CONF,
)
if config.ENV_DATA.get("sync_time_with_host"):
sync_time_with_host(sync_time_with_host_file, True)
# get the RHCOS worker list
rhcos_ips = get_node_ips()
logger.info(f"RHCOS IP's: {json.dumps(rhcos_ips)}")
# generate terraform variable for scaling nodes
self.scale_up_terraform_var = helpers.generate_terraform_vars_for_scaleup(
rhcos_ips)
# choose the vsphere_dir based on OCP version
# generate cluster_info and config yaml files
# for OCP version greater than 4.4
vsphere_dir = constants.SCALEUP_VSPHERE_DIR
rhel_module = "rhel-worker"
if Version.coerce(self.ocp_version) >= Version.coerce("4.5"):
vsphere_dir = os.path.join(
constants.CLUSTER_LAUNCHER_VSPHERE_DIR,
f"aos-{get_ocp_version('_')}",
"vsphere",
)
helpers.generate_cluster_info()
helpers.generate_config_yaml()
rhel_module = "RHEL_WORKER_LIST"
# Add nodes using terraform
scaleup_terraform = Terraform(vsphere_dir)
previous_dir = os.getcwd()
os.chdir(scaleup_terraform_data_dir)
scaleup_terraform.initialize()
scaleup_terraform.apply(self.scale_up_terraform_var)
scaleup_terraform_tfstate = os.path.join(scaleup_terraform_data_dir,
"terraform.tfstate")
out = scaleup_terraform.output(scaleup_terraform_tfstate, rhel_module)
if config.ENV_DATA["folder_structure"]:
rhel_worker_nodes = out.strip().replace('"', "").split(",")
else:
rhel_worker_nodes = json.loads(out)["value"]
logger.info(f"RHEL worker nodes: {rhel_worker_nodes}")
os.chdir(previous_dir)
# Install OCP on rhel nodes
rhel_install = OCPINSTALLRHEL(rhel_worker_nodes)
rhel_install.prepare_rhel_nodes()
rhel_install.execute_ansible_playbook()
# Giving some time to settle down the new nodes
time.sleep(self.wait_time)
# wait for nodes to be in READY state
wait_for_nodes_status(timeout=300)
def destroy_cluster(self, log_level="DEBUG"):
"""
Destroy OCP cluster specific to vSphere UPI
Args:
log_level (str): log level openshift-installer (default: DEBUG)
"""
previous_dir = os.getcwd()
# Download terraform binary based on terraform version
# in terraform.log
terraform_log_path = os.path.join(config.ENV_DATA.get("cluster_path"),
config.ENV_DATA.get("TF_LOG_FILE"))
# check for terraform.log, this check is for partially
# deployed clusters
try:
with open(terraform_log_path, "r") as fd:
logger.debug(
f"Reading terraform version from {terraform_log_path}")
version_line = fd.readline()
terraform_version = version_line.split()[-1]
except FileNotFoundError:
logger.debug(f"{terraform_log_path} file not found")
terraform_version = config.DEPLOYMENT["terraform_version"]
terraform_installer = get_terraform(version=terraform_version)
config.ENV_DATA["terraform_installer"] = terraform_installer
# getting OCP version here since we run destroy job as
# separate job in jenkins
ocp_version = get_ocp_version()
self.folder_structure = False
if Version.coerce(ocp_version) >= Version.coerce("4.5"):
set_aws_region()
self.folder_structure = True
config.ENV_DATA["folder_structure"] = self.folder_structure
# delete the extra disks
self.delete_disks()
# check whether cluster has scale-up nodes
scale_up_terraform_data_dir = os.path.join(
self.cluster_path,
constants.TERRAFORM_DATA_DIR,
constants.SCALEUP_TERRAFORM_DATA_DIR,
)
scale_up_terraform_var = os.path.join(scale_up_terraform_data_dir,
"scale_up_terraform.tfvars")
if os.path.exists(scale_up_terraform_var):
os.chdir(scale_up_terraform_data_dir)
self.destroy_scaleup_nodes(scale_up_terraform_data_dir,
scale_up_terraform_var)
os.chdir(previous_dir)
terraform_data_dir = os.path.join(self.cluster_path,
constants.TERRAFORM_DATA_DIR)
upi_repo_path = os.path.join(
constants.EXTERNAL_DIR,
"installer",
)
tfvars = os.path.join(
config.ENV_DATA.get("cluster_path"),
constants.TERRAFORM_DATA_DIR,
constants.TERRAFORM_VARS,
)
clone_openshift_installer()
if os.path.exists(
f"{constants.VSPHERE_MAIN}.backup") and os.path.exists(
f"{constants.VSPHERE_MAIN}.json"):
os.rename(
f"{constants.VSPHERE_MAIN}.json",
f"{constants.VSPHERE_MAIN}.json.backup",
)
# terraform initialization and destroy cluster
terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/"))
os.chdir(terraform_data_dir)
if Version.coerce(ocp_version) >= Version.coerce("4.6"):
# Download terraform ignition provider. For OCP upgrade clusters,
# ignition provider doesn't exist, so downloading in destroy job
# as well
terraform_plugins_path = ".terraform/plugins/linux_amd64/"
terraform_ignition_provider_path = os.path.join(
terraform_data_dir,
terraform_plugins_path,
"terraform-provider-ignition",
)
# check the upgrade history of cluster and checkout to the
# original installer release. This is due to the issue of not
# supporting terraform state of OCP 4.5 in installer
# release of 4.6 branch. More details in
# https://github.com/red-hat-storage/ocs-ci/issues/2941
is_cluster_upgraded = False
try:
upgrade_history = get_ocp_upgrade_history()
if len(upgrade_history) > 1:
is_cluster_upgraded = True
original_installed_ocp_version = upgrade_history[-1]
installer_release_branch = (
f"release-{original_installed_ocp_version[0:3]}")
clone_repo(
constants.VSPHERE_INSTALLER_REPO,
upi_repo_path,
installer_release_branch,
)
except Exception as ex:
logger.error(ex)
if not (os.path.exists(terraform_ignition_provider_path)
or is_cluster_upgraded):
get_terraform_ignition_provider(terraform_data_dir)
terraform.initialize()
else:
terraform.initialize(upgrade=True)
terraform.destroy(tfvars, refresh=(not self.folder_structure))
os.chdir(previous_dir)
# post destroy checks
self.post_destroy_checks()
