def __init__(self): super(VSPHEREUPI.OCPDeployment, self).__init__() self.public_key = {} self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, "installer") self.previous_dir = os.getcwd() # get OCP version ocp_version = get_ocp_version() # create terraform_data directory self.terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR) create_directory_path(self.terraform_data_dir) # Download terraform binary based on ocp version and # update the installer path in ENV_DATA # use "0.11.14" for releases below OCP 4.5 terraform_version = config.DEPLOYMENT["terraform_version"] terraform_installer = get_terraform(version=terraform_version) config.ENV_DATA["terraform_installer"] = terraform_installer # Download terraform ignition provider # ignition provider dependancy from OCP 4.6 if Version.coerce(ocp_version) >= Version.coerce("4.6"): get_terraform_ignition_provider(self.terraform_data_dir) # Initialize Terraform self.terraform_work_dir = constants.VSPHERE_DIR self.terraform = Terraform(self.terraform_work_dir) self.folder_structure = False if Version.coerce(ocp_version) >= Version.coerce("4.5"): self.folder_structure = True config.ENV_DATA["folder_structure"] = self.folder_structure
def __init__(self): super(VSPHEREUPI.OCPDeployment, self).__init__() self.public_key = {} self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, 'installer') self.previous_dir = os.getcwd() # Download terraform binary based on ocp version and # update the installer path in ENV_DATA # use "0.11.14" for releases below OCP 4.5 terraform_version = config.DEPLOYMENT['terraform_version'] terraform_installer = get_terraform(version=terraform_version) config.ENV_DATA['terraform_installer'] = terraform_installer # Initialize Terraform self.terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR) create_directory_path(self.terraform_data_dir) self.terraform_work_dir = constants.VSPHERE_DIR self.terraform = Terraform(self.terraform_work_dir) ocp_version = get_ocp_version() self.folder_structure = False if Version.coerce(ocp_version) >= Version.coerce('4.5'): self.folder_structure = True config.ENV_DATA['folder_structure'] = self.folder_structure
def __init__(self): super(VSPHEREUPI.OCPDeployment, self).__init__() self.public_key = {} self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, 'installer') self.previous_dir = os.getcwd() self.terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR) create_directory_path(self.terraform_data_dir) self.terraform_work_dir = constants.VSPHERE_DIR self.terraform = Terraform(self.terraform_work_dir)
def add_nodes(self): """ Add new nodes to the cluster """ # create separate directory for scale-up terraform data scaleup_terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, constants.SCALEUP_TERRAFORM_DATA_DIR) create_directory_path(scaleup_terraform_data_dir) logger.info( f"scale-up terraform data directory: {scaleup_terraform_data_dir}") # git clone repo from openshift-misc clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path) # modify scale-up repo self.modify_scaleup_repo() config.ENV_DATA['vsphere_resource_pool'] = config.ENV_DATA.get( "cluster_name") # sync guest time with host if config.ENV_DATA.get('sync_time_with_host'): sync_time_with_host(constants.SCALEUP_VSPHERE_MACHINE_CONF, True) # get the RHCOS worker list self.rhcos_ips = get_node_ips() logger.info(f"RHCOS IP's: {json.dumps(self.rhcos_ips)}") # generate terraform variable for scaling nodes self.generate_terraform_vars_for_scaleup() # Add nodes using terraform scaleup_terraform = Terraform(constants.SCALEUP_VSPHERE_DIR) previous_dir = os.getcwd() os.chdir(scaleup_terraform_data_dir) scaleup_terraform.initialize() scaleup_terraform.apply(self.scale_up_terraform_var) scaleup_terraform_tfstate = os.path.join(scaleup_terraform_data_dir, "terraform.tfstate") out = scaleup_terraform.output(scaleup_terraform_tfstate, "rhel_worker") rhel_worker_nodes = json.loads(out)['value'] logger.info(f"RHEL worker nodes: {rhel_worker_nodes}") os.chdir(previous_dir) # Install OCP on rhel nodes rhel_install = OCPINSTALLRHEL(rhel_worker_nodes) rhel_install.prepare_rhel_nodes() rhel_install.execute_ansible_playbook() # Giving some time to settle down the new nodes time.sleep(self.wait_time) # wait for nodes to be in READY state wait_for_nodes_status(timeout=300)
def destroy_scaleup_nodes(self, scale_up_terraform_data_dir, scale_up_terraform_var): """ Destroy the scale-up nodes Args: scale_up_terraform_data_dir (str): Path to scale-up terraform data directory scale_up_terraform_var (str): Path to scale-up terraform.tfvars file """ clone_repo( constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path ) # modify scale-up repo self.modify_scaleup_repo() terraform_scale_up = Terraform( os.path.join( self.upi_scale_up_repo_path, "v4-testing-misc/v4-scaleup/vsphere/" ) ) os.chdir(scale_up_terraform_data_dir) terraform_scale_up.initialize(upgrade=True) terraform_scale_up.destroy(scale_up_terraform_var)
def destroy_scaleup_nodes(self, scale_up_terraform_data_dir, scale_up_terraform_var): """ Destroy the scale-up nodes Args: scale_up_terraform_data_dir (str): Path to scale-up terraform data directory scale_up_terraform_var (str): Path to scale-up terraform.tfvars file """ clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path) # git clone repo from cluster-launcher clone_repo(constants.VSPHERE_CLUSTER_LAUNCHER, self.cluster_launcer_repo_path) # modify scale-up repo helpers = VSPHEREHELPERS() helpers.modify_scaleup_repo() vsphere_dir = constants.SCALEUP_VSPHERE_DIR if Version.coerce(self.ocp_version) >= Version.coerce("4.5"): vsphere_dir = os.path.join( constants.CLUSTER_LAUNCHER_VSPHERE_DIR, f"aos-{get_ocp_version('_')}", "vsphere", ) terraform_scale_up = Terraform(vsphere_dir) os.chdir(scale_up_terraform_data_dir) terraform_scale_up.initialize(upgrade=True) terraform_scale_up.destroy(scale_up_terraform_var)
def destroy_cluster(self, log_level="DEBUG"): """ Destroy OCP cluster specific to vSphere UPI Args: log_level (str): log level openshift-installer (default: DEBUG) """ previous_dir = os.getcwd() terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) upi_repo_path = os.path.join( constants.EXTERNAL_DIR, 'installer', ) tfvars = os.path.join( config.ENV_DATA.get('cluster_path'), constants.TERRAFORM_DATA_DIR, constants.TERRAFORM_VARS ) clone_repo( constants.VSPHERE_INSTALLER_REPO, upi_repo_path ) if ( os.path.exists(f"{constants.VSPHERE_MAIN}.backup") and os.path.exists(f"{constants.VSPHERE_MAIN}.json") ): os.rename(f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup") terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/")) os.chdir(terraform_data_dir) terraform.initialize(upgrade=True) terraform.destroy(tfvars) os.chdir(previous_dir)
def destroy_cluster(self, log_level="DEBUG"): """ Destroy OCP cluster specific to vSphere UPI Args: log_level (str): log level openshift-installer (default: DEBUG) """ previous_dir = os.getcwd() # delete the extra disks self.delete_disks() # check whether cluster has scale-up nodes scale_up_terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, constants.SCALEUP_TERRAFORM_DATA_DIR ) scale_up_terraform_var = os.path.join( scale_up_terraform_data_dir, "scale_up_terraform.tfvars" ) if os.path.exists(scale_up_terraform_var): os.chdir(scale_up_terraform_data_dir) self.destroy_scaleup_nodes( scale_up_terraform_data_dir, scale_up_terraform_var ) os.chdir(previous_dir) terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) upi_repo_path = os.path.join( constants.EXTERNAL_DIR, 'installer', ) tfvars = os.path.join( config.ENV_DATA.get('cluster_path'), constants.TERRAFORM_DATA_DIR, constants.TERRAFORM_VARS ) clone_repo( constants.VSPHERE_INSTALLER_REPO, upi_repo_path, f'release-{get_ocp_version()}' ) if ( os.path.exists(f"{constants.VSPHERE_MAIN}.backup") and os.path.exists(f"{constants.VSPHERE_MAIN}.json") ): os.rename(f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup") terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/")) os.chdir(terraform_data_dir) terraform.initialize(upgrade=True) terraform.destroy(tfvars) os.chdir(previous_dir) # post destroy checks self.post_destroy_checks()
class OCPDeployment(BaseOCPDeployment): def __init__(self): super(VSPHEREUPI.OCPDeployment, self).__init__() self.public_key = {} self.upi_repo_path = os.path.join( constants.EXTERNAL_DIR, 'installer' ) self.previous_dir = os.getcwd() self.terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) create_directory_path(self.terraform_data_dir) self.terraform_work_dir = constants.VSPHERE_DIR self.terraform = Terraform(self.terraform_work_dir) def deploy_prereq(self): """ Pre-Requisites for vSphere UPI Deployment """ super(VSPHEREUPI.OCPDeployment, self).deploy_prereq() # create ignitions self.create_ignitions() self.kubeconfig = os.path.join(self.cluster_path, config.RUN.get('kubeconfig_location')) # git clone repo from openshift installer clone_openshift_installer() # upload bootstrap ignition to public access server bootstrap_path = os.path.join(config.ENV_DATA.get('cluster_path'), constants.BOOTSTRAP_IGN) remote_path = os.path.join( config.ENV_DATA.get('path_to_upload'), f"{config.RUN.get('run_id')}_{constants.BOOTSTRAP_IGN}" ) upload_file( config.ENV_DATA.get('httpd_server'), bootstrap_path, remote_path, config.ENV_DATA.get('httpd_server_user'), config.ENV_DATA.get('httpd_server_password') ) # generate bootstrap ignition url path_to_bootstrap_on_remote = remote_path.replace("/var/www/html/", "") bootstrap_ignition_url = ( f"http://{config.ENV_DATA.get('httpd_server')}/" f"{path_to_bootstrap_on_remote}" ) logger.info(f"bootstrap_ignition_url: {bootstrap_ignition_url}") config.ENV_DATA['bootstrap_ignition_url'] = bootstrap_ignition_url # load master and worker ignitions to variables master_ignition_path = os.path.join( config.ENV_DATA.get('cluster_path'), constants.MASTER_IGN ) master_ignition = read_file_as_str(f"{master_ignition_path}") config.ENV_DATA['control_plane_ignition'] = master_ignition worker_ignition_path = os.path.join( config.ENV_DATA.get('cluster_path'), constants.WORKER_IGN ) worker_ignition = read_file_as_str(f"{worker_ignition_path}") config.ENV_DATA['compute_ignition'] = worker_ignition cluster_domain = ( f"{config.ENV_DATA.get('cluster_name')}." f"{config.ENV_DATA.get('base_domain')}" ) config.ENV_DATA['cluster_domain'] = cluster_domain # generate terraform variables from template logger.info("Generating terraform variables") _templating = Templating() terraform_var_template = "terraform.tfvars.j2" terraform_var_template_path = os.path.join( "ocp-deployment", terraform_var_template ) terraform_config_str = _templating.render_template( terraform_var_template_path, config.ENV_DATA ) terraform_var_yaml = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, "terraform.tfvars.yaml" ) with open(terraform_var_yaml, "w") as f: f.write(terraform_config_str) self.terraform_var = convert_yaml2tfvars(terraform_var_yaml) # update the machine configurations update_machine_conf() # sync guest time with host if config.ENV_DATA.get('sync_time_with_host'): sync_time_with_host(constants.INSTALLER_MACHINE_CONF, True) def create_config(self): """ Creates the OCP deploy config for the vSphere """ # Generate install-config from template _templating = Templating() ocp_install_template = ( f"install-config-{self.deployment_platform}-" f"{self.deployment_type}.yaml.j2" ) ocp_install_template_path = os.path.join( "ocp-deployment", ocp_install_template ) install_config_str = _templating.render_template( ocp_install_template_path, config.ENV_DATA ) # Parse the rendered YAML so that we can manipulate the object directly install_config_obj = yaml.safe_load(install_config_str) install_config_obj['pullSecret'] = self.get_pull_secret() install_config_obj['sshKey'] = self.get_ssh_key() install_config_str = yaml.safe_dump(install_config_obj) install_config = os.path.join(self.cluster_path, "install-config.yaml") with open(install_config, "w") as f: f.write(install_config_str) def create_ignitions(self): """ Creates the ignition files """ logger.info("creating ignition files for the cluster") run_cmd( f"{self.installer} create ignition-configs " f"--dir {self.cluster_path} " ) def configure_storage_for_image_registry(self, kubeconfig): """ Configures storage for the image registry """ logger.info("configuring storage for image registry") patch = " '{\"spec\":{\"storage\":{\"emptyDir\":{}}}}' " run_cmd( f"oc --kubeconfig {kubeconfig} patch " f"configs.imageregistry.operator.openshift.io " f"cluster --type merge --patch {patch}" ) def deploy(self, log_cli_level='DEBUG'): """ Deployment specific to OCP cluster on this platform Args: log_cli_level (str): openshift installer's log level (default: "DEBUG") """ logger.info("Deploying OCP cluster for vSphere platform") logger.info( f"Openshift-installer will be using loglevel:{log_cli_level}" ) os.chdir(self.terraform_data_dir) self.terraform.initialize() self.terraform.apply(self.terraform_var) os.chdir(self.previous_dir) logger.info("waiting for bootstrap to complete") try: run_cmd( f"{self.installer} wait-for bootstrap-complete " f"--dir {self.cluster_path} " f"--log-level {log_cli_level}", timeout=3600 ) except CommandFailed as e: if constants.GATHER_BOOTSTRAP_PATTERN in str(e): try: gather_bootstrap() except Exception as ex: logger.error(ex) raise e if not config.DEPLOYMENT['preserve_bootstrap_node']: logger.info("removing bootstrap node") os.chdir(self.terraform_data_dir) self.terraform.apply( self.terraform_var, bootstrap_complete=True ) os.chdir(self.previous_dir) OCP.set_kubeconfig(self.kubeconfig) # wait for all nodes to generate CSR # From OCP version 4.4 and above, we have to approve CSR manually # for all the nodes ocp_version = get_ocp_version() if Version.coerce(ocp_version) >= Version.coerce('4.4'): wait_for_all_nodes_csr_and_approve(timeout=1200, sleep=30) # wait for image registry to show-up co = "image-registry" wait_for_co(co) # patch image registry to null self.configure_storage_for_image_registry(self.kubeconfig) # wait for install to complete logger.info("waiting for install to complete") run_cmd( f"{self.installer} wait-for install-complete " f"--dir {self.cluster_path} " f"--log-level {log_cli_level}", timeout=1800 ) # Approving CSRs here in-case if any exists approve_pending_csr() self.test_cluster()
class OCPDeployment(BaseOCPDeployment): def __init__(self): super(VSPHEREUPI.OCPDeployment, self).__init__() self.public_key = {} self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, 'installer') self.previous_dir = os.getcwd() self.terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR) create_directory_path(self.terraform_data_dir) self.terraform_work_dir = constants.VSPHERE_DIR self.terraform = Terraform(self.terraform_work_dir) def deploy_prereq(self): """ Pre-Requisites for vSphere UPI Deployment """ super(VSPHEREUPI.OCPDeployment, self).deploy_prereq() # create ignitions self.create_ignitions() self.kubeconfig = os.path.join( self.cluster_path, config.RUN.get('kubeconfig_location')) # git clone repo from openshift installer clone_repo(constants.VSPHERE_INSTALLER_REPO, self.upi_repo_path) # upload bootstrap ignition to public access server bootstrap_path = os.path.join(config.ENV_DATA.get('cluster_path'), constants.BOOTSTRAP_IGN) remote_path = os.path.join( config.ENV_DATA.get('path_to_upload'), f"{config.RUN.get('run_id')}_{constants.BOOTSTRAP_IGN}") upload_file(config.ENV_DATA.get('httpd_server'), bootstrap_path, remote_path, config.ENV_DATA.get('httpd_server_user'), config.ENV_DATA.get('httpd_server_password')) # generate bootstrap ignition url path_to_bootstrap_on_remote = remote_path.replace( "/var/www/html/", "") bootstrap_ignition_url = ( f"http://{config.ENV_DATA.get('httpd_server')}/" f"{path_to_bootstrap_on_remote}") logger.info(f"bootstrap_ignition_url: {bootstrap_ignition_url}") config.ENV_DATA['bootstrap_ignition_url'] = bootstrap_ignition_url # load master and worker ignitions to variables master_ignition_path = os.path.join( config.ENV_DATA.get('cluster_path'), constants.MASTER_IGN) master_ignition = read_file_as_str(f"{master_ignition_path}") config.ENV_DATA['control_plane_ignition'] = master_ignition worker_ignition_path = os.path.join( config.ENV_DATA.get('cluster_path'), constants.WORKER_IGN) worker_ignition = read_file_as_str(f"{worker_ignition_path}") config.ENV_DATA['compute_ignition'] = worker_ignition cluster_domain = (f"{config.ENV_DATA.get('cluster_name')}." f"{config.ENV_DATA.get('base_domain')}") config.ENV_DATA['cluster_domain'] = cluster_domain # generate terraform variables from template logger.info("Generating terraform variables") _templating = Templating() terraform_var_template = "terraform.tfvars.j2" terraform_var_template_path = os.path.join("ocp-deployment", terraform_var_template) terraform_config_str = _templating.render_template( terraform_var_template_path, config.ENV_DATA) terraform_var_yaml = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR, "terraform.tfvars.yaml") with open(terraform_var_yaml, "w") as f: f.write(terraform_config_str) self.terraform_var = convert_yaml2tfvars(terraform_var_yaml) # update gateway and DNS if config.ENV_DATA.get('gateway'): replace_content_in_file(constants.INSTALLER_IGNITION, '${cidrhost(var.machine_cidr,1)}', f"{config.ENV_DATA.get('gateway')}") if config.ENV_DATA.get('dns'): replace_content_in_file(constants.INSTALLER_IGNITION, constants.INSTALLER_DEFAULT_DNS, f"{config.ENV_DATA.get('dns')}") # update the zone in route if config.ENV_DATA.get('region'): def_zone = 'provider "aws" { region = "%s" } \n' % config.ENV_DATA.get( 'region') replace_content_in_file(constants.INSTALLER_ROUTE53, "xyz", def_zone) # increase memory if config.ENV_DATA.get('memory'): replace_content_in_file(constants.INSTALLER_MACHINE_CONF, '${var.memory}', config.ENV_DATA.get('memory')) # increase CPUs worker_num_cpus = config.ENV_DATA.get('worker_num_cpus') master_num_cpus = config.ENV_DATA.get('master_num_cpus') if worker_num_cpus or master_num_cpus: with open(constants.VSPHERE_MAIN, 'r') as fd: obj = hcl.load(fd) if worker_num_cpus: obj['module']['compute']['num_cpu'] = worker_num_cpus if master_num_cpus: obj['module']['control_plane'][ 'num_cpu'] = master_num_cpus # Dump data to json file since hcl module # doesn't support dumping of data in HCL format dump_data_to_json(obj, f"{constants.VSPHERE_MAIN}.json") os.rename(constants.VSPHERE_MAIN, f"{constants.VSPHERE_MAIN}.backup") def create_config(self): """ Creates the OCP deploy config for the vSphere """ # Generate install-config from template _templating = Templating() ocp_install_template = ( f"install-config-{self.deployment_platform}-" f"{self.deployment_type}.yaml.j2") ocp_install_template_path = os.path.join("ocp-deployment", ocp_install_template) install_config_str = _templating.render_template( ocp_install_template_path, config.ENV_DATA) # Parse the rendered YAML so that we can manipulate the object directly install_config_obj = yaml.safe_load(install_config_str) install_config_obj['pullSecret'] = self.get_pull_secret() install_config_obj['sshKey'] = self.get_ssh_key() install_config_str = yaml.safe_dump(install_config_obj) install_config = os.path.join(self.cluster_path, "install-config.yaml") with open(install_config, "w") as f: f.write(install_config_str) def create_ignitions(self): """ Creates the ignition files """ logger.info("creating ignition files for the cluster") run_cmd(f"{self.installer} create ignition-configs " f"--dir {self.cluster_path} ") def configure_storage_for_image_registry(self, kubeconfig): """ Configures storage for the image registry """ logger.info("configuring storage for image registry") patch = " '{\"spec\":{\"storage\":{\"emptyDir\":{}}}}' " run_cmd(f"oc --kubeconfig {kubeconfig} patch " f"configs.imageregistry.operator.openshift.io " f"cluster --type merge --patch {patch}") def deploy(self, log_cli_level='DEBUG'): """ Deployment specific to OCP cluster on this platform Args: log_cli_level (str): openshift installer's log level (default: "DEBUG") """ logger.info("Deploying OCP cluster for vSphere platform") logger.info( f"Openshift-installer will be using loglevel:{log_cli_level}") os.chdir(self.terraform_data_dir) self.terraform.initialize() self.terraform.apply(self.terraform_var) os.chdir(self.previous_dir) logger.info("waiting for bootstrap to complete") run_cmd( f"{self.installer} wait-for bootstrap-complete " f"--dir {self.cluster_path} " f"--log-level {log_cli_level}", timeout=3600) logger.info("removing bootstrap node") os.chdir(self.terraform_data_dir) self.terraform.apply(self.terraform_var, bootstrap_complete=True) os.chdir(self.previous_dir) OCP.set_kubeconfig(self.kubeconfig) # wait for image registry to show-up co = "image-registry" wait_for_co(co) # patch image registry to null self.configure_storage_for_image_registry(self.kubeconfig) # wait for install to complete logger.info("waiting for install to complete") run_cmd( f"{self.installer} wait-for install-complete " f"--dir {self.cluster_path} " f"--log-level {log_cli_level}", timeout=1800) self.test_cluster()
def destroy_cluster(self, log_level="DEBUG"): """ Destroy OCP cluster specific to vSphere UPI Args: log_level (str): log level openshift-installer (default: DEBUG) """ previous_dir = os.getcwd() # Download terraform binary based on terraform version # in terraform.log terraform_log_path = os.path.join(config.ENV_DATA.get("cluster_path"), config.ENV_DATA.get("TF_LOG_FILE")) # check for terraform.log, this check is for partially # deployed clusters try: with open(terraform_log_path, "r") as fd: logger.debug( f"Reading terraform version from {terraform_log_path}") version_line = fd.readline() terraform_version = version_line.split()[-1] except FileNotFoundError: logger.debug(f"{terraform_log_path} file not found") terraform_version = config.DEPLOYMENT["terraform_version"] terraform_installer = get_terraform(version=terraform_version) config.ENV_DATA["terraform_installer"] = terraform_installer # getting OCP version here since we run destroy job as # separate job in jenkins ocp_version = get_ocp_version() self.folder_structure = False if Version.coerce(ocp_version) >= Version.coerce("4.5"): set_aws_region() self.folder_structure = True config.ENV_DATA["folder_structure"] = self.folder_structure # delete the extra disks self.delete_disks() # check whether cluster has scale-up nodes scale_up_terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, constants.SCALEUP_TERRAFORM_DATA_DIR, ) scale_up_terraform_var = os.path.join(scale_up_terraform_data_dir, "scale_up_terraform.tfvars") if os.path.exists(scale_up_terraform_var): os.chdir(scale_up_terraform_data_dir) self.destroy_scaleup_nodes(scale_up_terraform_data_dir, scale_up_terraform_var) os.chdir(previous_dir) terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) upi_repo_path = os.path.join( constants.EXTERNAL_DIR, "installer", ) tfvars = os.path.join( config.ENV_DATA.get("cluster_path"), constants.TERRAFORM_DATA_DIR, constants.TERRAFORM_VARS, ) clone_openshift_installer() if os.path.exists( f"{constants.VSPHERE_MAIN}.backup") and os.path.exists( f"{constants.VSPHERE_MAIN}.json"): os.rename( f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup", ) # terraform initialization and destroy cluster terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/")) os.chdir(terraform_data_dir) if Version.coerce(ocp_version) >= Version.coerce("4.6"): # Download terraform ignition provider. For OCP upgrade clusters, # ignition provider doesn't exist, so downloading in destroy job # as well terraform_plugins_path = ".terraform/plugins/linux_amd64/" terraform_ignition_provider_path = os.path.join( terraform_data_dir, terraform_plugins_path, "terraform-provider-ignition", ) # check the upgrade history of cluster and checkout to the # original installer release. This is due to the issue of not # supporting terraform state of OCP 4.5 in installer # release of 4.6 branch. More details in # https://github.com/red-hat-storage/ocs-ci/issues/2941 is_cluster_upgraded = False try: upgrade_history = get_ocp_upgrade_history() if len(upgrade_history) > 1: is_cluster_upgraded = True original_installed_ocp_version = upgrade_history[-1] installer_release_branch = ( f"release-{original_installed_ocp_version[0:3]}") clone_repo( constants.VSPHERE_INSTALLER_REPO, upi_repo_path, installer_release_branch, ) except Exception as ex: logger.error(ex) if not (os.path.exists(terraform_ignition_provider_path) or is_cluster_upgraded): get_terraform_ignition_provider(terraform_data_dir) terraform.initialize() else: terraform.initialize(upgrade=True) terraform.destroy(tfvars, refresh=(not self.folder_structure)) os.chdir(previous_dir) # post destroy checks self.post_destroy_checks()
class OCPDeployment(BaseOCPDeployment): def __init__(self): super(VSPHEREUPI.OCPDeployment, self).__init__() self.public_key = {} self.upi_repo_path = os.path.join(constants.EXTERNAL_DIR, "installer") self.previous_dir = os.getcwd() # get OCP version ocp_version = get_ocp_version() # create terraform_data directory self.terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR) create_directory_path(self.terraform_data_dir) # Download terraform binary based on ocp version and # update the installer path in ENV_DATA # use "0.11.14" for releases below OCP 4.5 terraform_version = config.DEPLOYMENT["terraform_version"] terraform_installer = get_terraform(version=terraform_version) config.ENV_DATA["terraform_installer"] = terraform_installer # Download terraform ignition provider # ignition provider dependancy from OCP 4.6 if Version.coerce(ocp_version) >= Version.coerce("4.6"): get_terraform_ignition_provider(self.terraform_data_dir) # Initialize Terraform self.terraform_work_dir = constants.VSPHERE_DIR self.terraform = Terraform(self.terraform_work_dir) self.folder_structure = False if Version.coerce(ocp_version) >= Version.coerce("4.5"): self.folder_structure = True config.ENV_DATA["folder_structure"] = self.folder_structure def deploy_prereq(self): """ Pre-Requisites for vSphere UPI Deployment """ super(VSPHEREUPI.OCPDeployment, self).deploy_prereq() # generate manifests self.generate_manifests() # create ignitions self.create_ignitions() self.kubeconfig = os.path.join( self.cluster_path, config.RUN.get("kubeconfig_location")) self.terraform_var = os.path.join( config.ENV_DATA["cluster_path"], constants.TERRAFORM_DATA_DIR, "terraform.tfvars", ) # git clone repo from openshift installer clone_openshift_installer() # generate terraform variable file generate_terraform_vars_and_update_machine_conf() # sync guest time with host vm_file = (constants.VM_MAIN if self.folder_structure else constants.INSTALLER_MACHINE_CONF) if config.ENV_DATA.get("sync_time_with_host"): sync_time_with_host(vm_file, True) def create_config(self): """ Creates the OCP deploy config for the vSphere """ # Generate install-config from template _templating = Templating() ocp_install_template = ( f"install-config-{self.deployment_platform}-" f"{self.deployment_type}.yaml.j2") ocp_install_template_path = os.path.join("ocp-deployment", ocp_install_template) install_config_str = _templating.render_template( ocp_install_template_path, config.ENV_DATA) # Parse the rendered YAML so that we can manipulate the object directly install_config_obj = yaml.safe_load(install_config_str) install_config_obj["pullSecret"] = self.get_pull_secret() install_config_obj["sshKey"] = self.get_ssh_key() install_config_str = yaml.safe_dump(install_config_obj) install_config = os.path.join(self.cluster_path, "install-config.yaml") with open(install_config, "w") as f: f.write(install_config_str) def generate_manifests(self): """ Generates manifest files """ logger.info("creating manifest files for the cluster") run_cmd( f"{self.installer} create manifests --dir {self.cluster_path}") # remove machines and machinesets # Some of the manifests produced are for creating machinesets # and machine objects. We should remove these, because we don't # want to involve the machine-API operator and # machine-api-operator during install. manifest_files_path = os.path.join(self.cluster_path, "openshift") files_to_remove = glob.glob( f"{manifest_files_path}/99_openshift-cluster-api_" f"master-machines-*.yaml") files_to_remove.extend( glob.glob(f"{manifest_files_path}/99_openshift-cluster-api_" f"worker-machineset-*.yaml")) logger.debug( f"Removing machines and machineset files: {files_to_remove}") for each_file in files_to_remove: os.remove(each_file) def create_ignitions(self): """ Creates the ignition files """ logger.info("creating ignition files for the cluster") run_cmd(f"{self.installer} create ignition-configs " f"--dir {self.cluster_path} ") @retry(exceptions.CommandFailed, tries=10, delay=30, backoff=1) def configure_storage_for_image_registry(self, kubeconfig): """ Configures storage for the image registry """ logger.info("configuring storage for image registry") patch = ' \'{"spec":{"storage":{"emptyDir":{}}}}\' ' run_cmd(f"oc --kubeconfig {kubeconfig} patch " f"configs.imageregistry.operator.openshift.io " f"cluster --type merge --patch {patch}") def deploy(self, log_cli_level="DEBUG"): """ Deployment specific to OCP cluster on this platform Args: log_cli_level (str): openshift installer's log level (default: "DEBUG") """ logger.info("Deploying OCP cluster for vSphere platform") logger.info( f"Openshift-installer will be using loglevel:{log_cli_level}") os.chdir(self.terraform_data_dir) self.terraform.initialize() self.terraform.apply(self.terraform_var) os.chdir(self.previous_dir) logger.info("waiting for bootstrap to complete") try: run_cmd( f"{self.installer} wait-for bootstrap-complete " f"--dir {self.cluster_path} " f"--log-level {log_cli_level}", timeout=3600, ) except CommandFailed as e: if constants.GATHER_BOOTSTRAP_PATTERN in str(e): try: gather_bootstrap() except Exception as ex: logger.error(ex) raise e if self.folder_structure: # comment bootstrap module comment_bootstrap_in_lb_module() # remove bootstrap IP in load balancer and # restart haproxy lb = LoadBalancer() lb.remove_boostrap_in_proxy() lb.restart_haproxy() # remove bootstrap node if not config.DEPLOYMENT["preserve_bootstrap_node"]: logger.info("removing bootstrap node") os.chdir(self.terraform_data_dir) if self.folder_structure: self.terraform.destroy_module(self.terraform_var, constants.BOOTSTRAP_MODULE) else: self.terraform.apply(self.terraform_var, bootstrap_complete=True) os.chdir(self.previous_dir) OCP.set_kubeconfig(self.kubeconfig) # wait for all nodes to generate CSR # From OCP version 4.4 and above, we have to approve CSR manually # for all the nodes ocp_version = get_ocp_version() if Version.coerce(ocp_version) >= Version.coerce("4.4"): wait_for_all_nodes_csr_and_approve(timeout=1200, sleep=30) # wait for image registry to show-up co = "image-registry" wait_for_co(co) # patch image registry to null self.configure_storage_for_image_registry(self.kubeconfig) # wait for install to complete logger.info("waiting for install to complete") run_cmd( f"{self.installer} wait-for install-complete " f"--dir {self.cluster_path} " f"--log-level {log_cli_level}", timeout=1800, ) # Approving CSRs here in-case if any exists approve_pending_csr() self.test_cluster()
def add_nodes(self): """ Add new nodes to the cluster """ # create separate directory for scale-up terraform data scaleup_terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, constants.SCALEUP_TERRAFORM_DATA_DIR, ) create_directory_path(scaleup_terraform_data_dir) logger.info( f"scale-up terraform data directory: {scaleup_terraform_data_dir}") # git clone repo from openshift-misc clone_repo(constants.VSPHERE_SCALEUP_REPO, self.upi_scale_up_repo_path) # git clone repo from cluster-launcher clone_repo(constants.VSPHERE_CLUSTER_LAUNCHER, self.cluster_launcer_repo_path) helpers = VSPHEREHELPERS() helpers.modify_scaleup_repo() config.ENV_DATA["vsphere_resource_pool"] = config.ENV_DATA.get( "cluster_name") # sync guest time with host sync_time_with_host_file = constants.SCALEUP_VSPHERE_MACHINE_CONF if config.ENV_DATA["folder_structure"]: sync_time_with_host_file = os.path.join( constants.CLUSTER_LAUNCHER_VSPHERE_DIR, f"aos-{get_ocp_version(seperator='_')}", constants.CLUSTER_LAUNCHER_MACHINE_CONF, ) if config.ENV_DATA.get("sync_time_with_host"): sync_time_with_host(sync_time_with_host_file, True) # get the RHCOS worker list rhcos_ips = get_node_ips() logger.info(f"RHCOS IP's: {json.dumps(rhcos_ips)}") # generate terraform variable for scaling nodes self.scale_up_terraform_var = helpers.generate_terraform_vars_for_scaleup( rhcos_ips) # choose the vsphere_dir based on OCP version # generate cluster_info and config yaml files # for OCP version greater than 4.4 vsphere_dir = constants.SCALEUP_VSPHERE_DIR rhel_module = "rhel-worker" if Version.coerce(self.ocp_version) >= Version.coerce("4.5"): vsphere_dir = os.path.join( constants.CLUSTER_LAUNCHER_VSPHERE_DIR, f"aos-{get_ocp_version('_')}", "vsphere", ) helpers.generate_cluster_info() helpers.generate_config_yaml() rhel_module = "RHEL_WORKER_LIST" # Add nodes using terraform scaleup_terraform = Terraform(vsphere_dir) previous_dir = os.getcwd() os.chdir(scaleup_terraform_data_dir) scaleup_terraform.initialize() scaleup_terraform.apply(self.scale_up_terraform_var) scaleup_terraform_tfstate = os.path.join(scaleup_terraform_data_dir, "terraform.tfstate") out = scaleup_terraform.output(scaleup_terraform_tfstate, rhel_module) if config.ENV_DATA["folder_structure"]: rhel_worker_nodes = out.strip().replace('"', "").split(",") else: rhel_worker_nodes = json.loads(out)["value"] logger.info(f"RHEL worker nodes: {rhel_worker_nodes}") os.chdir(previous_dir) # Install OCP on rhel nodes rhel_install = OCPINSTALLRHEL(rhel_worker_nodes) rhel_install.prepare_rhel_nodes() rhel_install.execute_ansible_playbook() # Giving some time to settle down the new nodes time.sleep(self.wait_time) # wait for nodes to be in READY state wait_for_nodes_status(timeout=300)
def destroy_cluster(self, log_level="DEBUG"): """ Destroy OCP cluster specific to vSphere UPI Args: log_level (str): log level openshift-installer (default: DEBUG) """ previous_dir = os.getcwd() # Download terraform binary based on terraform version # in terraform.log terraform_log_path = os.path.join(config.ENV_DATA.get('cluster_path'), config.ENV_DATA.get('TF_LOG_FILE')) # check for terraform.log, this check is for partially # deployed clusters try: with open(terraform_log_path, 'r') as fd: logger.debug( f"Reading terraform version from {terraform_log_path}") version_line = fd.readline() terraform_version = version_line.split()[-1] except FileNotFoundError: logger.debug(f"{terraform_log_path} file not found") terraform_version = config.DEPLOYMENT['terraform_version'] terraform_installer = get_terraform(version=terraform_version) config.ENV_DATA['terraform_installer'] = terraform_installer # getting OCP version here since we run destroy job as # separate job in jenkins ocp_version = get_ocp_version() self.folder_structure = False if Version.coerce(ocp_version) >= Version.coerce('4.5'): set_aws_region() self.folder_structure = True config.ENV_DATA['folder_structure'] = self.folder_structure # delete the extra disks self.delete_disks() # check whether cluster has scale-up nodes scale_up_terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, constants.SCALEUP_TERRAFORM_DATA_DIR) scale_up_terraform_var = os.path.join(scale_up_terraform_data_dir, "scale_up_terraform.tfvars") if os.path.exists(scale_up_terraform_var): os.chdir(scale_up_terraform_data_dir) self.destroy_scaleup_nodes(scale_up_terraform_data_dir, scale_up_terraform_var) os.chdir(previous_dir) terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) upi_repo_path = os.path.join( constants.EXTERNAL_DIR, 'installer', ) tfvars = os.path.join(config.ENV_DATA.get('cluster_path'), constants.TERRAFORM_DATA_DIR, constants.TERRAFORM_VARS) clone_openshift_installer() if (os.path.exists(f"{constants.VSPHERE_MAIN}.backup") and os.path.exists(f"{constants.VSPHERE_MAIN}.json")): os.rename(f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup") # terraform initialization and destroy cluster terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/")) os.chdir(terraform_data_dir) if Version.coerce(ocp_version) >= Version.coerce('4.6'): terraform.initialize() else: terraform.initialize(upgrade=True) terraform.destroy(tfvars, refresh=(not self.folder_structure)) os.chdir(previous_dir) # post destroy checks self.post_destroy_checks()
def destroy_cluster(self, log_level="DEBUG"): """ Destroy OCP cluster specific to vSphere UPI Args: log_level (str): log level openshift-installer (default: DEBUG) """ previous_dir = os.getcwd() # Download terraform binary based on ocp version and # update the installer path in ENV_DATA # use "0.11.14" for releases below OCP 4.5 # TODO: For cluster installed by old version of terraform we need to # still run old version for destroy ( upgrade scenario ) terraform_version = config.DEPLOYMENT['terraform_version'] terraform_installer = get_terraform(version=terraform_version) config.ENV_DATA['terraform_installer'] = terraform_installer # delete the extra disks self.delete_disks() # check whether cluster has scale-up nodes scale_up_terraform_data_dir = os.path.join( self.cluster_path, constants.TERRAFORM_DATA_DIR, constants.SCALEUP_TERRAFORM_DATA_DIR) scale_up_terraform_var = os.path.join(scale_up_terraform_data_dir, "scale_up_terraform.tfvars") if os.path.exists(scale_up_terraform_var): os.chdir(scale_up_terraform_data_dir) self.destroy_scaleup_nodes(scale_up_terraform_data_dir, scale_up_terraform_var) os.chdir(previous_dir) terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) upi_repo_path = os.path.join( constants.EXTERNAL_DIR, 'installer', ) tfvars = os.path.join(config.ENV_DATA.get('cluster_path'), constants.TERRAFORM_DATA_DIR, constants.TERRAFORM_VARS) clone_openshift_installer() if (os.path.exists(f"{constants.VSPHERE_MAIN}.backup") and os.path.exists(f"{constants.VSPHERE_MAIN}.json")): os.rename(f"{constants.VSPHERE_MAIN}.json", f"{constants.VSPHERE_MAIN}.json.backup") # getting OCP version here since we run destroy job as # separate job in jenkins ocp_version = get_ocp_version() self.folder_structure = False if Version.coerce(ocp_version) >= Version.coerce('4.5'): set_aws_region() self.folder_structure = True # terraform initialization and destroy cluster terraform = Terraform(os.path.join(upi_repo_path, "upi/vsphere/")) os.chdir(terraform_data_dir) terraform.initialize(upgrade=True) terraform.destroy(tfvars, refresh=(not self.folder_structure)) os.chdir(previous_dir) # post destroy checks self.post_destroy_checks()