def deploy_powerflex_gw(setts, sah_node, tester, powerflexgw_vm): powerflexgw_ip = setts.powerflexgw_vm.public_api_ip Ssh.execute_command(powerflexgw_ip, "root", setts.powerflexgw_vm.root_password, "subscription-manager remove --all") Ssh.execute_command(powerflexgw_ip, "root", setts.powerflexgw_vm.root_password, "subscription-manager unregister") logger.info("=== deleting any existing powerflex gateway vm") sah_node.delete_powerflexgw_vm() logger.info("=== Creating the powerflex gateway vm") sah_node.create_powerflexgw_vm() tester.powerflexgw_vm_health_check() logger.info("Installing the powerflex gateway UI") powerflexgw_vm.upload_rpm() powerflexgw_vm.install_gateway() logger.info("Configuring the powerflex gateway vm") powerflexgw_vm.configure_gateway() logger.info("Retrieving and injecting SSL certificates") powerflexgw_vm.get_ssl_certificates() powerflexgw_vm.inject_ssl_certificates() logger.info("Restarting the gateway and cinder-volume") powerflexgw_vm.restart_gateway() powerflexgw_vm.restart_cinder_volume()
def deploy_undercloud(setts, sah_node, tester, director_vm): director_ip = setts.director_node.public_api_ip Ssh.execute_command(director_ip, "root", setts.director_node.root_password, "subscription-manager remove --all") Ssh.execute_command(director_ip, "root", setts.director_node.root_password, "subscription-manager unregister") sah_node.delete_director_vm() logger.info("=== create the director vm") sah_node.create_director_vm() tester.director_vm_health_check() logger.info("Preparing the Director VM") director_vm.apply_internal_repos() logger.debug("=== Uploading & configuring undercloud.conf . " "environment yaml ===") director_vm.upload_update_conf_files() logger.info("=== installing the director & undercloud ===") director_vm.inject_ssh_key() director_vm.upload_cloud_images() director_vm.install_director() _is_failed, _error = tester.verify_undercloud_installed() if _is_failed: raise _error
def cleanup_sah(self): logger.info("- Clean up any existing installation ") cmds = [ ' killall -u core', 'userdel -r core', 'rm -rf /var/lib/tftpboot/uefi/*' ] for cmd in cmds: Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, cmd)
def deploy_powerflex_mgmt(setts, sah_node, tester, powerflexmgmt_vm): powerflexmgmt_ip = setts.powerflexmgmt_vm.public_api_ip Ssh.execute_command(powerflexmgmt_ip, "root", setts.powerflexmgmt_vm.root_password, "subscription-manager remove --all") Ssh.execute_command(powerflexmgmt_ip, "root", setts.powerflexmgmt_vm.root_password, "subscription-manager unregister") logger.info("=== deleting any existing powerflex presentation server vm") sah_node.delete_powerflexmgmt_vm() logger.info("=== Creating the powerflex presentation server vm") sah_node.create_powerflexmgmt_vm() tester.powerflexmgmt_vm_health_check() logger.info("Installing the powerflex presentation server UI") powerflexmgmt_vm.upload_rpm() powerflexmgmt_vm.install_presentation_server()
def verify_subscription_status(public_api_ip, user, password, retries): i = 0 subscription_status = Ssh.execute_command( public_api_ip, user, password, "subscription-manager status")[0] while "Current" not in subscription_status and i < retries: if "Unknown" in subscription_status: return subscription_status time.sleep(60) subscription_status = \ Ssh.execute_command(public_api_ip, user, password, "subscription-manager status")[0] i += 1 return subscription_status
def run_playbooks(self): logger.info("- Run ansible playbook to generate ignition files etc") logfile = logger.handlers[0].baseFilename cmd = 'export ANSIBLE_LOG_PATH=' + logfile + '; ansible-playbook -i generated_inventory haocp.yaml' subprocess.call(cmd, shell=True, cwd='/home/ansible/JetPack/src/pilot/ansible') logger.info("Updating the dns settings") cmds = [ 'nmcli connection modify br0 ipv4.dns ' + self.settings.csah_node.os_ip, 'sed -i "s/nameserver.*/nameserver ' + self.settings.csah_node.os_ip + '/" /etc/resolv.conf' ] for cmd in cmds: Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, cmd)
def create_bootstrap_vm(self): logger.info("- Create the bootstrap VM") bootstrap_mac = self.get_inventory( )['all']['vars']['bootstrap_node'][0]['mac'] cmd = 'virt-install --name bootstrapkvm --ram 20480 --vcpu 8 --disk path=/home/bootstrapvm-disk.qcow2,format=qcow2,size=20 --os-variant generic --network=bridge=br0,model=virtio,mac=' + bootstrap_mac + ' --pxe --boot uefi,hd,network --noautoconsole --autostart &' re = Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, cmd) time.sleep(320)
def sah_health_check(self): logger.info("SAH node health check") if self.verify_rhsm_status: logger.debug("*** Verify the SAH node registered properly ***") for _ in range(60): subscription_status = self.verify_subscription_status( self.sah_ip, "root", self.settings.sah_node.root_password, self.settings.subscription_check_retries) if "Current" in subscription_status: break time.sleep(2) else: raise AssertionError("SAH did not register properly : " + subscription_status) logger.debug("*** Verify the SAH can ping its public gateway") gateway = self.settings.public_api_gateway test = self.ping_host(self.sah_ip, "root", self.settings.sah_node.root_password, gateway) if self.ping_success not in test: raise AssertionError("SAH cannot ping its public gateway : " + test) logger.debug("*** Verify the SAH can ping the outside world (ip)") test = self.ping_host(self.sah_ip, "root", self.settings.sah_node.root_password, "8.8.8.8") if self.ping_success not in test: raise AssertionError("SAH cannot ping the outside world (ip) : " + test) logger.debug("*** Verify the SAH can ping the outside world (dns)") test = self.ping_host(self.sah_ip, "root", self.settings.sah_node.root_password, "google.com") if self.ping_success not in test: raise AssertionError("SAH cannot ping the outside world (dns) : " + test) logger.debug("*** Verify the SAH can ping the idrac network") test = self.ping_host(self.sah_ip, "root", self.settings.sah_node.root_password, self.settings.sah_node.idrac_ip) if self.ping_success not in test: raise AssertionError("SAH cannot ping idrac networkn (ip) : " + test) logger.debug("*** Verify the SAH has KVM enabled *** ") cmd = 'ls -al /dev/kvm' if "No such file" in \ Ssh.execute_command(self.sah_ip, "root", self.settings.sah_node.root_password, cmd)[1]: raise AssertionError( "KVM Not running on the SAH node - make sure " "the node has been DTK'ed/Virtualization enabled " "in the Bios")
def wait_for_vm_to_come_up(self, target_ip, user, password): while True: status = Ssh.execute_command(target_ip, user, password, "ps")[0] if status != "host not up": break logger.debug("vm is not up. Sleeping...") time.sleep(10)
def ping_host(self, external_ip, user, passwd, target_host): for i in range(1, 30): ping_status = Ssh.execute_command( external_ip, user, passwd, "ping " + target_host + " -c 1 -w 30 ")[0] if self.ping_success in ping_status: logger.debug("Ping {} successful on attempt #{}".format( target_host, i)) break # noinspection PyUnboundLocalVariable return ping_status
def delete_bootstrap_vm(self): logger.info(" Destroy any existing bootstrap Vm") cmd = 'virsh list --all' bBoostrapDestroyed = False while bBoostrapDestroyed is False: re = Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, cmd) if 'bootstrap' in str(re): cmds = [ 'virsh undefine --nvram "bootstrapkvm"', 'virsh destroy bootstrapkvm'] for cm in cmds: Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, cm) else: bBoostrapDestroyed = True
def wait_for_vm_to_go_down(self, target_ip, user, password): while True: status = Ssh.execute_command( target_ip, user, password, "ps")[0] if status == "host not up": break time.sleep(5)
def verify_pools_attached(ip_addr, user, password, logfile): # check the xxxxx-posts.log for pool id's/repo's related errors. log_out = \ Ssh.execute_command(ip_addr, user, password, "cat " + logfile)[0] error1 = 'No subscriptions are available from the pool with' error2 = 'Removed temporarly as this error will show when ' \ 'not pulling from the cdn but internal repos' error3 = 'Could not find an OpenStack pool to attach to' if error1 in log_out or error2 in log_out or error3 in log_out: logger.info("*** post install log ***") logger.info(log_out) return False return True
def wait_for_bootstrap_ready(self): bBootstrap_ready = False while bBootstrap_ready is False: cmd = 'sudo su - core -c \'ssh -o "StrictHostKeyChecking no " bootstrap sudo ss -tulpn | grep -E "6443|22623|2379"\'' openedPorts = Ssh.execute_command_tty("localhost", "root", self.settings.csah_root_pwd, cmd) if ("22623" in str(openedPorts)) and ( "2379" in str(openedPorts)) and ("6443" in str(openedPorts)): logger.info(" ,, boostrap UP! ") bBootstrap_ready = True re = Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, "virsh list --all | grep bootstrapkvm")[0] if "shut off" in re: bPXe_complete = True logger.info("- Powering on the bootstrap VM") Ssh.execute_command("localhost", "root", self.settings.csah_root_pwd, "virsh start bootstrapkvm") time.sleep(60) logger.info("- Bootstrap VM is ready")
def provisioning_subnet_exists(self, subnet): logger.debug("Check if edge subnet {} already " "exists or not".format(subnet)) setts = self.settings user = setts.director_install_account_user ip = setts.director_node.public_api_ip pwd = setts.director_install_account_pwd is_subnet = False subnet_cmd = ("{} openstack subnet " "show {} -c name " "-f value".format(self.source_stackrc, subnet)) sn_out = Ssh.execute_command(ip, user, pwd, subnet_cmd)[0] if sn_out.strip() == subnet: is_subnet = True logger.info("Subnet {} already exists".format(subnet)) return is_subnet
def deploy(): ret_code = 0 # noinspection PyBroadException logger.debug("=================================") logger.info("=== Starting up ...") logger.debug("=================================") try: settings, args = get_settings() if args.validate_only is True: logger.info("Only validating ini/properties config values") else: if args.overcloud_only is True: logger.info("Only redeploying the overcloud") if args.skip_dashboard_vm is True: logger.info("Skipping Dashboard VM install") logger.info("Settings .ini: " + settings.settings_file) logger.info("Settings .properties " + settings.network_conf) settings.get_version_info() logger.info("source version # : " + settings.source_version) tester = Checkpoints() tester.verify_deployer_settings() if args.validate_only is True: logger.info("Settings validated") os._exit(0) tester.retreive_switches_config() non_sah_nodes = (settings.controller_nodes + settings.compute_nodes + settings.ceph_nodes) sah_node = Sah() tester.sah_health_check() # mutually exclusive command, configure tempest and quit. if args.tempest_config_only: logger.info("Only (re-)generating tempest.conf") director_vm = Director() director_vm.configure_tempest() os._exit(0) # mutually exclusive command, run tempest and quit. if args.run_tempest_only: logger.info("Only running tempest, will configure " + "tempest.conf if needed.") director_vm = Director() director_vm.run_tempest() os._exit(0) logger.info("Uploading configs/iso/scripts.") sah_node.clear_known_hosts() sah_node.handle_lock_files() sah_node.upload_iso() sah_node.upload_director_scripts() director_ip = settings.director_node.public_api_ip if args.overcloud_only is False: Ssh.execute_command(director_ip, "root", settings.director_node.root_password, "subscription-manager remove --all") Ssh.execute_command(director_ip, "root", settings.director_node.root_password, "subscription-manager unregister") sah_node.delete_director_vm() logger.info("=== create the director vm") sah_node.create_director_vm() tester.director_vm_health_check() logger.info("Preparing the Director VM") director_vm = Director() director_vm.apply_internal_repos() logger.debug("=== Uploading & configuring undercloud.conf . " "environment yaml ===") director_vm.upload_update_conf_files() logger.info("=== installing the director & undercloud ===") director_vm.inject_ssh_key() director_vm.upload_cloud_images() director_vm.install_director() tester.verify_undercloud_installed() if args.undercloud_only: return else: logger.info("=== Skipped Director VM/Undercloud install") director_vm = Director() logger.debug("Deleting overcloud stack") director_vm.delete_overcloud() if args.skip_dashboard_vm is False: logger.debug("Delete the Dashboard VM") dashboard_ip = settings.dashboard_node.public_api_ip logger.debug( Ssh.execute_command(dashboard_ip, "root", settings.dashboard_node.root_password, "subscription-manager remove --all")) Ssh.execute_command(dashboard_ip, "root", settings.dashboard_node.root_password, "subscription-manager unregister") sah_node.delete_dashboard_vm() logger.info("=== creating Dashboard VM") sah_node.create_dashboard_vm() tester.dashboard_vm_health_check() else: logger.info("Skipped the Dashboard VM install") logger.info("=== Preparing the overcloud ===") # The network-environment.yaml must be setup for use during DHCP # server configuration director_vm.setup_net_envt() director_vm.configure_dhcp_server() director_vm.node_discovery() director_vm.configure_idracs() director_vm.import_nodes() director_vm.node_introspection() director_vm.update_sshd_conf() director_vm.assign_node_roles() director_vm.revert_sshd_conf() director_vm.setup_templates() logger.info("=== Installing the overcloud ") logger.debug("installing the overcloud ... this might take a while") director_vm.deploy_overcloud() cmd = "source ~/stackrc; openstack stack list | grep " \ + settings.overcloud_name + " | awk '{print $8}'" overcloud_status = \ Ssh.execute_command_tty(director_ip, settings.director_install_account_user, settings.director_install_account_pwd, cmd)[0] logger.debug("=== Overcloud stack state : " + overcloud_status) if settings.hpg_enable: logger.info( " HugePages has been successfully configured with size: " + settings.hpg_size) if settings.numa_enable: logger.info(" NUMA has been successfully configured" " with hostos_cpus count: " + settings.hostos_cpu_count) logger.info("====================================") logger.info(" OverCloud deployment status: " + overcloud_status) logger.info(" log : /auto_results/ ") logger.info("====================================") if "CREATE_COMPLETE" not in overcloud_status: raise AssertionError("OverCloud did not install properly : " + overcloud_status) director_vm.summarize_deployment() tester.verify_computes_virtualization_enabled() tester.verify_backends_connectivity() if args.skip_dashboard_vm is False: director_vm.configure_dashboard() director_vm.enable_fencing() director_vm.run_sanity_test() external_sub_guid = director_vm.get_sanity_subnet() if external_sub_guid: director_vm.configure_tempest() run_tempest() logger.info("Deployment summary info; useful ip's etc.. " + "/auto_results/deployment_summary.log") except: # noqa: E722 logger.error(traceback.format_exc()) e = sys.exc_info()[0] logger.error(e) print e print traceback.format_exc() ret_code = 1 logger.info("log : /auto_results/ ") sys.exit(ret_code)
def deploy(): ret_code = 0 # noinspection PyBroadException try: logger.debug("=================================") logger.info("=== Starting up ...") logger.debug("=================================") parser = argparse.ArgumentParser( description='JetPack 10.x deployer') parser.add_argument('-s', '--settings', help='ini settings file, e.g settings/acme.ini', required=True) parser.add_argument('-undercloud_only', '--undercloud_only', help='Only reinstall the undercloud', action='store_true', required=False) parser.add_argument('-overcloud_only', '--overcloud_only', help='Only reinstall the overcloud', action='store_true', required=False) parser.add_argument('-skip_dashboard_vm', '--skip_dashboard_vm', help='Do not reinstall the Dashboard VM', action='store_true', required=False) parser.add_argument('-validate_only', '--validate_only', help='No deployment - just validate config values', action='store_true', required=False) args, others = parser.parse_known_args() if len(others) > 0: parser.print_help() msg = "Invalid argument(s) :" for each in others: msg += " " + each + ";" raise AssertionError(msg) if args.validate_only is True: logger.info("Only validating ini/properties config values") else: if args.overcloud_only is True: logger.info("Only redeploying the overcloud") if args.skip_dashboard_vm is True: logger.info("Skipping Dashboard VM install") logger.debug("loading settings files " + args.settings) settings = Settings(args.settings) logger.info("Settings .ini: " + settings.settings_file) logger.info("Settings .properties " + settings.network_conf) settings.get_version_info() logger.info("source version # : " + settings.source_version) tester = Checkpoints() tester.verify_deployer_settings() if args.validate_only is True: logger.info("Settings validated") os._exit(0) if settings.retreive_switches_config is True: tester.retreive_switches_config() non_sah_nodes = (settings.controller_nodes + settings.compute_nodes + settings.ceph_nodes) sah_node = Sah() tester.sah_health_check() logger.info("Uploading configs/iso/scripts..") sah_node.clear_known_hosts() sah_node.handle_lock_files() sah_node.upload_iso() sah_node.upload_director_scripts() director_ip = settings.director_node.public_api_ip if args.overcloud_only is False: Ssh.execute_command(director_ip, "root", settings.director_node.root_password, "subscription-manager remove --all") Ssh.execute_command(director_ip, "root", settings.director_node.root_password, "subscription-manager unregister") sah_node.delete_director_vm() logger.info("=== create the director vm") sah_node.create_director_vm() tester.director_vm_health_check() logger.info("Preparing the Director VM") director_vm = Director() director_vm.apply_internal_repos() logger.debug( "=== Uploading & configuring undercloud.conf . " "environment yaml ===") director_vm.upload_update_conf_files() logger.info("=== installing the director & undercloud ===") director_vm.inject_ssh_key() director_vm.upload_cloud_images() director_vm.install_director() tester.verify_undercloud_installed() if args.undercloud_only: return else: logger.info("=== Skipped Director VM/Undercloud install") director_vm = Director() logger.debug("Deleting overcloud stack") director_vm.delete_overcloud() if args.skip_dashboard_vm is False: logger.debug("Delete the Dashboard VM") dashboard_ip = settings.dashboard_node.public_api_ip logger.debug( Ssh.execute_command(dashboard_ip, "root", settings.dashboard_node.root_password, "subscription-manager remove --all")) Ssh.execute_command(dashboard_ip, "root", settings.dashboard_node.root_password, "subscription-manager unregister") sah_node.delete_dashboard_vm() logger.info("=== creating Dashboard VM") sah_node.create_dashboard_vm() tester.dashboard_vm_health_check() else: logger.info("Skipped the Dashboard VM install") logger.info("=== Preparing the overcloud ===") # The network-environment.yaml must be setup for use during DHCP # server configuration director_vm.setup_net_envt() director_vm.configure_dhcp_server() director_vm.node_discovery() director_vm.configure_idracs() director_vm.import_nodes() director_vm.node_introspection() director_vm.update_sshd_conf() director_vm.assign_node_roles() director_vm.revert_sshd_conf director_vm.setup_templates() logger.info("=== Installing the overcloud ") logger.debug("installing the overcloud ... this might take a while") director_vm.deploy_overcloud() cmd = "source ~/stackrc; openstack stack list | grep " \ + settings.overcloud_name + " | awk '{print $6}'" overcloud_status = \ Ssh.execute_command_tty(director_ip, settings.director_install_account_user, settings.director_install_account_pwd, cmd)[0] logger.debug("=== Overcloud stack state : " + overcloud_status) if settings.hpg_enable: logger.info( " HugePages has been successfully configured with size: " + settings.hpg_size) if settings.numa_enable: logger.info( " NUMA has been successfully configured" " with hostos_cpus count: " + settings.hostos_cpu_count) logger.info("====================================") logger.info(" OverCloud deployment status: " + overcloud_status) logger.info(" log : /auto_results/ ") logger.info("====================================") if "CREATE_COMPLETE" not in overcloud_status: raise AssertionError( "OverCloud did not install properly : " + overcloud_status) director_vm.summarize_deployment() tester.verify_computes_virtualization_enabled() tester.verify_backends_connectivity() if args.skip_dashboard_vm is False: director_vm.configure_dashboard() director_vm.enable_fencing() director_vm.enable_instance_ha() director_vm.configure_tempest() director_vm.run_sanity_test() run_tempest() logger.info("Deployment summary info; useful ip's etc.. " + "/auto_results/deployment_summary.log") except: logger.error(traceback.format_exc()) e = sys.exc_info()[0] logger.error(e) print e print traceback.format_exc() ret_code = 1 logger.info("log : /auto_results/ ") sys.exit(ret_code)
def run_as_root(self, command): return Ssh.execute_command(self.ip, "root", self.root_pwd, command)
def run(self, command): return Ssh.execute_command(self.ip, self.user, self.pwd, command)