def deploy_powerflex_gw(setts, sah_node, tester, powerflexgw_vm):
powerflexgw_ip = setts.powerflexgw_vm.public_api_ip
Ssh.execute_command(powerflexgw_ip, "root",
setts.powerflexgw_vm.root_password,
"subscription-manager remove --all")
Ssh.execute_command(powerflexgw_ip, "root",
setts.powerflexgw_vm.root_password,
"subscription-manager unregister")
logger.info("=== deleting any existing powerflex gateway vm")
sah_node.delete_powerflexgw_vm()
logger.info("=== Creating the powerflex gateway vm")
sah_node.create_powerflexgw_vm()
tester.powerflexgw_vm_health_check()
logger.info("Installing the powerflex gateway UI")
powerflexgw_vm.upload_rpm()
powerflexgw_vm.install_gateway()
logger.info("Configuring the powerflex gateway vm")
powerflexgw_vm.configure_gateway()
logger.info("Retrieving and injecting SSL certificates")
powerflexgw_vm.get_ssl_certificates()
powerflexgw_vm.inject_ssl_certificates()
logger.info("Restarting the gateway and cinder-volume")
powerflexgw_vm.restart_gateway()
powerflexgw_vm.restart_cinder_volume()
def deploy_undercloud(setts, sah_node, tester, director_vm):
director_ip = setts.director_node.public_api_ip
Ssh.execute_command(director_ip, "root", setts.director_node.root_password,
"subscription-manager remove --all")
Ssh.execute_command(director_ip, "root", setts.director_node.root_password,
"subscription-manager unregister")
sah_node.delete_director_vm()
logger.info("=== create the director vm")
sah_node.create_director_vm()
tester.director_vm_health_check()
logger.info("Preparing the Director VM")
director_vm.apply_internal_repos()
logger.debug("=== Uploading & configuring undercloud.conf . "
"environment yaml ===")
director_vm.upload_update_conf_files()
logger.info("=== installing the director & undercloud ===")
director_vm.inject_ssh_key()
director_vm.upload_cloud_images()
director_vm.install_director()
_is_failed, _error = tester.verify_undercloud_installed()
if _is_failed:
raise _error
def cleanup_sah(self):
logger.info("- Clean up any existing installation ")
cmds = [
' killall -u core', 'userdel -r core',
'rm -rf /var/lib/tftpboot/uefi/*'
]
for cmd in cmds:
Ssh.execute_command("localhost", "root",
self.settings.csah_root_pwd, cmd)
def deploy_powerflex_mgmt(setts, sah_node, tester, powerflexmgmt_vm):
powerflexmgmt_ip = setts.powerflexmgmt_vm.public_api_ip
Ssh.execute_command(powerflexmgmt_ip, "root",
setts.powerflexmgmt_vm.root_password,
"subscription-manager remove --all")
Ssh.execute_command(powerflexmgmt_ip, "root",
setts.powerflexmgmt_vm.root_password,
"subscription-manager unregister")
logger.info("=== deleting any existing powerflex presentation server vm")
sah_node.delete_powerflexmgmt_vm()
logger.info("=== Creating the powerflex presentation server vm")
sah_node.create_powerflexmgmt_vm()
tester.powerflexmgmt_vm_health_check()
logger.info("Installing the powerflex presentation server UI")
powerflexmgmt_vm.upload_rpm()
powerflexmgmt_vm.install_presentation_server()
def verify_subscription_status(public_api_ip, user, password, retries):
i = 0
subscription_status = Ssh.execute_command(
public_api_ip, user, password, "subscription-manager status")[0]
while "Current" not in subscription_status and i < retries:
if "Unknown" in subscription_status:
return subscription_status
time.sleep(60)
subscription_status = \
Ssh.execute_command(public_api_ip,
user,
password,
"subscription-manager status")[0]
i += 1
return subscription_status
def run_playbooks(self):
logger.info("- Run ansible playbook to generate ignition files etc")
logfile = logger.handlers[0].baseFilename
cmd = 'export ANSIBLE_LOG_PATH=' + logfile + '; ansible-playbook -i generated_inventory haocp.yaml'
subprocess.call(cmd,
shell=True,
cwd='/home/ansible/JetPack/src/pilot/ansible')
logger.info("Updating the dns settings")
cmds = [
'nmcli connection modify br0 ipv4.dns ' +
self.settings.csah_node.os_ip,
'sed -i "s/nameserver.*/nameserver ' +
self.settings.csah_node.os_ip + '/" /etc/resolv.conf'
]
for cmd in cmds:
Ssh.execute_command("localhost", "root",
self.settings.csah_root_pwd, cmd)
def create_bootstrap_vm(self):
logger.info("- Create the bootstrap VM")
bootstrap_mac = self.get_inventory(
)['all']['vars']['bootstrap_node'][0]['mac']
cmd = 'virt-install --name bootstrapkvm --ram 20480 --vcpu 8 --disk path=/home/bootstrapvm-disk.qcow2,format=qcow2,size=20 --os-variant generic --network=bridge=br0,model=virtio,mac=' + bootstrap_mac + ' --pxe --boot uefi,hd,network --noautoconsole --autostart &'
re = Ssh.execute_command("localhost", "root",
self.settings.csah_root_pwd, cmd)
time.sleep(320)
def sah_health_check(self):
logger.info("SAH node health check")
if self.verify_rhsm_status:
logger.debug("*** Verify the SAH node registered properly ***")
for _ in range(60):
subscription_status = self.verify_subscription_status(
self.sah_ip, "root", self.settings.sah_node.root_password,
self.settings.subscription_check_retries)
if "Current" in subscription_status:
break
time.sleep(2)
else:
raise AssertionError("SAH did not register properly : " +
subscription_status)
logger.debug("*** Verify the SAH can ping its public gateway")
gateway = self.settings.public_api_gateway
test = self.ping_host(self.sah_ip, "root",
self.settings.sah_node.root_password, gateway)
if self.ping_success not in test:
raise AssertionError("SAH cannot ping its public gateway : " +
test)
logger.debug("*** Verify the SAH can ping the outside world (ip)")
test = self.ping_host(self.sah_ip, "root",
self.settings.sah_node.root_password, "8.8.8.8")
if self.ping_success not in test:
raise AssertionError("SAH cannot ping the outside world (ip) : " +
test)
logger.debug("*** Verify the SAH can ping the outside world (dns)")
test = self.ping_host(self.sah_ip, "root",
self.settings.sah_node.root_password,
"google.com")
if self.ping_success not in test:
raise AssertionError("SAH cannot ping the outside world (dns) : " +
test)
logger.debug("*** Verify the SAH can ping the idrac network")
test = self.ping_host(self.sah_ip, "root",
self.settings.sah_node.root_password,
self.settings.sah_node.idrac_ip)
if self.ping_success not in test:
raise AssertionError("SAH cannot ping idrac networkn (ip) : " +
test)
logger.debug("*** Verify the SAH has KVM enabled *** ")
cmd = 'ls -al /dev/kvm'
if "No such file" in \
Ssh.execute_command(self.sah_ip,
"root",
self.settings.sah_node.root_password,
cmd)[1]:
raise AssertionError(
"KVM Not running on the SAH node - make sure "
"the node has been DTK'ed/Virtualization enabled "
"in the Bios")
def wait_for_vm_to_come_up(self, target_ip, user, password):
while True:
status = Ssh.execute_command(target_ip, user, password, "ps")[0]
if status != "host not up":
break
logger.debug("vm is not up. Sleeping...")
time.sleep(10)
def ping_host(self, external_ip, user, passwd, target_host):
for i in range(1, 30):
ping_status = Ssh.execute_command(
external_ip, user, passwd,
"ping " + target_host + " -c 1 -w 30 ")[0]
if self.ping_success in ping_status:
logger.debug("Ping {} successful on attempt #{}".format(
target_host, i))
break
# noinspection PyUnboundLocalVariable
return ping_status
def delete_bootstrap_vm(self):
logger.info(" Destroy any existing bootstrap Vm")
cmd = 'virsh list --all'
bBoostrapDestroyed = False
while bBoostrapDestroyed is False:
re = Ssh.execute_command("localhost",
"root",
self.settings.csah_root_pwd,
cmd)
if 'bootstrap' in str(re):
cmds = [
'virsh undefine --nvram "bootstrapkvm"',
'virsh destroy bootstrapkvm']
for cm in cmds:
Ssh.execute_command("localhost",
"root",
self.settings.csah_root_pwd,
cm)
else:
bBoostrapDestroyed = True
def wait_for_vm_to_go_down(self, target_ip, user, password):
while True:
status = Ssh.execute_command(
target_ip,
user,
password,
"ps")[0]
if status == "host not up":
break
time.sleep(5)
def verify_pools_attached(ip_addr, user, password, logfile):
# check the xxxxx-posts.log for pool id's/repo's related errors.
log_out = \
Ssh.execute_command(ip_addr, user, password, "cat " + logfile)[0]
error1 = 'No subscriptions are available from the pool with'
error2 = 'Removed temporarly as this error will show when ' \
'not pulling from the cdn but internal repos'
error3 = 'Could not find an OpenStack pool to attach to'
if error1 in log_out or error2 in log_out or error3 in log_out:
logger.info("*** post install log ***")
logger.info(log_out)
return False
return True
def wait_for_bootstrap_ready(self):
bBootstrap_ready = False
while bBootstrap_ready is False:
cmd = 'sudo su - core -c \'ssh -o "StrictHostKeyChecking no " bootstrap sudo ss -tulpn | grep -E "6443|22623|2379"\''
openedPorts = Ssh.execute_command_tty("localhost", "root",
self.settings.csah_root_pwd,
cmd)
if ("22623" in str(openedPorts)) and (
"2379" in str(openedPorts)) and ("6443"
in str(openedPorts)):
logger.info(" ,, boostrap UP! ")
bBootstrap_ready = True
re = Ssh.execute_command("localhost", "root",
self.settings.csah_root_pwd,
"virsh list --all | grep bootstrapkvm")[0]
if "shut off" in re:
bPXe_complete = True
logger.info("- Powering on the bootstrap VM")
Ssh.execute_command("localhost", "root",
self.settings.csah_root_pwd,
"virsh start bootstrapkvm")
time.sleep(60)
logger.info("- Bootstrap VM is ready")
def provisioning_subnet_exists(self, subnet):
logger.debug("Check if edge subnet {} already "
"exists or not".format(subnet))
setts = self.settings
user = setts.director_install_account_user
ip = setts.director_node.public_api_ip
pwd = setts.director_install_account_pwd
is_subnet = False
subnet_cmd = ("{} openstack subnet "
"show {} -c name "
"-f value".format(self.source_stackrc, subnet))
sn_out = Ssh.execute_command(ip, user, pwd, subnet_cmd)[0]
if sn_out.strip() == subnet:
is_subnet = True
logger.info("Subnet {} already exists".format(subnet))
return is_subnet
def deploy():
ret_code = 0
# noinspection PyBroadException
logger.debug("=================================")
logger.info("=== Starting up ...")
logger.debug("=================================")
try:
settings, args = get_settings()
if args.validate_only is True:
logger.info("Only validating ini/properties config values")
else:
if args.overcloud_only is True:
logger.info("Only redeploying the overcloud")
if args.skip_dashboard_vm is True:
logger.info("Skipping Dashboard VM install")
logger.info("Settings .ini: " + settings.settings_file)
logger.info("Settings .properties " + settings.network_conf)
settings.get_version_info()
logger.info("source version # : " + settings.source_version)
tester = Checkpoints()
tester.verify_deployer_settings()
if args.validate_only is True:
logger.info("Settings validated")
os._exit(0)
tester.retreive_switches_config()
non_sah_nodes = (settings.controller_nodes + settings.compute_nodes +
settings.ceph_nodes)
sah_node = Sah()
tester.sah_health_check()
# mutually exclusive command, configure tempest and quit.
if args.tempest_config_only:
logger.info("Only (re-)generating tempest.conf")
director_vm = Director()
director_vm.configure_tempest()
os._exit(0)
# mutually exclusive command, run tempest and quit.
if args.run_tempest_only:
logger.info("Only running tempest, will configure " +
"tempest.conf if needed.")
director_vm = Director()
director_vm.run_tempest()
os._exit(0)
logger.info("Uploading configs/iso/scripts.")
sah_node.clear_known_hosts()
sah_node.handle_lock_files()
sah_node.upload_iso()
sah_node.upload_director_scripts()
director_ip = settings.director_node.public_api_ip
if args.overcloud_only is False:
Ssh.execute_command(director_ip, "root",
settings.director_node.root_password,
"subscription-manager remove --all")
Ssh.execute_command(director_ip, "root",
settings.director_node.root_password,
"subscription-manager unregister")
sah_node.delete_director_vm()
logger.info("=== create the director vm")
sah_node.create_director_vm()
tester.director_vm_health_check()
logger.info("Preparing the Director VM")
director_vm = Director()
director_vm.apply_internal_repos()
logger.debug("=== Uploading & configuring undercloud.conf . "
"environment yaml ===")
director_vm.upload_update_conf_files()
logger.info("=== installing the director & undercloud ===")
director_vm.inject_ssh_key()
director_vm.upload_cloud_images()
director_vm.install_director()
tester.verify_undercloud_installed()
if args.undercloud_only:
return
else:
logger.info("=== Skipped Director VM/Undercloud install")
director_vm = Director()
logger.debug("Deleting overcloud stack")
director_vm.delete_overcloud()
if args.skip_dashboard_vm is False:
logger.debug("Delete the Dashboard VM")
dashboard_ip = settings.dashboard_node.public_api_ip
logger.debug(
Ssh.execute_command(dashboard_ip, "root",
settings.dashboard_node.root_password,
"subscription-manager remove --all"))
Ssh.execute_command(dashboard_ip, "root",
settings.dashboard_node.root_password,
"subscription-manager unregister")
sah_node.delete_dashboard_vm()
logger.info("=== creating Dashboard VM")
sah_node.create_dashboard_vm()
tester.dashboard_vm_health_check()
else:
logger.info("Skipped the Dashboard VM install")
logger.info("=== Preparing the overcloud ===")
# The network-environment.yaml must be setup for use during DHCP
# server configuration
director_vm.setup_net_envt()
director_vm.configure_dhcp_server()
director_vm.node_discovery()
director_vm.configure_idracs()
director_vm.import_nodes()
director_vm.node_introspection()
director_vm.update_sshd_conf()
director_vm.assign_node_roles()
director_vm.revert_sshd_conf()
director_vm.setup_templates()
logger.info("=== Installing the overcloud ")
logger.debug("installing the overcloud ... this might take a while")
director_vm.deploy_overcloud()
cmd = "source ~/stackrc; openstack stack list | grep " \
+ settings.overcloud_name + " | awk '{print $8}'"
overcloud_status = \
Ssh.execute_command_tty(director_ip,
settings.director_install_account_user,
settings.director_install_account_pwd,
cmd)[0]
logger.debug("=== Overcloud stack state : " + overcloud_status)
if settings.hpg_enable:
logger.info(
" HugePages has been successfully configured with size: " +
settings.hpg_size)
if settings.numa_enable:
logger.info(" NUMA has been successfully configured"
" with hostos_cpus count: " +
settings.hostos_cpu_count)
logger.info("====================================")
logger.info(" OverCloud deployment status: " + overcloud_status)
logger.info(" log : /auto_results/ ")
logger.info("====================================")
if "CREATE_COMPLETE" not in overcloud_status:
raise AssertionError("OverCloud did not install properly : " +
overcloud_status)
director_vm.summarize_deployment()
tester.verify_computes_virtualization_enabled()
tester.verify_backends_connectivity()
if args.skip_dashboard_vm is False:
director_vm.configure_dashboard()
director_vm.enable_fencing()
director_vm.run_sanity_test()
external_sub_guid = director_vm.get_sanity_subnet()
if external_sub_guid:
director_vm.configure_tempest()
run_tempest()
logger.info("Deployment summary info; useful ip's etc.. " +
"/auto_results/deployment_summary.log")
except: # noqa: E722
logger.error(traceback.format_exc())
e = sys.exc_info()[0]
logger.error(e)
print e
print traceback.format_exc()
ret_code = 1
logger.info("log : /auto_results/ ")
sys.exit(ret_code)
def deploy():
ret_code = 0
# noinspection PyBroadException
try:
logger.debug("=================================")
logger.info("=== Starting up ...")
logger.debug("=================================")
parser = argparse.ArgumentParser(
description='JetPack 10.x deployer')
parser.add_argument('-s', '--settings',
help='ini settings file, e.g settings/acme.ini',
required=True)
parser.add_argument('-undercloud_only', '--undercloud_only',
help='Only reinstall the undercloud',
action='store_true', required=False)
parser.add_argument('-overcloud_only', '--overcloud_only',
help='Only reinstall the overcloud',
action='store_true', required=False)
parser.add_argument('-skip_dashboard_vm', '--skip_dashboard_vm',
help='Do not reinstall the Dashboard VM',
action='store_true',
required=False)
parser.add_argument('-validate_only', '--validate_only',
help='No deployment - just validate config values',
action='store_true',
required=False)
args, others = parser.parse_known_args()
if len(others) > 0:
parser.print_help()
msg = "Invalid argument(s) :"
for each in others:
msg += " " + each + ";"
raise AssertionError(msg)
if args.validate_only is True:
logger.info("Only validating ini/properties config values")
else:
if args.overcloud_only is True:
logger.info("Only redeploying the overcloud")
if args.skip_dashboard_vm is True:
logger.info("Skipping Dashboard VM install")
logger.debug("loading settings files " + args.settings)
settings = Settings(args.settings)
logger.info("Settings .ini: " + settings.settings_file)
logger.info("Settings .properties " + settings.network_conf)
settings.get_version_info()
logger.info("source version # : " + settings.source_version)
tester = Checkpoints()
tester.verify_deployer_settings()
if args.validate_only is True:
logger.info("Settings validated")
os._exit(0)
if settings.retreive_switches_config is True:
tester.retreive_switches_config()
non_sah_nodes = (settings.controller_nodes +
settings.compute_nodes +
settings.ceph_nodes)
sah_node = Sah()
tester.sah_health_check()
logger.info("Uploading configs/iso/scripts..")
sah_node.clear_known_hosts()
sah_node.handle_lock_files()
sah_node.upload_iso()
sah_node.upload_director_scripts()
director_ip = settings.director_node.public_api_ip
if args.overcloud_only is False:
Ssh.execute_command(director_ip,
"root",
settings.director_node.root_password,
"subscription-manager remove --all")
Ssh.execute_command(director_ip,
"root",
settings.director_node.root_password,
"subscription-manager unregister")
sah_node.delete_director_vm()
logger.info("=== create the director vm")
sah_node.create_director_vm()
tester.director_vm_health_check()
logger.info("Preparing the Director VM")
director_vm = Director()
director_vm.apply_internal_repos()
logger.debug(
"=== Uploading & configuring undercloud.conf . "
"environment yaml ===")
director_vm.upload_update_conf_files()
logger.info("=== installing the director & undercloud ===")
director_vm.inject_ssh_key()
director_vm.upload_cloud_images()
director_vm.install_director()
tester.verify_undercloud_installed()
if args.undercloud_only:
return
else:
logger.info("=== Skipped Director VM/Undercloud install")
director_vm = Director()
logger.debug("Deleting overcloud stack")
director_vm.delete_overcloud()
if args.skip_dashboard_vm is False:
logger.debug("Delete the Dashboard VM")
dashboard_ip = settings.dashboard_node.public_api_ip
logger.debug(
Ssh.execute_command(dashboard_ip,
"root",
settings.dashboard_node.root_password,
"subscription-manager remove --all"))
Ssh.execute_command(dashboard_ip,
"root",
settings.dashboard_node.root_password,
"subscription-manager unregister")
sah_node.delete_dashboard_vm()
logger.info("=== creating Dashboard VM")
sah_node.create_dashboard_vm()
tester.dashboard_vm_health_check()
else:
logger.info("Skipped the Dashboard VM install")
logger.info("=== Preparing the overcloud ===")
# The network-environment.yaml must be setup for use during DHCP
# server configuration
director_vm.setup_net_envt()
director_vm.configure_dhcp_server()
director_vm.node_discovery()
director_vm.configure_idracs()
director_vm.import_nodes()
director_vm.node_introspection()
director_vm.update_sshd_conf()
director_vm.assign_node_roles()
director_vm.revert_sshd_conf
director_vm.setup_templates()
logger.info("=== Installing the overcloud ")
logger.debug("installing the overcloud ... this might take a while")
director_vm.deploy_overcloud()
cmd = "source ~/stackrc; openstack stack list | grep " \
+ settings.overcloud_name + " | awk '{print $6}'"
overcloud_status = \
Ssh.execute_command_tty(director_ip,
settings.director_install_account_user,
settings.director_install_account_pwd,
cmd)[0]
logger.debug("=== Overcloud stack state : " + overcloud_status)
if settings.hpg_enable:
logger.info(
" HugePages has been successfully configured with size: " +
settings.hpg_size)
if settings.numa_enable:
logger.info(
" NUMA has been successfully configured"
" with hostos_cpus count: " +
settings.hostos_cpu_count)
logger.info("====================================")
logger.info(" OverCloud deployment status: " + overcloud_status)
logger.info(" log : /auto_results/ ")
logger.info("====================================")
if "CREATE_COMPLETE" not in overcloud_status:
raise AssertionError(
"OverCloud did not install properly : " + overcloud_status)
director_vm.summarize_deployment()
tester.verify_computes_virtualization_enabled()
tester.verify_backends_connectivity()
if args.skip_dashboard_vm is False:
director_vm.configure_dashboard()
director_vm.enable_fencing()
director_vm.enable_instance_ha()
director_vm.configure_tempest()
director_vm.run_sanity_test()
run_tempest()
logger.info("Deployment summary info; useful ip's etc.. " +
"/auto_results/deployment_summary.log")
except:
logger.error(traceback.format_exc())
e = sys.exc_info()[0]
logger.error(e)
print e
print traceback.format_exc()
ret_code = 1
logger.info("log : /auto_results/ ")
sys.exit(ret_code)
def run_as_root(self, command):
return Ssh.execute_command(self.ip,
"root",
self.root_pwd,
command)
def run(self, command):
return Ssh.execute_command(self.ip,
self.user,
self.pwd,
command)
