def test():
global vm
global vip_s_vm_cfg_lst
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
#test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)
test_stub.exec_zsha2_demote(vip_s_vm_cfg_lst[0].ip_, "root", "password")
time.sleep(5)
expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("close MN-VM and all zs-ha services in cluster")
test_stub.stop_zsha(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) != 0:
test_util.test_fail('MN VM is still running after shutdown ha cluster')
test_util.test_logger("start ha cluster")
test_stub.start_zsha(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait 50s for MN VM to start")
time.sleep(50)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail(
'MN VM is running on %d host(s) after shutdown and start ha cluster'
% len(mn_host))
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_util.test_pass('Shutdown HA Cluster Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_is_multiple_networks()
for i in range(0, 10):
test_util.test_logger("shutdown host network round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
else:
test_util.test_logger("management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_ or new_mn_host_ip == mn_host[0].managementIp_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global ori_ip
global mn_vm_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger("change mn vm ip in config file of host [%s]" %
(mn_vm_host[0].ip_))
new_mn_vm_ip = "172.20.199.99"
ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP')
test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip,
test_lib.all_scenario_config)
test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config)
test_util.test_logger(
"wait for 40 seconds to see if management node VM starts on another host"
)
time.sleep(40)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not start after change ip and destroyed"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM starts on more than one host after change ip and destroyed"
)
except:
test_util.test_fail(
"management node VM does not start after change ip and destroyed")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail(
'management node VM starts on another host after change ip and destroyed'
)
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global ha_vm
global mn_host
ha_vm = test_stub.create_ha_vm()
ha_vm.check()
ha_vm_host = test_lib.lib_get_vm_host(ha_vm.vm)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
if ha_vm_host.managementIp != mn_host[0].ip_:
conditions = res_ops.gen_query_conditions('managementIp', '=', mn_host[0].ip_)
host = res_ops.query_resource(res_ops.HOST, conditions)
ha_vm_host.migrate(host[0].uuid)
test_util.test_logger("force shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold')
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
conditions = res_ops.gen_query_conditions('managementIp', '=', mn_host[0].ip_)
host = res_ops.query_resource(res_ops.HOST, conditions)
if host[0].status != "Connected":
test_util.test_fail("Target host:%s is not connected as expected." %(host[0].uuid))
ha_vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail("management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":# or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
#test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)
buildtype = test_stub.get_buildtype_by_sce_file(test_lib.scenario_file)
buildid = test_stub.get_buildid_by_sce_file(test_lib.scenario_file)
build_server = os.environ.get('BUILD_SERVER')
cmd = "cd /root/mirror/" + buildtype+ "/" + buildid + "/ && ls ZStack*-installer-*.bin"
zstack_bin_name = test_lib.lib_execute_ssh_cmd(build_server, "root", "password", cmd).replace("\n", "")
zstack_bin_path = "/root/" + zstack_bin_name
cmd = "wget -c http://" + build_server + "/mirror/" + buildtype + "/" + buildid + "/" + zstack_bin_name + " -O " + zstack_bin_path
test_lib.lib_execute_ssh_cmd(vip_s_vm_cfg_lst[0].ip_, "root", "password", cmd)
test_stub.exec_upgrade_mn(vip_s_vm_cfg_lst[0].ip_, "root", "password", zstack_bin_path)
print ":%s:" %(cmd)
time.sleep(5)
#expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
#if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
# test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("close MN-VM and all zs-ha services in cluster")
test_stub.stop_zsha(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) != 0:
test_util.test_fail('MN VM is still running after shutdown ha cluster')
test_util.test_logger("start ha cluster")
test_stub.start_zsha(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait 50s for MN VM to start")
time.sleep(50)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s) after shutdown and start ha cluster' % len(mn_host))
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_util.test_pass('Shutdown HA Cluster Test Success')
def test():
global vm
global ori_ip
global mn_vm_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_))
new_mn_vm_ip = "172.20.199.99"
ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP')
test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config)
test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 40 seconds to see if management node VM starts on another host")
time.sleep(40)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not start after change ip and destroyed")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after change ip and destroyed")
except:
test_util.test_fail("management node VM does not start after change ip and destroyed")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host after change ip and destroyed')
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
global s_vm0
global s_vm1
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
s_vm0 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0)
s_vm1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1)
test_util.test_logger("force stop both hosts [%s] and [%s]" % (s_vm0.ip_, s_vm1.ip_))
test_stub.stop_host(s_vm0, test_lib.all_scenario_config, 'cold')
test_stub.stop_host(s_vm1, test_lib.all_scenario_config, 'cold')
time.sleep(5)
test_stub.start_host(s_vm0, test_lib.all_scenario_config)
test_stub.start_host(s_vm1, test_lib.all_scenario_config)
test_stub.recover_vlan_in_host(s_vm0.ip_, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.recover_vlan_in_host(s_vm1.ip_, test_lib.all_scenario_config, test_lib.deploy_config)
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file, 3)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
test_util.test_skip("2 hosts down at the same time is not support")
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("force stop host [%s]" %
(mn_host_list[index].ip_))
test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config,
'cold')
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
test_util.test_logger("recover host [%s]" %
(mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]],
test_lib.all_scenario_config,
test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file,
test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
test_util.test_skip("2 hosts down at the same time is not support")
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (mn_host_list[index].ip_))
test_stub.shutdown_host_network(mn_host_list[index], test_lib.all_scenario_config)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
if test_lib.scenario_config == None or test_lib.scenario_file == None:
test_util.test_fail('Suite Setup Fail without scenario')
if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists(
test_lib.scenario_file):
scenario_operations.deploy_scenario(test_lib.all_scenario_config,
test_lib.scenario_file,
test_lib.deploy_config)
test_util.test_skip('Suite Setup Success')
if test_lib.scenario_config != None and test_lib.scenario_destroy != None:
scenario_operations.destroy_scenario(test_lib.all_scenario_config,
test_lib.scenario_destroy)
nic_name = "eth0"
if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(
test_lib.scenario_file):
nic_name = "zsn0"
#This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine.
#no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases.
linux.create_vlan_eth(nic_name, 10)
linux.create_vlan_eth(nic_name, 11)
#If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run()
test_lib.setup_plan.deploy_test_agent()
cmd = host_plugin.CreateVlanDeviceCmd()
cmd.ethname = nic_name
cmd.vlan = 10
cmd2 = host_plugin.CreateVlanDeviceCmd()
cmd2.ethname = nic_name
cmd2.vlan = 11
testHosts = test_lib.lib_get_all_hosts_from_plan()
if type(testHosts) != type([]):
testHosts = [testHosts]
for host in testHosts:
http.json_dump_post(
testagent.build_http_path(host.managementIp_,
host_plugin.CREATE_VLAN_DEVICE_PATH),
cmd)
http.json_dump_post(
testagent.build_http_path(host.managementIp_,
host_plugin.CREATE_VLAN_DEVICE_PATH),
cmd2)
if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-nfs.xml"], \
["scenario-config-storage-separate-nfs.xml"]):
config_json = os.environ.get('configJsonSepStor')
elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-nonmon-ceph.xml"], \
["scenario-config-storage-separate-ceph.xml"]):
config_json = os.environ.get('configJsonNonMon')
elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-sep-pub-man.xml"], \
["scenario-config-nfs-sep-man.xml", \
"scenario-config-nfs-sep-pub.xml"]):
config_json = os.environ.get('configJsonSepPub')
elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-ceph-3-nets-sep.xml"], \
["scenario-config-ceph-sep-man.xml", \
"scenario-config-ceph-sep-pub.xml", \
"scenario-config-ceph-3-nets-sep.xml"]):
config_json = os.environ.get('configJsonSepPub')
elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-fusionstor-3-nets-sep.xml"], \
["scenario-config-fusionstor-3-nets-sep.xml"]):
config_json = os.environ.get('configJson3Net')
elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-mul-net-pubs.xml"], \
["scenario-config-nfs-sep-man.xml", \
"scenario-config-nfs-sep-pub.xml"]):
config_json = os.environ.get('configJsonAllOne')
else:
config_json = os.environ.get('configJson')
ha_deploy_tool = os.environ.get('zstackHaInstaller')
mn_img = os.environ.get('mnImage')
test_stub.deploy_ha_env(test_lib.all_scenario_config,
test_lib.scenario_file, test_lib.deploy_config,
config_json, ha_deploy_tool, mn_img)
#if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
# test_util.test_logger("@@@DEBUG->IS VPC CEPH@@@")
# old_mn_ip = os.environ['zstackHaVip']
# test_stub.auto_set_mn_ip(test_lib.scenario_file)
# cmd = 'sed -i "s/%s/%s/g" %s' %(old_mn_ip, os.environ['zstackHaVip'], EXTRA_SUITE_SETUP_SCRIPT)
# os.system(cmd)
#node_operations.wait_for_management_server_start(600)
test_stub.wrapper_of_wait_for_management_server_start(
600, EXTRA_SUITE_SETUP_SCRIPT)
test_util.test_logger("@@@DEBUG->suite_setup@@@ os\.environ\[\'ZSTACK_BUILT_IN_HTTP_SERVER_IP\'\]=%s; os\.environ\[\'zstackHaVip\'\]=%s" \
%(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], os.environ['zstackHaVip']) )
ssh.scp_file("/home/license-10host-10days-hp.txt",
"/home/license-10host-10days-hp.txt",
os.environ.get('zstackHaVip'), 'root', 'password')
if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT):
os.system("bash %s" % EXTRA_SUITE_SETUP_SCRIPT)
deploy_operations.deploy_initial_database(test_lib.deploy_config,
test_lib.all_scenario_config,
test_lib.scenario_file)
for host in testHosts:
os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_))
#if test_lib.lib_get_ha_selffencer_maxattempts() != None:
# test_lib.lib_set_ha_selffencer_maxattempts('60')
# test_lib.lib_set_ha_selffencer_storagechecker_timeout('60')
test_lib.lib_set_primary_storage_imagecache_gc_interval(1)
test_lib.lib_set_reserved_memory('8G')
if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-sep-pub-man.xml"], ["scenario-config-nfs-sep-pub.xml"]) or \
test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-ceph-3-nets-sep.xml"], ["scenario-config-ceph-sep-pub.xml"]) or \
test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-fusionstor-3-nets-sep.xml"], ["scenario-config-fusionstor-3-nets-sep.xml"]):
add_ps_network_gateway_sys_tag()
test_util.test_pass('Suite Setup Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on one host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
if os.path.basename(
os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip(
) == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global ha_vm
global mn_host
ha_vm = test_stub.create_ha_vm()
ha_vm.check()
ha_vm_host = test_lib.lib_get_vm_host(ha_vm.vm)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
if ha_vm_host.managementIp != mn_host[0].ip_:
conditions = res_ops.gen_query_conditions('managementIp', '=',
mn_host[0].ip_)
host = res_ops.query_resource(res_ops.HOST, conditions)
ha_vm_host.migrate(host[0].uuid)
test_util.test_logger("force shutdown host [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold')
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
conditions = res_ops.gen_query_conditions('managementIp', '=',
mn_host[0].ip_)
host = res_ops.query_resource(res_ops.HOST, conditions)
if host[0].status != "Connected":
test_util.test_fail("Target host:%s is not connected as expected." %
(host[0].uuid))
ha_vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
ha_vm = test_stub.create_ha_vm()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("force stop host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config, 'cold')
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.return_pass_ahead_if_3sites("TEST PASS")
ha_vm.set_state(vm_header.RUNNING)
ha_vm.check()
test_util.test_logger("try to create vm, timeout is 30s")
time_out = 30
while time_out > 0:
try:
vm = test_stub.create_basic_vm()
break
except:
time.sleep(1)
time_out -= 1
if time_out == 0:
test_util.test_fail('Fail to create vm after mn is ready')
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) == 0:
test_util.test_fail('MN VM has been murdered, expected result should not be impacted when the separated network is down.')
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config,
downMagt=False)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[
0].ip_ or new_mn_host_ip == mn_host[0].managementIp_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 120s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index],
test_lib.all_scenario_config,
test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(
exclude_host=[mn_host_list[need_recover_mn_host_list[0]]])
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("force shutdown host round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger(
"force shutdown host [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold')
test_util.test_logger(
"wait for 40 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(
test_lib.scenario_file, test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
if os.path.basename(
os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip(
) == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.recover_host(mn_host[0], test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.return_skip_ahead_if_3sites(
"Skip the test in vpc ceph because create vm with vyos is not supported."
)
test_stub.skip_if_vr_not_vyos("vr")
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
vm = test_stub.create_basic_vm()
vm.check()
ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop")
vm.set_state(vm_header.RUNNING)
vm.check()
for host in mn_host_list:
test_util.test_logger("shutdown host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index],
test_lib.all_scenario_config,
test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.return_pass_ahead_if_3sites("TEST PASS")
test_util.test_logger("Delay 60s and then check if the vm is running")
time.sleep(180)
if test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 300):
vm.update()
vm.check()
vm.destroy()
else:
test_util.test_fail(
"ha vm has not changed to running after 2 hosts recover with 300s")
test_util.test_pass('Check Never Stop VM Test Success')
def test():
if test_lib.scenario_config == None or test_lib.scenario_file ==None:
test_util.test_fail('Suite Setup Fail without scenario')
if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists(test_lib.scenario_file):
scenario_operations.deploy_scenario(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config)
test_util.test_skip('Suite Setup Success')
if test_lib.scenario_config != None and test_lib.scenario_destroy != None:
scenario_operations.destroy_scenario(test_lib.all_scenario_config, test_lib.scenario_destroy)
nic_name = "eth0"
if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file):
nic_name = "zsn0"
#This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine.
#no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases.
linux.create_vlan_eth(nic_name, 10)
linux.create_vlan_eth(nic_name, 11)
#If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run()
test_lib.setup_plan.deploy_test_agent()
cmd = host_plugin.CreateVlanDeviceCmd()
cmd.ethname = nic_name
cmd.vlan = 10
cmd2 = host_plugin.CreateVlanDeviceCmd()
cmd2.ethname = nic_name
cmd2.vlan = 11
testHosts = test_lib.lib_get_all_hosts_from_plan()
if type(testHosts) != type([]):
testHosts = [testHosts]
for host in testHosts:
http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd)
http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd2)
test_stub.deploy_2ha(test_lib.all_scenario_config, test_lib.scenario_file)
mn_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0).ip_
mn_ip2 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1).ip_
host_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 2).ip_
test_stub.recover_vlan_in_host(host_ip1, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.wrapper_of_wait_for_management_server_start(600, EXTRA_SUITE_SETUP_SCRIPT)
test_util.test_logger("@@@DEBUG->suite_setup@@@ os\.environ\[\'ZSTACK_BUILT_IN_HTTP_SERVER_IP\'\]=%s; os\.environ\[\'zstackHaVip\'\]=%s" \
%(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], os.environ['zstackHaVip']) )
ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip1, 'root', 'password')
ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip2, 'root', 'password')
if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT):
os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip1))
os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip2))
deploy_operations.deploy_initial_database(test_lib.deploy_config, test_lib.all_scenario_config, test_lib.scenario_file)
for host in testHosts:
os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_))
test_lib.lib_set_primary_storage_imagecache_gc_interval(1)
#test_lib.lib_set_reserved_memory('1G')
test_util.test_pass('Suite Setup Success')
def test():
if test_lib.scenario_config == None or test_lib.scenario_file ==None:
test_util.test_fail('Suite Setup Fail without scenario')
if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists(test_lib.scenario_file):
scenario_operations.deploy_scenario(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config)
test_util.test_skip('Suite Setup Success')
if test_lib.scenario_config != None and test_lib.scenario_destroy != None:
scenario_operations.destroy_scenario(test_lib.all_scenario_config, test_lib.scenario_destroy)
nic_name = "eth0"
if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file):
nic_name = "zsn0"
#This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine.
#no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases.
linux.create_vlan_eth(nic_name, 10)
linux.create_vlan_eth(nic_name, 11)
#If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run()
test_lib.setup_plan.deploy_test_agent()
cmd = host_plugin.CreateVlanDeviceCmd()
cmd.ethname = nic_name
cmd.vlan = 10
cmd2 = host_plugin.CreateVlanDeviceCmd()
cmd2.ethname = nic_name
cmd2.vlan = 11
testHosts = test_lib.lib_get_all_hosts_from_plan()
if type(testHosts) != type([]):
testHosts = [testHosts]
for host in testHosts:
http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd)
http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd2)
test_stub.deploy_2ha(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config)
mn_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0).ip_
mn_ip2 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1).ip_
if not xmlobject.has_element(test_lib.deploy_config, 'backupStorages.miniBackupStorage'):
host_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 2).ip_
test_stub.recover_vlan_in_host(host_ip1, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.wrapper_of_wait_for_management_server_start(600, EXTRA_SUITE_SETUP_SCRIPT)
test_util.test_logger("@@@DEBUG->suite_setup@@@ os\.environ\[\'ZSTACK_BUILT_IN_HTTP_SERVER_IP\'\]=%s; os\.environ\[\'zstackHaVip\'\]=%s" \
%(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], os.environ['zstackHaVip']) )
ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip1, 'root', 'password')
ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip2, 'root', 'password')
if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT):
os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip1))
os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip2))
deploy_operations.deploy_initial_database(test_lib.deploy_config, test_lib.all_scenario_config, test_lib.scenario_file)
for host in testHosts:
os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_))
test_lib.lib_set_primary_storage_imagecache_gc_interval(1)
#test_lib.lib_set_reserved_memory('1G')
if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-local-ps.xml"], ["scenario-config-upgrade-3.1.1.xml"]):
cmd = r"sed -i '$a\172.20.198.8 rsync.repo.zstack.io' /etc/hosts"
ssh.execute(cmd, mn_ip1, "root", "password", False, 22)
ssh.execute(cmd, mn_ip2, "root", "password", False, 22)
test_util.test_pass('Suite Setup Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config,
downMagt=True)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and
new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail(
"management network down, mn host should not changed. Expected on [%s] while is on [%s]"
% (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 120s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
