def test():
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if not test_stub.upgrade_zsha(test_lib.all_scenario_config, test_lib.scenario_file):
test_util.test_fail('Fail to upgrade zsha')
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger('wait for 10s to see if something happens')
time.sleep(10)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM was destroyed after upgrade zsha")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after upgrade zsha")
except:
test_util.test_fail("management node VM was destroyed after upgrade zsha")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host after upgrade zsha')
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
#test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)
test_stub.exec_zsha2_demote(vip_s_vm_cfg_lst[0].ip_, "root", "password")
time.sleep(5)
expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if not test_stub.upgrade_zsha(test_lib.all_scenario_config, test_lib.scenario_file):
test_util.test_fail('Fail to upgrade zsha')
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger('wait for 10s to see if something happens')
time.sleep(10)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM was destroyed after upgrade zsha")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after upgrade zsha")
except:
test_util.test_fail("management node VM was destroyed after upgrade zsha")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host after upgrade zsha')
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_is_multiple_networks()
for i in range(0, 10):
test_util.test_logger("shutdown host network round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
else:
test_util.test_logger("management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global test_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
for mn_host in mn_host_list:
if mn_host.ip_ != mn_vm_host[0].ip_:
test_host = mn_host
break
if not test_host:
test_util.test_fail('there is only one mn host')
test_util.test_logger(
"stop consul for host [%s] that mn vm is running on" % (test_host.ip_))
test_stub.stop_consul(test_host, test_lib.all_scenario_config)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM was destroyed after stopping another host's consul"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM starts on more than one host after stopping another host's consul"
)
except:
test_util.test_fail(
"management node VM was destroyed after stopping another host's consul"
)
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail(
'management node VM starts on another host when another host stopped consul'
)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file,
test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_is_multiple_networks()
for i in range(0, 10):
test_util.test_logger("shutdown host network round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
else:
test_util.test_logger("management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
try:
node_ops.wait_for_management_server_start()
except:
test_util.test_fail("management node does not recover after its former host's network down")
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_ or new_mn_host_ip == mn_host[0].managementIp_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
try:
node_ops.wait_for_management_server_start()
except:
test_util.test_fail("management node does not recover after its former host's network down")
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global ori_ip
global mn_vm_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger("change mn vm ip in config file of host [%s]" %
(mn_vm_host[0].ip_))
new_mn_vm_ip = "172.20.199.99"
ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP')
test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip,
test_lib.all_scenario_config)
test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config)
test_util.test_logger(
"wait for 40 seconds to see if management node VM starts on another host"
)
time.sleep(40)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not start after change ip and destroyed"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM starts on more than one host after change ip and destroyed"
)
except:
test_util.test_fail(
"management node VM does not start after change ip and destroyed")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail(
'management node VM starts on another host after change ip and destroyed'
)
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after destroyed")
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_ or new_mn_host_ip == mn_host[0].managementIp_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail("management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global test_host
test_stub.skip_if_scenario_is_multiple_networks()
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
for mn_host in mn_host_list:
if mn_host.ip_ != mn_vm_host[0].ip_:
test_host = mn_host
break
if not test_host:
test_util.test_fail('there is only one mn host')
test_util.test_logger(
"shutdown host's network [%s] that mn vm is not running on" %
(test_host.ip_))
test_stub.shutdown_host_network(test_host, test_lib.all_scenario_config)
test_util.test_logger("wait for the target host totally shutdown")
test_stub.ensure_host_disconnected(test_host, 300)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM was destroyed after another host network down"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM starts on more than one host after another host network down"
)
except:
test_util.test_fail(
"management node VM was destroyed after another host network down")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail(
'management node VM starts on another host when its former host was not down'
)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
#test_stub.ensure_hosts_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":# or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
#test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)
buildtype = test_stub.get_buildtype_by_sce_file(test_lib.scenario_file)
buildid = test_stub.get_buildid_by_sce_file(test_lib.scenario_file)
build_server = os.environ.get('BUILD_SERVER')
cmd = "cd /root/mirror/" + buildtype+ "/" + buildid + "/ && ls ZStack*-installer-*.bin"
zstack_bin_name = test_lib.lib_execute_ssh_cmd(build_server, "root", "password", cmd).replace("\n", "")
zstack_bin_path = "/root/" + zstack_bin_name
cmd = "wget -c http://" + build_server + "/mirror/" + buildtype + "/" + buildid + "/" + zstack_bin_name + " -O " + zstack_bin_path
test_lib.lib_execute_ssh_cmd(vip_s_vm_cfg_lst[0].ip_, "root", "password", cmd)
test_stub.exec_upgrade_mn(vip_s_vm_cfg_lst[0].ip_, "root", "password", zstack_bin_path)
print ":%s:" %(cmd)
time.sleep(5)
#expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
#if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
# test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("stop consul for host [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.stop_consul(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after its former host's consul down")
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global ori_ip
global mn_vm_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_))
new_mn_vm_ip = "172.20.199.99"
ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP')
test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config)
test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 40 seconds to see if management node VM starts on another host")
time.sleep(40)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not start after change ip and destroyed")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after change ip and destroyed")
except:
test_util.test_fail("management node VM does not start after change ip and destroyed")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host after change ip and destroyed')
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global test_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
for mn_host in mn_host_list:
if mn_host.ip_ != mn_vm_host[0].ip_:
test_host = mn_host
break
if not test_host:
test_util.test_fail('there is only one mn host')
test_util.test_logger("shutdown host [%s] that mn vm is not running on" % (test_host.ip_))
test_stub.stop_host(test_host, test_lib.all_scenario_config)
test_util.test_logger("wait for non-mn host shutdown totally")
test_stub.ensure_host_disconnected(test_host, 300)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM was destroyed after another host down")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after another host down")
except:
test_util.test_fail("management node VM was destroyed after another host down")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host when its former host was not down')
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
#test_stub.ensure_hosts_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global test_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
for mn_host in mn_host_list:
if mn_host.ip_ != mn_vm_host[0].ip_:
test_host = mn_host
break
if not test_host:
test_util.test_fail('there is only one mn host')
test_util.test_logger("force shutdown host [%s] that mn vm is not running on" % (test_host.ip_))
test_stub.stop_host(test_host, test_lib.all_scenario_config, 'cold')
test_util.test_logger("wait the target non-mn host force shutdown totally")
test_stub.ensure_host_disconnected(test_host, 300)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM was destroyed after another host down")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after another host down")
except:
test_util.test_fail("management node VM was destroyed after another host down")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host when its former host was not down')
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
#test_stub.ensure_hosts_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
global s_vm0
global s_vm1
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
s_vm0 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0)
s_vm1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1)
test_util.test_logger("force stop both hosts [%s] and [%s]" % (s_vm0.ip_, s_vm1.ip_))
test_stub.stop_host(s_vm0, test_lib.all_scenario_config, 'cold')
test_stub.stop_host(s_vm1, test_lib.all_scenario_config, 'cold')
time.sleep(5)
test_stub.start_host(s_vm0, test_lib.all_scenario_config)
test_stub.start_host(s_vm1, test_lib.all_scenario_config)
test_stub.recover_vlan_in_host(s_vm0.ip_, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.recover_vlan_in_host(s_vm1.ip_, test_lib.all_scenario_config, test_lib.deploy_config)
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file, 3)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
test_util.test_skip("2 hosts down at the same time is not support")
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (mn_host_list[index].ip_))
test_stub.shutdown_host_network(mn_host_list[index], test_lib.all_scenario_config)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config,
downMagt=False)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and
new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail(
"mn host should not changed if public network is down when public network is separated. Except [%s], while new host is [%s]"
% (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 120s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on one host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
if os.path.basename(
os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip(
) == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.skip_if_scenario_not_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=True)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) == 0:
test_util.test_fail('MN VM has been murdered!!! Expected should not be impacted when management network is down.')
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after MN VM is running")
for index in need_recover_mn_host_list:
test_util.test_logger("recover host: %s" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config,
downMagt=True)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and
new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail(
"management network down, mn host should not changed. Expected on [%s] while is on [%s]"
% (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 120s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
try:
node_ops.wait_for_management_server_start()
except:
test_util.test_fail(
"management node does not recover after its former host's network down"
)
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("force stop host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config, 'cold')
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after MN VM is running")
test_stub.ensure_hosts_connected(exclude_host=[mn_host_list[need_recover_mn_host_list[0]]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
test_util.test_skip("2 hosts down at the same time is not support")
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("force stop host [%s]" %
(mn_host_list[index].ip_))
test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config,
'cold')
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
test_util.test_logger("recover host [%s]" %
(mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]],
test_lib.all_scenario_config,
test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file,
test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("force stop host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config, 'cold')
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host_list[need_recover_mn_host_list[0]]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("force shutdown host round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger(
"force shutdown host [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold')
test_util.test_logger(
"wait for 40 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(
test_lib.scenario_file, test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
if os.path.basename(
os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip(
) == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.recover_host(mn_host[0], test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index],
test_lib.all_scenario_config,
test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(
exclude_host=[mn_host_list[need_recover_mn_host_list[0]]])
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) == 0:
test_util.test_fail('MN VM has been murdered, expected result should not be impacted when the separated network is down.')
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
