def test():
global vm
test_stub.return_skip_ahead_if_3sites("TEST SKIP")
#ensure mn works normally.
vm0 = test_stub.create_basic_vm()
vm0.check()
vm0.destroy()
ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(ori_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(ori_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
ori_host = ori_host[0]
target_host = None
for host in mn_host_list:
if host.ip_ != ori_host.ip_:
target_host = host
break
if target_host:
test_stub.migrate_mn_vm(ori_host, target_host, test_lib.all_scenario_config)
cur_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(cur_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(cur_host))
cur_host = cur_host[0]
if cur_host.ip_ != target_host.ip_:
test_util.test_fail("mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]" % (target_host.ip_, cur_host.ip_))
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Migrate MN VM Test Success')
else:
test_util.test_fail("there is no host for mn vm to migrate to")
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network: %s" % (node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s" % (l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
#test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s, and create vm" % (node_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
#test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test with one node network down and recovery Success')
def test():
global vm1
global vm2
global node_ip
vm1 = test_stub.create_basic_vm()
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop rabbitmq on node: %s" % (node_ip))
cmd = "rabbitmqctl stop_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(120)
test_stub.exercise_connection(1200)
test_util.test_logger("create vm to check if it still works")
vm1.check()
vm1.destroy()
vm2 = test_stub.create_basic_vm()
vm2.check()
vm2.destroy()
cmd = "rabbitmqctl start_app"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(60)
test_stub.exercise_connection(1200)
test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if not test_stub.upgrade_zsha(test_lib.all_scenario_config, test_lib.scenario_file):
test_util.test_fail('Fail to upgrade zsha')
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger('wait for 10s to see if something happens')
time.sleep(10)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM was destroyed after upgrade zsha")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after upgrade zsha")
except:
test_util.test_fail("management node VM was destroyed after upgrade zsha")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host after upgrade zsha')
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node1_ip
global node2_ip
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
end_time = int(time.time())
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
time.sleep(180)
host_uuid = vm.get_vm().hostUuid
data1 = test_lib.lib_get_host_cpu_prometheus_data(node1_ip, end_time, 200, host_uuid)
data2 = test_lib.lib_get_host_cpu_prometheus_data(node2_ip, end_time, 200, host_uuid)
if cmp(data1,data2) != 0:
test_util.test_fail('Prometheus Sync Test Fail, data on node:%s is [%s], data on node:%s is [%s]' % (node1_ip, data1, node2_ip, data2))
test_util.test_pass('Prometheus Sync Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
#test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)
test_stub.exec_zsha2_demote(vip_s_vm_cfg_lst[0].ip_, "root", "password")
time.sleep(5)
expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global tag_uuid
test_stub.skip_if_scenario_not_multiple_networks()
vm = test_stub.create_basic_vm()
vm.check()
console = test_lib.lib_get_vm_console_address(vm.get_vm().uuid)
console_management_ip = console.hostIp
cluster = res_ops.get_resource(res_ops.CLUSTER)[0]
tag_inv = tag_ops.create_system_tag('ClusterVO', cluster.uuid, "display::network::cidr::172.20.0.0/16")
tag_uuid = tag_inv.uuid
console = test_lib.lib_get_vm_console_address(vm.get_vm().uuid)
console_display_ip = console.hostIp
if console_management_ip == console_display_ip:
test_util.test_fail("console ip has not been switched as expected: %s and %s" %(console_management_ip, console_display_ip))
console1 = test_lib.lib_request_console_access(vm.get_vm().uuid)
console_req_ip = console1.hostname
if console_management_ip == console_req_ip:
test_util.test_fail("console ip has not been switched as expected: %s and %s" %(console_management_ip, console_req_ip))
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_is_multiple_networks()
for i in range(0, 10):
test_util.test_logger("shutdown host network round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
else:
test_util.test_logger("management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm, vm2
test_stub.skip_if_scenario_is_multiple_networks(mul_nets_sce_list=["scenario-config-ceph-sep-man.xml", "scenario-config-ceph-sep-pub.xml"])
test_stub.skip_if_vr_not_vyos("vr")
test_lib.clean_up_all_vr()
vm = test_stub.create_basic_vm()
vm.check()
vr_vm = test_lib.lib_find_vr_by_vm(vm.vm)[0]
vm.destroy()
t = threading.Thread(target=async_exec_reboot_vr, args=(vr_vm.uuid,))
t.start()
vm2 = test_stub.create_basic_vm(wait_vr_running=False)
vm2.check()
vm2.destroy()
test_util.test_pass('Create VM when vyos is rebooting Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_ or new_mn_host_ip == mn_host[0].managementIp_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail("management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":# or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30)
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_pass('Scheduler Test Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
test_util.test_dsc('Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail')
cmd = "rm /var/lib/zstack/ha/ha.yaml"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
cmd = "zstack-ctl recover_ha"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
if rsp == False:
test_util.test_logger("Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False")
else:
test_util.test_fail('Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha')
test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass')
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
#test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)
buildtype = test_stub.get_buildtype_by_sce_file(test_lib.scenario_file)
buildid = test_stub.get_buildid_by_sce_file(test_lib.scenario_file)
build_server = os.environ.get('BUILD_SERVER')
cmd = "cd /root/mirror/" + buildtype+ "/" + buildid + "/ && ls ZStack*-installer-*.bin"
zstack_bin_name = test_lib.lib_execute_ssh_cmd(build_server, "root", "password", cmd).replace("\n", "")
zstack_bin_path = "/root/" + zstack_bin_name
cmd = "wget -c http://" + build_server + "/mirror/" + buildtype + "/" + buildid + "/" + zstack_bin_name + " -O " + zstack_bin_path
test_lib.lib_execute_ssh_cmd(vip_s_vm_cfg_lst[0].ip_, "root", "password", cmd)
test_stub.exec_upgrade_mn(vip_s_vm_cfg_lst[0].ip_, "root", "password", zstack_bin_path)
print ":%s:" %(cmd)
time.sleep(5)
#expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
#if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
# test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network for a small while: %s" % (node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface, l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover mysql")
cmd = "service mysql start"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_stub.exercise_connection(600)
test_util.test_logger("check create vm")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test with one node network down for a while Success')
def test():
global vm
global node1_ip
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30)
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
for j in range(0, 6):
if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
time.sleep(180)
test_stub.exercise_connection(600)
test_util.test_pass('Scheduler Test Success')
def test():
global vm
global ori_ip
global mn_vm_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_))
new_mn_vm_ip = "172.20.199.99"
ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP')
test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config)
test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config)
test_util.test_logger("wait for 40 seconds to see if management node VM starts on another host")
time.sleep(40)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not start after change ip and destroyed")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after change ip and destroyed")
except:
test_util.test_fail("management node VM does not start after change ip and destroyed")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host after change ip and destroyed')
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if not test_stub.upgrade_zsha(test_lib.all_scenario_config,
test_lib.scenario_file):
test_util.test_fail('Fail to upgrade zsha')
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger('wait for 10s to see if something happens')
time.sleep(10)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM was destroyed after upgrade zsha")
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM starts on more than one host after upgrade zsha"
)
except:
test_util.test_fail(
"management node VM was destroyed after upgrade zsha")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail(
'management node VM starts on another host after upgrade zsha')
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global test_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
for mn_host in mn_host_list:
if mn_host.ip_ != mn_vm_host[0].ip_:
test_host = mn_host
break
if not test_host:
test_util.test_fail('there is only one mn host')
test_util.test_logger("shutdown host [%s] that mn vm is not running on" % (test_host.ip_))
test_stub.stop_host(test_host, test_lib.all_scenario_config)
test_util.test_logger("wait for non-mn host shutdown totally")
test_stub.ensure_host_disconnected(test_host, 300)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail("management node VM was destroyed after another host down")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM starts on more than one host after another host down")
except:
test_util.test_fail("management node VM was destroyed after another host down")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail('management node VM starts on another host when its former host was not down')
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
#test_stub.ensure_hosts_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_util.test_logger(
"wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
test_util.test_logger("recover node: %s, and create vm" % (node_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node_ip))
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(1200)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s" % (node_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(1200)
test_util.test_pass('Create VM Test Success')
def test():
global vm
for i in range(0, 10):
test_util.test_logger("migrate MN VM round %s" % (i))
ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(ori_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(ori_host))
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
ori_host = ori_host[0]
target_host = None
for host in mn_host_list:
if host.ip_ != ori_host.ip_:
target_host = host
break
if target_host:
test_stub.migrate_mn_vm(ori_host, target_host,
test_lib.all_scenario_config)
cur_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(cur_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(cur_host))
cur_host = cur_host[0]
if cur_host.ip_ != target_host.ip_:
test_util.test_fail(
"mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]"
% (target_host.ip_, cur_host.ip_))
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
else:
test_util.test_fail("there is no host for mn vm to migrate to")
test_util.test_pass('Migrate MN VM Test Success')
def test():
global vm
global node1_ip
global node2_ip
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
test_util.test_logger(
"wait for 2 minutes to see if http api still works well")
time.sleep(180)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
end_time = int(time.time())
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
time.sleep(180)
host_uuid = vm.get_vm().hostUuid
data1 = test_lib.lib_get_host_cpu_prometheus_data(node1_ip, end_time, 200,
host_uuid)
data2 = test_lib.lib_get_host_cpu_prometheus_data(node2_ip, end_time, 200,
host_uuid)
if cmp(data1, data2) != 0:
test_util.test_fail(
'Prometheus Sync Test Fail, data on node:%s is [%s], data on node:%s is [%s]'
% (node1_ip, data1, node2_ip, data2))
test_util.test_pass('Prometheus Sync Test Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global vip_s_vm_cfg_lst
global s_vm0
global s_vm1
vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
if len(vip_s_vm_cfg_lst) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))
s_vm0 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0)
s_vm1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1)
test_util.test_logger("force stop both hosts [%s] and [%s]" % (s_vm0.ip_, s_vm1.ip_))
test_stub.stop_host(s_vm0, test_lib.all_scenario_config, 'cold')
test_stub.stop_host(s_vm1, test_lib.all_scenario_config, 'cold')
time.sleep(5)
test_stub.start_host(s_vm0, test_lib.all_scenario_config)
test_stub.start_host(s_vm1, test_lib.all_scenario_config)
test_stub.recover_vlan_in_host(s_vm0.ip_, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.recover_vlan_in_host(s_vm1.ip_, test_lib.all_scenario_config, test_lib.deploy_config)
vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file, 3)
if len(vip_s_vm_cfg_lst_new) != 1:
test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.skip_if_vr_not_vyos("vr")
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
vm = test_stub.create_basic_vm()
vm.check()
ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop")
vm.set_state(vm_header.RUNNING)
vm.check()
for host in mn_host_list:
test_util.test_logger("shutdown host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index],
test_lib.all_scenario_config,
test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail(
"management node does not recover after MN VM is running")
test_util.test_logger("Delay 60s and then check if the vm is running")
time.sleep(180)
if test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 300):
vm.update()
vm.check()
vm.destroy()
else:
test_util.test_fail(
"ha vm has not changed to running after 2 hosts recover with 300s")
test_util.test_pass('Check Never Stop VM Test Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node2_ip))
test_util.test_dsc(
'Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail'
)
cmd = "rm /var/lib/zstack/ha/ha.yaml"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
cmd = "zstack-ctl recover_ha"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
if rsp == False:
test_util.test_logger(
"Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False"
)
else:
test_util.test_fail(
'Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha'
)
test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass')
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global ori_ip
global mn_vm_host
mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_vm_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host))
test_util.test_logger("change mn vm ip in config file of host [%s]" %
(mn_vm_host[0].ip_))
new_mn_vm_ip = "172.20.199.99"
ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP')
test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip,
test_lib.all_scenario_config)
test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config)
test_util.test_logger(
"wait for 40 seconds to see if management node VM starts on another host"
)
time.sleep(40)
try:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not start after change ip and destroyed"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM starts on more than one host after change ip and destroyed"
)
except:
test_util.test_fail(
"management node VM does not start after change ip and destroyed")
if new_mn_host[0].ip_ != mn_vm_host[0].ip_:
test_util.test_fail(
'management node VM starts on another host after change ip and destroyed'
)
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after destroyed")
test_util.test_logger("try to create vm, timeout is 30s")
time_out = 30
while time_out > 0:
try:
vm = test_stub.create_basic_vm()
break
except:
time.sleep(1)
time_out -= 1
if time_out == 0:
test_util.test_fail('Fail to create vm after mn is ready')
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on one host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
if os.path.basename(
os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip(
) == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_is_multiple_networks()
for i in range(0, 10):
test_util.test_logger("shutdown host network round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
else:
test_util.test_logger(
"management node VM run on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(
test_lib.scenario_file, test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
if os.path.basename(
os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip(
) == "scenario-config-vpc-ceph-3-sites.xml":
pass
else:
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
ha_vm = test_stub.create_ha_vm()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("force stop host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config, 'cold')
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.return_pass_ahead_if_3sites("TEST PASS")
ha_vm.set_state(vm_header.RUNNING)
ha_vm.check()
test_util.test_logger("try to create vm, timeout is 30s")
time_out = 30
while time_out > 0:
try:
vm = test_stub.create_basic_vm()
break
except:
time.sleep(1)
time_out -= 1
if time_out == 0:
test_util.test_fail('Fail to create vm after mn is ready')
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node1_ip
global node2_ip
global host_username
global host_password
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_dsc('Prepare upgrade package and iso')
cmd = "scp 192.168.200.1:/httpd/zstack_iso_centos7/latest/test/ZStack-Community-x86_64-DVD-1.4.0.iso /root/"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 600)
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 600)
all_in_one_pkg = os.environ['zstackPkg']
local_ip = os.environ['sftpBackupStorageHostname']
cmd = "scp %s:%s /root/" % (local_ip, all_in_one_pkg)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_dsc('Start MN of node1 and node2')
cmd = "zstack-ctl start"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 600)
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 600)
iso_path = "/root/ZStack-Community-x86_64-DVD-1.4.0.iso"
upk_pkg = "/root/zstack-offline-installer-test.bin"
test_util.test_dsc('Upgrade HA')
cmd = "zstack-ctl upgrade_ha --mevoco-installer %s --iso %s" % (upk_pkg,
iso_path)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 1800)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 1800)
time.sleep(180)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
cmd = "echo 'y' | rm %s" % iso_path
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
cmd = "echo 'y' | rm %s" % upk_pkg
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
test_util.test_skip("2 hosts down at the same time is not support")
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (mn_host_list[index].ip_))
test_stub.shutdown_host_network(mn_host_list[index], test_lib.all_scenario_config)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
test_util.test_logger("wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger(
"stop consul for host [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.stop_consul(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
try:
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
except:
test_util.test_fail(
"management node does not recover after its former host's consul down"
)
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger("shutdown host [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail(
"management node does not recover after its former host down")
test_util.test_logger("try to create vm, timeout is 30s")
time_out = 30
while time_out > 0:
try:
vm = test_stub.create_basic_vm()
break
except:
time.sleep(1)
time_out -= 1
if time_out == 0:
test_util.test_fail('Fail to create vm after mn is ready')
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple',
'simple_reboot_vm_scheduler',
start_date + 60, 30)
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
if test_lib.lib_find_in_remote_management_server_log(
node1_ip, host_username, host_password,
start_date + 60 + 30 * i,
'[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg',
vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(
node2_ip, host_username, host_password,
start_date + 60 + 30 * i,
'[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg',
vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail(
'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times'
% (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_pass('Scheduler Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("force stop host [%s]" % (host.ip_))
test_stub.stop_host(host, test_lib.all_scenario_config, 'cold')
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 30s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after MN VM is running")
test_stub.ensure_hosts_connected(exclude_host=[mn_host_list[need_recover_mn_host_list[0]]])
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) == 0:
test_util.test_fail('MN VM has been murdered, expected result should not be impacted when the separated network is down.')
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
global pub_mn_ip
global mag_mn_ip
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config,
downMagt=False)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[
0].ip_ or new_mn_host_ip == mn_host[0].managementIp_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 120s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start()
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("force shutdown host round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger(
"force shutdown host [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold')
test_util.test_logger(
"wait for 40 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail(
"management node does not recover after its former host down")
test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]])
test_stub.ensure_bss_host_connected_from_stop(
test_lib.scenario_file, test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_stub.recover_host(mn_host[0], test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
test_stub.skip_if_scenario_not_multiple_networks()
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host))
test_util.test_logger(
"shutdown host's network [%s] that mn vm is running on" %
(mn_host[0].ip_))
test_stub.shutdown_host_network(mn_host[0],
test_lib.all_scenario_config,
downMagt=True)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on another host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and
new_mn_host_ip != mn_host[0].managementIp_):
test_util.test_fail(
"management network down, mn host should not changed. Expected on [%s] while is on [%s]"
% (mn_host[0].ip_, new_mn_host_ip))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 120s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
try:
node_ops.wait_for_management_server_start()
except:
test_util.test_fail(
"management node does not recover after its former host's network down"
)
test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config)
test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config,
test_lib.scenario_file)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index],
test_lib.all_scenario_config,
test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected(
exclude_host=[mn_host_list[need_recover_mn_host_list[0]]])
test_stub.ensure_bss_host_connected_from_sep_net_down(
test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
test_stub.return_pass_ahead_if_3sites("TEST PASS")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
test_util.test_skip("2 hosts down at the same time is not support")
test_stub.skip_if_scenario_is_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("force stop host [%s]" %
(mn_host_list[index].ip_))
test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config,
'cold')
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
test_util.test_logger("recover host [%s]" %
(mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]],
test_lib.all_scenario_config,
test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
#node_ops.wait_for_management_server_start(300)
test_stub.wrapper_of_wait_for_management_server_start(600)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file,
test_lib.all_scenario_config,
test_lib.deploy_config)
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host
for i in range(0, 10):
test_util.test_logger("destroy mn vm round %s" % (i))
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 1:
test_util.test_fail('MN VM is running on %d host(s)' %
len(mn_host))
test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_)
test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config)
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 0:
test_util.test_logger("mn vm was destroyed successfully")
else:
test_util.test_fail("mn vm was not destroyed successfully")
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on one host"
)
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_:
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(
test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s"
)
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail(
"management node does not recover after mn vm was destroyed")
test_stub.ensure_hosts_connected()
test_stub.ensure_pss_connected()
test_stub.ensure_bss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global test_mn_host_list
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config,
test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num),
(mn_host_num + 1) / 2)
for index in test_mn_host_list:
test_util.test_logger("shutdown host [%s]" % (mn_host_list[index].ip_))
test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(mn_host) != 0:
test_util.test_fail('MN VM is still running on %d host(s)' %
len(mn_host))
test_util.test_logger("recover host [%s]" %
(mn_host_list[test_mn_host_list[-1]]))
test_stub.recover_host(mn_host_list[test_mn_host_list[-1]],
test_lib.all_scenario_config,
test_lib.deploy_config)
test_mn_host_list.pop()
test_util.test_logger(
"wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(
test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail(
"management node VM not run correctly on [%s] after its former host [%s] down for 20s"
% (new_mn_host_ip, mn_host_list[0].ip_))
count = 60
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config,
test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger(
"management node VM run after its former host down for 30s")
break
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail(
"management node VM does not run after its former host down for 30s"
)
elif len(new_mn_host) > 1:
test_util.test_fail(
"management node VM runs on more than one host after its former host down"
)
test_util.test_logger(
"wait for 5 minutes to see if management node starts again")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail(
"management node does not recover after recover one mn host")
test_util.test_logger("try to create vm, timeout is 30s")
time_out = 30
while time_out > 0:
try:
vm = test_stub.create_basic_vm()
break
except:
time.sleep(1)
time_out -= 1
if time_out == 0:
test_util.test_fail('Fail to create vm after mn is ready')
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
global mn_host_list
global need_recover_mn_host_list
test_stub.skip_if_scenario_not_multiple_networks()
mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file)
mn_host_num = len(mn_host_list)
test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2)
for host in mn_host_list:
test_util.test_logger("shutdown host's network [%s]" % (host.ip_))
test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=True)
need_recover_mn_host_list = range(mn_host_num)
test_util.test_logger("wait 10s for MN VM to stop")
time.sleep(10)
mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(mn_host) == 0:
test_util.test_fail('MN VM has been murdered!!! Expected should not be impacted when management network is down.')
for index in test_mn_host_list:
test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
need_recover_mn_host_list.remove(index)
test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host")
time.sleep(20)
new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file)
if new_mn_host_ip == "":
test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_))
count = 120
while count > 0:
new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file)
if len(new_mn_host) == 1:
test_util.test_logger("management node VM run after its former host down for 120s")
break
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
time.sleep(5)
count -= 1
if len(new_mn_host) == 0:
test_util.test_fail("management node VM does not run after its former host down for 120s")
elif len(new_mn_host) > 1:
test_util.test_fail("management node VM runs on more than one host after its former host down")
try:
node_ops.wait_for_management_server_start(300)
except:
test_util.test_fail("management node does not recover after MN VM is running")
for index in need_recover_mn_host_list:
test_util.test_logger("recover host: %s" % (mn_host_list[index].ip_))
test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config)
test_stub.ensure_hosts_connected()
test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True)
test_stub.ensure_bss_connected()
test_stub.ensure_pss_connected()
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
