def test(): global vm test_stub.return_skip_ahead_if_3sites("TEST SKIP") #ensure mn works normally. vm0 = test_stub.create_basic_vm() vm0.check() vm0.destroy() ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(ori_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(ori_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) ori_host = ori_host[0] target_host = None for host in mn_host_list: if host.ip_ != ori_host.ip_: target_host = host break if target_host: test_stub.migrate_mn_vm(ori_host, target_host, test_lib.all_scenario_config) cur_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(cur_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(cur_host)) cur_host = cur_host[0] if cur_host.ip_ != target_host.ip_: test_util.test_fail("mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]" % (target_host.ip_, cur_host.ip_)) else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Migrate MN VM Test Success') else: test_util.test_fail("there is no host for mn vm to migrate to")
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node network: %s" % (node_ip)) l2_network_interface = os.environ.get('l2ManagementNetworkInterface') cmd = "ifdown %s" % (l2_network_interface) host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) #test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_logger("recover node: %s, and create vm" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) #test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test with one node network down and recovery Success')
def test(): global vm1 global vm2 global node_ip vm1 = test_stub.create_basic_vm() node_ip = os.environ.get('node1Ip') test_util.test_logger("stop rabbitmq on node: %s" % (node_ip)) cmd = "rabbitmqctl stop_app" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(120) test_stub.exercise_connection(1200) test_util.test_logger("create vm to check if it still works") vm1.check() vm1.destroy() vm2 = test_stub.create_basic_vm() vm2.check() vm2.destroy() cmd = "rabbitmqctl start_app" rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(60) test_stub.exercise_connection(1200) test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if not test_stub.upgrade_zsha(test_lib.all_scenario_config, test_lib.scenario_file): test_util.test_fail('Fail to upgrade zsha') if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) test_util.test_logger('wait for 10s to see if something happens') time.sleep(10) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail("management node VM was destroyed after upgrade zsha") elif len(new_mn_host) > 1: test_util.test_fail("management node VM starts on more than one host after upgrade zsha") except: test_util.test_fail("management node VM was destroyed after upgrade zsha") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail('management node VM starts on another host after upgrade zsha') test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global node1_ip global node2_ip node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) vm = test_stub.create_basic_vm() vm.check() vm.destroy() end_time = int(time.time()) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) host_uuid = vm.get_vm().hostUuid data1 = test_lib.lib_get_host_cpu_prometheus_data(node1_ip, end_time, 200, host_uuid) data2 = test_lib.lib_get_host_cpu_prometheus_data(node2_ip, end_time, 200, host_uuid) if cmp(data1,data2) != 0: test_util.test_fail('Prometheus Sync Test Fail, data on node:%s is [%s], data on node:%s is [%s]' % (node1_ip, data1, node2_ip, data2)) test_util.test_pass('Prometheus Sync Test Success')
def test(): global vm global vip_s_vm_cfg_lst vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst)) test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_)) #test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config) test_stub.exec_zsha2_demote(vip_s_vm_cfg_lst[0].ip_, "root", "password") time.sleep(5) expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_) if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip): test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip)) vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst_new) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new)) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]]) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global tag_uuid test_stub.skip_if_scenario_not_multiple_networks() vm = test_stub.create_basic_vm() vm.check() console = test_lib.lib_get_vm_console_address(vm.get_vm().uuid) console_management_ip = console.hostIp cluster = res_ops.get_resource(res_ops.CLUSTER)[0] tag_inv = tag_ops.create_system_tag('ClusterVO', cluster.uuid, "display::network::cidr::172.20.0.0/16") tag_uuid = tag_inv.uuid console = test_lib.lib_get_vm_console_address(vm.get_vm().uuid) console_display_ip = console.hostIp if console_management_ip == console_display_ip: test_util.test_fail("console ip has not been switched as expected: %s and %s" %(console_management_ip, console_display_ip)) console1 = test_lib.lib_request_console_access(vm.get_vm().uuid) console_req_ip = console1.hostname if console_management_ip == console_req_ip: test_util.test_fail("console ip has not been switched as expected: %s and %s" %(console_management_ip, console_req_ip)) test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("destroy mn vm round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_is_multiple_networks() for i in range(0, 10): test_util.test_logger("shutdown host network round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) else: test_util.test_logger("management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_util.test_pass('Create VM Test Success')
def test(): global vm, vm2 test_stub.skip_if_scenario_is_multiple_networks(mul_nets_sce_list=["scenario-config-ceph-sep-man.xml", "scenario-config-ceph-sep-pub.xml"]) test_stub.skip_if_vr_not_vyos("vr") test_lib.clean_up_all_vr() vm = test_stub.create_basic_vm() vm.check() vr_vm = test_lib.lib_find_vr_by_vm(vm.vm)[0] vm.destroy() t = threading.Thread(target=async_exec_reboot_vr, args=(vr_vm.uuid,)) t.start() vm2 = test_stub.create_basic_vm(wait_vr_running=False) vm2.check() vm2.destroy() test_util.test_pass('Create VM when vyos is rebooting Test Success')
def test(): global vm global mn_host global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_ or new_mn_host_ip == mn_host[0].managementIp_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_): test_util.test_fail("management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "":# or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Scheduler Test Success')
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) test_util.test_dsc('Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail') cmd = "rm /var/lib/zstack/ha/ha.yaml" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) cmd = "zstack-ctl recover_ha" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) if rsp == False: test_util.test_logger("Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False") else: test_util.test_fail('Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha') test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass') cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global vip_s_vm_cfg_lst vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst)) test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_)) #test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config) buildtype = test_stub.get_buildtype_by_sce_file(test_lib.scenario_file) buildid = test_stub.get_buildid_by_sce_file(test_lib.scenario_file) build_server = os.environ.get('BUILD_SERVER') cmd = "cd /root/mirror/" + buildtype+ "/" + buildid + "/ && ls ZStack*-installer-*.bin" zstack_bin_name = test_lib.lib_execute_ssh_cmd(build_server, "root", "password", cmd).replace("\n", "") zstack_bin_path = "/root/" + zstack_bin_name cmd = "wget -c http://" + build_server + "/mirror/" + buildtype + "/" + buildid + "/" + zstack_bin_name + " -O " + zstack_bin_path test_lib.lib_execute_ssh_cmd(vip_s_vm_cfg_lst[0].ip_, "root", "password", cmd) test_stub.exec_upgrade_mn(vip_s_vm_cfg_lst[0].ip_, "root", "password", zstack_bin_path) print ":%s:" %(cmd) time.sleep(5) #expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_) #if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip): # test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip)) vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst_new) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new)) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]]) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node network for a small while: %s" % (node_ip)) l2_network_interface = os.environ.get('l2ManagementNetworkInterface') cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface, l2_network_interface) host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover mysql") cmd = "service mysql start" rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_stub.exercise_connection(600) test_util.test_logger("check create vm") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test with one node network down for a while Success')
def test(): global vm global node1_ip vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30) node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): for j in range(0, 6): if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) test_stub.exercise_connection(600) test_util.test_pass('Scheduler Test Success')
def test(): global vm global ori_ip global mn_vm_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_)) new_mn_vm_ip = "172.20.199.99" ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP') test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config) test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config) test_util.test_logger("wait for 40 seconds to see if management node VM starts on another host") time.sleep(40) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail("management node VM does not start after change ip and destroyed") elif len(new_mn_host) > 1: test_util.test_fail("management node VM starts on more than one host after change ip and destroyed") except: test_util.test_fail("management node VM does not start after change ip and destroyed") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail('management node VM starts on another host after change ip and destroyed') os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if not test_stub.upgrade_zsha(test_lib.all_scenario_config, test_lib.scenario_file): test_util.test_fail('Fail to upgrade zsha') if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) test_util.test_logger('wait for 10s to see if something happens') time.sleep(10) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail( "management node VM was destroyed after upgrade zsha") elif len(new_mn_host) > 1: test_util.test_fail( "management node VM starts on more than one host after upgrade zsha" ) except: test_util.test_fail( "management node VM was destroyed after upgrade zsha") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail( 'management node VM starts on another host after upgrade zsha') test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global test_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) for mn_host in mn_host_list: if mn_host.ip_ != mn_vm_host[0].ip_: test_host = mn_host break if not test_host: test_util.test_fail('there is only one mn host') test_util.test_logger("shutdown host [%s] that mn vm is not running on" % (test_host.ip_)) test_stub.stop_host(test_host, test_lib.all_scenario_config) test_util.test_logger("wait for non-mn host shutdown totally") test_stub.ensure_host_disconnected(test_host, 300) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail("management node VM was destroyed after another host down") elif len(new_mn_host) > 1: test_util.test_fail("management node VM starts on more than one host after another host down") except: test_util.test_fail("management node VM was destroyed after another host down") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail('management node VM starts on another host when its former host was not down') test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() #test_stub.ensure_hosts_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger( "wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) test_util.test_logger("recover node: %s, and create vm" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(1200) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_logger("recover node: %s" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) test_stub.exercise_connection(1200) test_util.test_pass('Create VM Test Success')
def test(): global vm for i in range(0, 10): test_util.test_logger("migrate MN VM round %s" % (i)) ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(ori_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(ori_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) ori_host = ori_host[0] target_host = None for host in mn_host_list: if host.ip_ != ori_host.ip_: target_host = host break if target_host: test_stub.migrate_mn_vm(ori_host, target_host, test_lib.all_scenario_config) cur_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(cur_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(cur_host)) cur_host = cur_host[0] if cur_host.ip_ != target_host.ip_: test_util.test_fail( "mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]" % (target_host.ip_, cur_host.ip_)) else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() else: test_util.test_fail("there is no host for mn vm to migrate to") test_util.test_pass('Migrate MN VM Test Success')
def test(): global vm global node1_ip global node2_ip node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger( "wait for 2 minutes to see if http api still works well") time.sleep(180) vm = test_stub.create_basic_vm() vm.check() vm.destroy() end_time = int(time.time()) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) host_uuid = vm.get_vm().hostUuid data1 = test_lib.lib_get_host_cpu_prometheus_data(node1_ip, end_time, 200, host_uuid) data2 = test_lib.lib_get_host_cpu_prometheus_data(node2_ip, end_time, 200, host_uuid) if cmp(data1, data2) != 0: test_util.test_fail( 'Prometheus Sync Test Fail, data on node:%s is [%s], data on node:%s is [%s]' % (node1_ip, data1, node2_ip, data2)) test_util.test_pass('Prometheus Sync Test Success')
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global vip_s_vm_cfg_lst global s_vm0 global s_vm1 vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst)) s_vm0 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0) s_vm1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1) test_util.test_logger("force stop both hosts [%s] and [%s]" % (s_vm0.ip_, s_vm1.ip_)) test_stub.stop_host(s_vm0, test_lib.all_scenario_config, 'cold') test_stub.stop_host(s_vm1, test_lib.all_scenario_config, 'cold') time.sleep(5) test_stub.start_host(s_vm0, test_lib.all_scenario_config) test_stub.start_host(s_vm1, test_lib.all_scenario_config) test_stub.recover_vlan_in_host(s_vm0.ip_, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.recover_vlan_in_host(s_vm1.ip_, test_lib.all_scenario_config, test_lib.deploy_config) vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file, 3) if len(vip_s_vm_cfg_lst_new) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new)) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]]) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_vr_not_vyos("vr") mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) vm = test_stub.create_basic_vm() vm.check() ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop") vm.set_state(vm_header.RUNNING) vm.check() for host in mn_host_list: test_util.test_logger("shutdown host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after MN VM is running") test_util.test_logger("Delay 60s and then check if the vm is running") time.sleep(180) if test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 300): vm.update() vm.check() vm.destroy() else: test_util.test_fail( "ha vm has not changed to running after 2 hosts recover with 300s") test_util.test_pass('Check Never Stop VM Test Success')
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) test_util.test_dsc( 'Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail' ) cmd = "rm /var/lib/zstack/ha/ha.yaml" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) cmd = "zstack-ctl recover_ha" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) if rsp == False: test_util.test_logger( "Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False" ) else: test_util.test_fail( 'Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha' ) test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass') cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global ori_ip global mn_vm_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_)) new_mn_vm_ip = "172.20.199.99" ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP') test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config) test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config) test_util.test_logger( "wait for 40 seconds to see if management node VM starts on another host" ) time.sleep(40) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not start after change ip and destroyed" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM starts on more than one host after change ip and destroyed" ) except: test_util.test_fail( "management node VM does not start after change ip and destroyed") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail( 'management node VM starts on another host after change ip and destroyed' ) os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip test_util.test_logger( "wait for 5 minutes to see if management node starts again") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail("management node does not recover after destroyed") test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("destroy mn vm round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger( "wait for 20 seconds to see if management node VM starts on one host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s" ) break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") if os.path.basename( os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip( ) == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_is_multiple_networks() for i in range(0, 10): test_util.test_logger("shutdown host network round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger( "shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) else: test_util.test_logger( "management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s" ) break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_stop( test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() if os.path.basename( os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip( ) == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list ha_vm = test_stub.create_ha_vm() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("force stop host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config, 'cold') need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.return_pass_ahead_if_3sites("TEST PASS") ha_vm.set_state(vm_header.RUNNING) ha_vm.check() test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global node1_ip global node2_ip global host_username global host_password host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_dsc('Prepare upgrade package and iso') cmd = "scp 192.168.200.1:/httpd/zstack_iso_centos7/latest/test/ZStack-Community-x86_64-DVD-1.4.0.iso /root/" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 600) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 600) all_in_one_pkg = os.environ['zstackPkg'] local_ip = os.environ['sftpBackupStorageHostname'] cmd = "scp %s:%s /root/" % (local_ip, all_in_one_pkg) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_dsc('Start MN of node1 and node2') cmd = "zstack-ctl start" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 600) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 600) iso_path = "/root/ZStack-Community-x86_64-DVD-1.4.0.iso" upk_pkg = "/root/zstack-offline-installer-test.bin" test_util.test_dsc('Upgrade HA') cmd = "zstack-ctl upgrade_ha --mevoco-installer %s --iso %s" % (upk_pkg, iso_path) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 1800) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 1800) time.sleep(180) vm = test_stub.create_basic_vm() vm.check() vm.destroy() cmd = "echo 'y' | rm %s" % iso_path rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) cmd = "echo 'y' | rm %s" % upk_pkg rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list test_util.test_skip("2 hosts down at the same time is not support") test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (mn_host_list[index].ip_)) test_stub.shutdown_host_network(mn_host_list[index], test_lib.all_scenario_config) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger( "stop consul for host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_consul(mn_host[0], test_lib.all_scenario_config) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") try: #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) except: test_util.test_fail( "management node does not recover after its former host's consul down" ) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_host(mn_host[0], test_lib.all_scenario_config) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after its former host down") test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date + 60, 30) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): if test_lib.lib_find_in_remote_management_server_log( node1_ip, host_username, host_password, start_date + 60 + 30 * i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log( node2_ip, host_username, host_password, start_date + 60 + 30 * i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail( 'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Scheduler Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("force stop host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config, 'cold') need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail("management node does not recover after MN VM is running") test_stub.ensure_hosts_connected(exclude_host=[mn_host_list[need_recover_mn_host_list[0]]]) test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) == 0: test_util.test_fail('MN VM has been murdered, expected result should not be impacted when the separated network is down.') for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip test_util.test_logger( "shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[ 0].ip_ or new_mn_host_ip == mn_host[0].managementIp_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 120s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("force shutdown host round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger( "force shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold') test_util.test_logger( "wait for 40 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s" ) break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after its former host down") test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_stop( test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_stub.recover_host(mn_host[0], test_lib.all_scenario_config, test_lib.deploy_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger( "shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_): test_util.test_fail( "management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 120s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) try: node_ops.wait_for_management_server_start() except: test_util.test_fail( "management node does not recover after its former host's network down" ) test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected( exclude_host=[mn_host_list[need_recover_mn_host_list[0]]]) test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list test_util.test_skip("2 hosts down at the same time is not support") test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("force stop host [%s]" % (mn_host_list[index].ip_)) test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config, 'cold') test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("destroy mn vm round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger( "wait for 20 seconds to see if management node VM starts on one host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s" ) break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after mn vm was destroyed") test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("shutdown host [%s]" % (mn_host_list[index].ip_)) test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after recover one mn host") test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_scenario_not_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=True) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) == 0: test_util.test_fail('MN VM has been murdered!!! Expected should not be impacted when management network is down.') for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail("management node does not recover after MN VM is running") for index in need_recover_mn_host_list: test_util.test_logger("recover host: %s" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')