def test(): global vm global vip_s_vm_cfg_lst vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst)) test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_)) #test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config) test_stub.exec_zsha2_demote(vip_s_vm_cfg_lst[0].ip_, "root", "password") time.sleep(5) expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_) if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip): test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip)) vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst_new) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new)) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]]) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("close MN-VM and all zs-ha services in cluster") test_stub.stop_zsha(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) != 0: test_util.test_fail('MN VM is still running after shutdown ha cluster') test_util.test_logger("start ha cluster") test_stub.start_zsha(mn_host[0], test_lib.all_scenario_config) test_util.test_logger("wait 50s for MN VM to start") time.sleep(50) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail( 'MN VM is running on %d host(s) after shutdown and start ha cluster' % len(mn_host)) test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_util.test_pass('Shutdown HA Cluster Test Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("destroy mn vm round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_is_multiple_networks() for i in range(0, 10): test_util.test_logger("shutdown host network round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) else: test_util.test_logger("management node VM run on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_ or new_mn_host_ip == mn_host[0].managementIp_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global ori_ip global mn_vm_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_)) new_mn_vm_ip = "172.20.199.99" ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP') test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config) test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config) test_util.test_logger( "wait for 40 seconds to see if management node VM starts on another host" ) time.sleep(40) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not start after change ip and destroyed" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM starts on more than one host after change ip and destroyed" ) except: test_util.test_fail( "management node VM does not start after change ip and destroyed") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail( 'management node VM starts on another host after change ip and destroyed' ) os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global ha_vm global mn_host ha_vm = test_stub.create_ha_vm() ha_vm.check() ha_vm_host = test_lib.lib_get_vm_host(ha_vm.vm) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) if ha_vm_host.managementIp != mn_host[0].ip_: conditions = res_ops.gen_query_conditions('managementIp', '=', mn_host[0].ip_) host = res_ops.query_resource(res_ops.HOST, conditions) ha_vm_host.migrate(host[0].uuid) test_util.test_logger("force shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold') test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) conditions = res_ops.gen_query_conditions('managementIp', '=', mn_host[0].ip_) host = res_ops.query_resource(res_ops.HOST, conditions) if host[0].status != "Connected": test_util.test_fail("Target host:%s is not connected as expected." %(host[0].uuid)) ha_vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_): test_util.test_fail("management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger("wait for 20 seconds to see if management node VM starts on one host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "":# or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global vip_s_vm_cfg_lst vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst)) test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_)) #test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config) buildtype = test_stub.get_buildtype_by_sce_file(test_lib.scenario_file) buildid = test_stub.get_buildid_by_sce_file(test_lib.scenario_file) build_server = os.environ.get('BUILD_SERVER') cmd = "cd /root/mirror/" + buildtype+ "/" + buildid + "/ && ls ZStack*-installer-*.bin" zstack_bin_name = test_lib.lib_execute_ssh_cmd(build_server, "root", "password", cmd).replace("\n", "") zstack_bin_path = "/root/" + zstack_bin_name cmd = "wget -c http://" + build_server + "/mirror/" + buildtype + "/" + buildid + "/" + zstack_bin_name + " -O " + zstack_bin_path test_lib.lib_execute_ssh_cmd(vip_s_vm_cfg_lst[0].ip_, "root", "password", cmd) test_stub.exec_upgrade_mn(vip_s_vm_cfg_lst[0].ip_, "root", "password", zstack_bin_path) print ":%s:" %(cmd) time.sleep(5) #expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_) #if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip): # test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip)) vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst_new) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new)) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]]) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_host(mn_host[0], test_lib.all_scenario_config) test_util.test_logger("wait for 20 seconds to see if management node VM starts on another host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("close MN-VM and all zs-ha services in cluster") test_stub.stop_zsha(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) != 0: test_util.test_fail('MN VM is still running after shutdown ha cluster') test_util.test_logger("start ha cluster") test_stub.start_zsha(mn_host[0], test_lib.all_scenario_config) test_util.test_logger("wait 50s for MN VM to start") time.sleep(50) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s) after shutdown and start ha cluster' % len(mn_host)) test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_util.test_pass('Shutdown HA Cluster Test Success')
def test(): global vm global ori_ip global mn_vm_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) test_util.test_logger("change mn vm ip in config file of host [%s]" % (mn_vm_host[0].ip_)) new_mn_vm_ip = "172.20.199.99" ori_ip = os.environ.get('ZSTACK_BUILT_IN_HTTP_SERVER_IP') test_stub.update_mn_vm_config(mn_vm_host[0], 'Ipaddr', new_mn_vm_ip, test_lib.all_scenario_config) test_stub.destroy_mn_vm(mn_vm_host[0], test_lib.all_scenario_config) test_util.test_logger("wait for 40 seconds to see if management node VM starts on another host") time.sleep(40) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail("management node VM does not start after change ip and destroyed") elif len(new_mn_host) > 1: test_util.test_fail("management node VM starts on more than one host after change ip and destroyed") except: test_util.test_fail("management node VM does not start after change ip and destroyed") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail('management node VM starts on another host after change ip and destroyed') os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = new_mn_vm_ip test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global vip_s_vm_cfg_lst global s_vm0 global s_vm1 vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file) if len(vip_s_vm_cfg_lst) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst)) s_vm0 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0) s_vm1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1) test_util.test_logger("force stop both hosts [%s] and [%s]" % (s_vm0.ip_, s_vm1.ip_)) test_stub.stop_host(s_vm0, test_lib.all_scenario_config, 'cold') test_stub.stop_host(s_vm1, test_lib.all_scenario_config, 'cold') time.sleep(5) test_stub.start_host(s_vm0, test_lib.all_scenario_config) test_stub.start_host(s_vm1, test_lib.all_scenario_config) test_stub.recover_vlan_in_host(s_vm0.ip_, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.recover_vlan_in_host(s_vm1.ip_, test_lib.all_scenario_config, test_lib.deploy_config) vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file, 3) if len(vip_s_vm_cfg_lst_new) != 1: test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new)) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]]) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list test_util.test_skip("2 hosts down at the same time is not support") test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("force stop host [%s]" % (mn_host_list[index].ip_)) test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config, 'cold') test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list test_util.test_skip("2 hosts down at the same time is not support") test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (mn_host_list[index].ip_)) test_stub.shutdown_host_network(mn_host_list[index], test_lib.all_scenario_config) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): if test_lib.scenario_config == None or test_lib.scenario_file == None: test_util.test_fail('Suite Setup Fail without scenario') if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists( test_lib.scenario_file): scenario_operations.deploy_scenario(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config) test_util.test_skip('Suite Setup Success') if test_lib.scenario_config != None and test_lib.scenario_destroy != None: scenario_operations.destroy_scenario(test_lib.all_scenario_config, test_lib.scenario_destroy) nic_name = "eth0" if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists( test_lib.scenario_file): nic_name = "zsn0" #This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine. #no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases. linux.create_vlan_eth(nic_name, 10) linux.create_vlan_eth(nic_name, 11) #If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run() test_lib.setup_plan.deploy_test_agent() cmd = host_plugin.CreateVlanDeviceCmd() cmd.ethname = nic_name cmd.vlan = 10 cmd2 = host_plugin.CreateVlanDeviceCmd() cmd2.ethname = nic_name cmd2.vlan = 11 testHosts = test_lib.lib_get_all_hosts_from_plan() if type(testHosts) != type([]): testHosts = [testHosts] for host in testHosts: http.json_dump_post( testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd) http.json_dump_post( testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd2) if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-nfs.xml"], \ ["scenario-config-storage-separate-nfs.xml"]): config_json = os.environ.get('configJsonSepStor') elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-nonmon-ceph.xml"], \ ["scenario-config-storage-separate-ceph.xml"]): config_json = os.environ.get('configJsonNonMon') elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-sep-pub-man.xml"], \ ["scenario-config-nfs-sep-man.xml", \ "scenario-config-nfs-sep-pub.xml"]): config_json = os.environ.get('configJsonSepPub') elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-ceph-3-nets-sep.xml"], \ ["scenario-config-ceph-sep-man.xml", \ "scenario-config-ceph-sep-pub.xml", \ "scenario-config-ceph-3-nets-sep.xml"]): config_json = os.environ.get('configJsonSepPub') elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-fusionstor-3-nets-sep.xml"], \ ["scenario-config-fusionstor-3-nets-sep.xml"]): config_json = os.environ.get('configJson3Net') elif test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-mul-net-pubs.xml"], \ ["scenario-config-nfs-sep-man.xml", \ "scenario-config-nfs-sep-pub.xml"]): config_json = os.environ.get('configJsonAllOne') else: config_json = os.environ.get('configJson') ha_deploy_tool = os.environ.get('zstackHaInstaller') mn_img = os.environ.get('mnImage') test_stub.deploy_ha_env(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config, config_json, ha_deploy_tool, mn_img) #if os.path.basename(os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip() == "scenario-config-vpc-ceph-3-sites.xml": # test_util.test_logger("@@@DEBUG->IS VPC CEPH@@@") # old_mn_ip = os.environ['zstackHaVip'] # test_stub.auto_set_mn_ip(test_lib.scenario_file) # cmd = 'sed -i "s/%s/%s/g" %s' %(old_mn_ip, os.environ['zstackHaVip'], EXTRA_SUITE_SETUP_SCRIPT) # os.system(cmd) #node_operations.wait_for_management_server_start(600) test_stub.wrapper_of_wait_for_management_server_start( 600, EXTRA_SUITE_SETUP_SCRIPT) test_util.test_logger("@@@DEBUG->suite_setup@@@ os\.environ\[\'ZSTACK_BUILT_IN_HTTP_SERVER_IP\'\]=%s; os\.environ\[\'zstackHaVip\'\]=%s" \ %(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], os.environ['zstackHaVip']) ) ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", os.environ.get('zstackHaVip'), 'root', 'password') if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT): os.system("bash %s" % EXTRA_SUITE_SETUP_SCRIPT) deploy_operations.deploy_initial_database(test_lib.deploy_config, test_lib.all_scenario_config, test_lib.scenario_file) for host in testHosts: os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_)) #if test_lib.lib_get_ha_selffencer_maxattempts() != None: # test_lib.lib_set_ha_selffencer_maxattempts('60') # test_lib.lib_set_ha_selffencer_storagechecker_timeout('60') test_lib.lib_set_primary_storage_imagecache_gc_interval(1) test_lib.lib_set_reserved_memory('8G') if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-sep-pub-man.xml"], ["scenario-config-nfs-sep-pub.xml"]) or \ test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-ceph-3-nets-sep.xml"], ["scenario-config-ceph-sep-pub.xml"]) or \ test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-fusionstor-3-nets-sep.xml"], ["scenario-config-fusionstor-3-nets-sep.xml"]): add_ps_network_gateway_sys_tag() test_util.test_pass('Suite Setup Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("destroy mn vm round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger("destroy mn vm on host [%s]" % mn_host[0].ip_) test_stub.destroy_mn_vm(mn_host[0], test_lib.all_scenario_config) new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_logger("mn vm was destroyed successfully") else: test_util.test_fail("mn vm was not destroyed successfully") test_util.test_logger( "wait for 20 seconds to see if management node VM starts on one host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip != mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s" ) break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") if os.path.basename( os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip( ) == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global ha_vm global mn_host ha_vm = test_stub.create_ha_vm() ha_vm.check() ha_vm_host = test_lib.lib_get_vm_host(ha_vm.vm) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) if ha_vm_host.managementIp != mn_host[0].ip_: conditions = res_ops.gen_query_conditions('managementIp', '=', mn_host[0].ip_) host = res_ops.query_resource(res_ops.HOST, conditions) ha_vm_host.migrate(host[0].uuid) test_util.test_logger("force shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold') test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) conditions = res_ops.gen_query_conditions('managementIp', '=', mn_host[0].ip_) host = res_ops.query_resource(res_ops.HOST, conditions) if host[0].status != "Connected": test_util.test_fail("Target host:%s is not connected as expected." % (host[0].uuid)) ha_vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list ha_vm = test_stub.create_ha_vm() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("force stop host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config, 'cold') need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.return_pass_ahead_if_3sites("TEST PASS") ha_vm.set_state(vm_header.RUNNING) ha_vm.check() test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) == 0: test_util.test_fail('MN VM has been murdered, expected result should not be impacted when the separated network is down.') for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip test_util.test_logger( "shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=False) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[ 0].ip_ or new_mn_host_ip == mn_host[0].managementIp_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 120s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected( exclude_host=[mn_host_list[need_recover_mn_host_list[0]]]) test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host for i in range(0, 10): test_util.test_logger("force shutdown host round %s" % (i)) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger( "force shutdown host [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.stop_host(mn_host[0], test_lib.all_scenario_config, 'cold') test_util.test_logger( "wait for 40 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or new_mn_host_ip == mn_host[0].ip_: test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s" ) break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host[0]]) test_stub.ensure_bss_host_connected_from_stop( test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() if os.path.basename( os.environ.get('WOODPECKER_SCENARIO_CONFIG_FILE')).strip( ) == "scenario-config-vpc-ceph-3-sites.xml": pass else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_stub.recover_host(mn_host[0], test_lib.all_scenario_config, test_lib.deploy_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.return_skip_ahead_if_3sites( "Skip the test in vpc ceph because create vm with vyos is not supported." ) test_stub.skip_if_vr_not_vyos("vr") mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) vm = test_stub.create_basic_vm() vm.check() ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop") vm.set_state(vm_header.RUNNING) vm.check() for host in mn_host_list: test_util.test_logger("shutdown host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.return_pass_ahead_if_3sites("TEST PASS") test_util.test_logger("Delay 60s and then check if the vm is running") time.sleep(180) if test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 300): vm.update() vm.check() vm.destroy() else: test_util.test_fail( "ha vm has not changed to running after 2 hosts recover with 300s") test_util.test_pass('Check Never Stop VM Test Success')
def test(): if test_lib.scenario_config == None or test_lib.scenario_file ==None: test_util.test_fail('Suite Setup Fail without scenario') if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists(test_lib.scenario_file): scenario_operations.deploy_scenario(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config) test_util.test_skip('Suite Setup Success') if test_lib.scenario_config != None and test_lib.scenario_destroy != None: scenario_operations.destroy_scenario(test_lib.all_scenario_config, test_lib.scenario_destroy) nic_name = "eth0" if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file): nic_name = "zsn0" #This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine. #no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases. linux.create_vlan_eth(nic_name, 10) linux.create_vlan_eth(nic_name, 11) #If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run() test_lib.setup_plan.deploy_test_agent() cmd = host_plugin.CreateVlanDeviceCmd() cmd.ethname = nic_name cmd.vlan = 10 cmd2 = host_plugin.CreateVlanDeviceCmd() cmd2.ethname = nic_name cmd2.vlan = 11 testHosts = test_lib.lib_get_all_hosts_from_plan() if type(testHosts) != type([]): testHosts = [testHosts] for host in testHosts: http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd) http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd2) test_stub.deploy_2ha(test_lib.all_scenario_config, test_lib.scenario_file) mn_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0).ip_ mn_ip2 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1).ip_ host_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 2).ip_ test_stub.recover_vlan_in_host(host_ip1, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.wrapper_of_wait_for_management_server_start(600, EXTRA_SUITE_SETUP_SCRIPT) test_util.test_logger("@@@DEBUG->suite_setup@@@ os\.environ\[\'ZSTACK_BUILT_IN_HTTP_SERVER_IP\'\]=%s; os\.environ\[\'zstackHaVip\'\]=%s" \ %(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], os.environ['zstackHaVip']) ) ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip1, 'root', 'password') ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip2, 'root', 'password') if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT): os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip1)) os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip2)) deploy_operations.deploy_initial_database(test_lib.deploy_config, test_lib.all_scenario_config, test_lib.scenario_file) for host in testHosts: os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_)) test_lib.lib_set_primary_storage_imagecache_gc_interval(1) #test_lib.lib_set_reserved_memory('1G') test_util.test_pass('Suite Setup Success')
def test(): if test_lib.scenario_config == None or test_lib.scenario_file ==None: test_util.test_fail('Suite Setup Fail without scenario') if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists(test_lib.scenario_file): scenario_operations.deploy_scenario(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config) test_util.test_skip('Suite Setup Success') if test_lib.scenario_config != None and test_lib.scenario_destroy != None: scenario_operations.destroy_scenario(test_lib.all_scenario_config, test_lib.scenario_destroy) nic_name = "eth0" if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file): nic_name = "zsn0" #This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine. #no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases. linux.create_vlan_eth(nic_name, 10) linux.create_vlan_eth(nic_name, 11) #If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run() test_lib.setup_plan.deploy_test_agent() cmd = host_plugin.CreateVlanDeviceCmd() cmd.ethname = nic_name cmd.vlan = 10 cmd2 = host_plugin.CreateVlanDeviceCmd() cmd2.ethname = nic_name cmd2.vlan = 11 testHosts = test_lib.lib_get_all_hosts_from_plan() if type(testHosts) != type([]): testHosts = [testHosts] for host in testHosts: http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd) http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd2) test_stub.deploy_2ha(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config) mn_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 0).ip_ mn_ip2 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 1).ip_ if not xmlobject.has_element(test_lib.deploy_config, 'backupStorages.miniBackupStorage'): host_ip1 = test_stub.get_host_by_index_in_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, 2).ip_ test_stub.recover_vlan_in_host(host_ip1, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.wrapper_of_wait_for_management_server_start(600, EXTRA_SUITE_SETUP_SCRIPT) test_util.test_logger("@@@DEBUG->suite_setup@@@ os\.environ\[\'ZSTACK_BUILT_IN_HTTP_SERVER_IP\'\]=%s; os\.environ\[\'zstackHaVip\'\]=%s" \ %(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], os.environ['zstackHaVip']) ) ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip1, 'root', 'password') ssh.scp_file("/home/license-10host-10days-hp.txt", "/home/license-10host-10days-hp.txt", mn_ip2, 'root', 'password') if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT): os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip1)) os.system("bash %s %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ip2)) deploy_operations.deploy_initial_database(test_lib.deploy_config, test_lib.all_scenario_config, test_lib.scenario_file) for host in testHosts: os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_)) test_lib.lib_set_primary_storage_imagecache_gc_interval(1) #test_lib.lib_set_reserved_memory('1G') if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-local-ps.xml"], ["scenario-config-upgrade-3.1.1.xml"]): cmd = r"sed -i '$a\172.20.198.8 rsync.repo.zstack.io' /etc/hosts" ssh.execute(cmd, mn_ip1, "root", "password", False, 22) ssh.execute(cmd, mn_ip2, "root", "password", False, 22) test_util.test_pass('Suite Setup Success')
def test(): global vm global mn_host test_stub.skip_if_scenario_not_multiple_networks() mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_host)) test_util.test_logger( "shutdown host's network [%s] that mn vm is running on" % (mn_host[0].ip_)) test_stub.shutdown_host_network(mn_host[0], test_lib.all_scenario_config, downMagt=True) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "" or (new_mn_host_ip != mn_host[0].ip_ and new_mn_host_ip != mn_host[0].managementIp_): test_util.test_fail( "management network down, mn host should not changed. Expected on [%s] while is on [%s]" % (mn_host[0].ip_, new_mn_host_ip)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 120s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start() test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.reopen_host_network(mn_host[0], test_lib.all_scenario_config) test_stub.wait_for_mn_ha_ready(test_lib.all_scenario_config, test_lib.scenario_file) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')