def test(): global vm test_stub.return_skip_ahead_if_3sites("TEST SKIP") #ensure mn works normally. vm0 = test_stub.create_basic_vm() vm0.check() vm0.destroy() ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(ori_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(ori_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) ori_host = ori_host[0] target_host = None for host in mn_host_list: if host.ip_ != ori_host.ip_: target_host = host break if target_host: test_stub.migrate_mn_vm(ori_host, target_host, test_lib.all_scenario_config) cur_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(cur_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(cur_host)) cur_host = cur_host[0] if cur_host.ip_ != target_host.ip_: test_util.test_fail("mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]" % (target_host.ip_, cur_host.ip_)) else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Migrate MN VM Test Success') else: test_util.test_fail("there is no host for mn vm to migrate to")
def test(): global vm test_stub.return_skip_ahead_if_3sites("TEST SKIP") ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(ori_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(ori_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) ori_host = ori_host[0] target_host = None for host in mn_host_list: if host.ip_ != ori_host.ip_: target_host = host break if target_host: test_stub.migrate_mn_vm(ori_host, target_host, test_lib.all_scenario_config) cur_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(cur_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(cur_host)) cur_host = cur_host[0] if cur_host.ip_ != target_host.ip_: test_util.test_fail( "mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]" % (target_host.ip_, cur_host.ip_)) else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Migrate MN VM Test Success') else: test_util.test_fail("there is no host for mn vm to migrate to")
def test(): global vm global test_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) for mn_host in mn_host_list: if mn_host.ip_ != mn_vm_host[0].ip_: test_host = mn_host break if not test_host: test_util.test_fail('there is only one mn host') test_util.test_logger( "stop consul for host [%s] that mn vm is running on" % (test_host.ip_)) test_stub.stop_consul(test_host, test_lib.all_scenario_config) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail( "management node VM was destroyed after stopping another host's consul" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM starts on more than one host after stopping another host's consul" ) except: test_util.test_fail( "management node VM was destroyed after stopping another host's consul" ) if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail( 'management node VM starts on another host when another host stopped consul' ) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global test_host test_stub.skip_if_scenario_is_multiple_networks() mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) for mn_host in mn_host_list: if mn_host.ip_ != mn_vm_host[0].ip_: test_host = mn_host break if not test_host: test_util.test_fail('there is only one mn host') test_util.test_logger( "shutdown host's network [%s] that mn vm is not running on" % (test_host.ip_)) test_stub.shutdown_host_network(test_host, test_lib.all_scenario_config) test_util.test_logger("wait for the target host totally shutdown") test_stub.ensure_host_disconnected(test_host, 300) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail( "management node VM was destroyed after another host network down" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM starts on more than one host after another host network down" ) except: test_util.test_fail( "management node VM was destroyed after another host network down") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail( 'management node VM starts on another host when its former host was not down' ) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() #test_stub.ensure_hosts_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global test_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) for mn_host in mn_host_list: if mn_host.ip_ != mn_vm_host[0].ip_: test_host = mn_host break if not test_host: test_util.test_fail('there is only one mn host') test_util.test_logger( "shutdown host's network [%s] that mn vm is not running on" % (test_host.ip_)) test_stub.shutdown_host_network(test_host, test_lib.all_scenario_config) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on another host" ) time.sleep(20) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail( "management node VM was destroyed after another host network down" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM starts on more than one host after another host network down" ) except: test_util.test_fail( "management node VM was destroyed after another host network down") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail( 'management node VM starts on another host when its former host was not down' ) else: try: vm = test_stub.create_basic_vm() vm.check() vm.destroy() except: test_util.test_fail('Fail to create vm after mn is ready') test_util.test_pass('Create VM Test Success')
def test(): global vm for i in range(0, 10): test_util.test_logger("migrate MN VM round %s" % (i)) ori_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(ori_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(ori_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) ori_host = ori_host[0] test_util.test_logger("ori_host%s, %s" % (ori_host.ip_, i)) target_host = None for host in mn_host_list: if host.ip_ != ori_host.ip_: target_host = host test_util.test_logger("target_host%s, %s" % (host.ip_, i)) break if target_host: test_stub.migrate_mn_vm(ori_host, target_host, test_lib.all_scenario_config) time.sleep(10) cur_host = test_stub.get_host_by_mn_vm( test_lib.all_scenario_config, test_lib.scenario_file) if len(cur_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(cur_host)) cur_host = cur_host[0] test_util.test_logger("cur_host%s ,%s" % (cur_host.ip_, i)) if cur_host.ip_ != target_host.ip_: test_util.test_fail( "mn vm should be migrated to host[ %s ], but it was migrated to host[ %s ]" % (target_host.ip_, cur_host.ip_)) else: vm = test_stub.create_basic_vm() vm.check() vm.destroy() else: test_util.test_fail("there is no host for mn vm to migrate to") test_util.test_pass('Migrate MN VM Test Success')
def test(): global vm global test_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) for mn_host in mn_host_list: if mn_host.ip_ != mn_vm_host[0].ip_: test_host = mn_host break if not test_host: test_util.test_fail('there is only one mn host') test_util.test_logger("shutdown host [%s] that mn vm is not running on" % (test_host.ip_)) test_stub.stop_host(test_host, test_lib.all_scenario_config) test_util.test_logger("wait for non-mn host shutdown totally") test_stub.ensure_host_disconnected(test_host, 300) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail("management node VM was destroyed after another host down") elif len(new_mn_host) > 1: test_util.test_fail("management node VM starts on more than one host after another host down") except: test_util.test_fail("management node VM was destroyed after another host down") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail('management node VM starts on another host when its former host was not down') test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() #test_stub.ensure_hosts_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global test_host mn_vm_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_vm_host) != 1: test_util.test_fail('MN VM is running on %d host(s)' % len(mn_vm_host)) mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) for mn_host in mn_host_list: if mn_host.ip_ != mn_vm_host[0].ip_: test_host = mn_host break if not test_host: test_util.test_fail('there is only one mn host') test_util.test_logger("force shutdown host [%s] that mn vm is not running on" % (test_host.ip_)) test_stub.stop_host(test_host, test_lib.all_scenario_config, 'cold') test_util.test_logger("wait the target non-mn host force shutdown totally") test_stub.ensure_host_disconnected(test_host, 300) try: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 0: test_util.test_fail("management node VM was destroyed after another host down") elif len(new_mn_host) > 1: test_util.test_fail("management node VM starts on more than one host after another host down") except: test_util.test_fail("management node VM was destroyed after another host down") if new_mn_host[0].ip_ != mn_vm_host[0].ip_: test_util.test_fail('management node VM starts on another host when its former host was not down') test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() #test_stub.ensure_hosts_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected( exclude_host=[mn_host_list[need_recover_mn_host_list[0]]]) test_stub.ensure_bss_host_connected_from_sep_net_down( test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("shutdown host [%s]" % (mn_host_list[index].ip_)) test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after recover one mn host") test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list test_util.test_skip("2 hosts down at the same time is not support") test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (mn_host_list[index].ip_)) test_stub.shutdown_host_network(mn_host_list[index], test_lib.all_scenario_config) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") test_util.test_logger("wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_pss_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global test_mn_host_list test_util.test_skip("2 hosts down at the same time is not support") test_stub.skip_if_scenario_is_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for index in test_mn_host_list: test_util.test_logger("force stop host [%s]" % (mn_host_list[index].ip_)) test_stub.stop_host(mn_host_list[index], test_lib.all_scenario_config, 'cold') test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) test_util.test_logger("recover host [%s]" % (mn_host_list[test_mn_host_list[-1]])) test_stub.recover_host(mn_host_list[test_mn_host_list[-1]], test_lib.all_scenario_config, test_lib.deploy_config) test_mn_host_list.pop() test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) test_util.test_logger( "wait for 5 minutes to see if management node starts again") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_pss_connected() test_stub.ensure_bss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_vr_not_vyos("vr") mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) vm = test_stub.create_basic_vm() vm.check() ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop") vm.set_state(vm_header.RUNNING) vm.check() for host in mn_host_list: test_util.test_logger("shutdown host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger( "wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader( test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail( "management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger( "management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail( "management node VM does not run after its former host down for 30s" ) elif len(new_mn_host) > 1: test_util.test_fail( "management node VM runs on more than one host after its former host down" ) try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail( "management node does not recover after MN VM is running") test_util.test_logger("Delay 60s and then check if the vm is running") time.sleep(180) if test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 300): vm.update() vm.check() vm.destroy() else: test_util.test_fail( "ha vm has not changed to running after 2 hosts recover with 300s") test_util.test_pass('Check Never Stop VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("force stop host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config, 'cold') need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail("management node does not recover after MN VM is running") test_stub.ensure_hosts_connected(exclude_host=[mn_host_list[need_recover_mn_host_list[0]]]) test_stub.ensure_bss_host_connected_from_stop(test_lib.scenario_file, test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list ha_vm = test_stub.create_ha_vm() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("force stop host [%s]" % (host.ip_)) test_stub.stop_host(host, test_lib.all_scenario_config, 'cold') need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 60 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 30s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 30s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.return_pass_ahead_if_3sites("TEST PASS") ha_vm.set_state(vm_header.RUNNING) ha_vm.check() test_util.test_logger("try to create vm, timeout is 30s") time_out = 30 while time_out > 0: try: vm = test_stub.create_basic_vm() break except: time.sleep(1) time_out -= 1 if time_out == 0: test_util.test_fail('Fail to create vm after mn is ready') vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) != 0: test_util.test_fail('MN VM is still running on %d host(s)' % len(mn_host)) for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected(exclude_host=[mn_host_list[need_recover_mn_host_list[0]]]) test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_scenario_not_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=True) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) == 0: test_util.test_fail('MN VM has been murdered!!! Expected should not be impacted when management network is down.') for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) for index in need_recover_mn_host_list: test_util.test_logger("recover host: %s" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list test_stub.skip_if_scenario_not_multiple_networks() mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=True) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) == 0: test_util.test_fail('MN VM has been murdered!!! Expected should not be impacted when management network is down.') for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") try: node_ops.wait_for_management_server_start(300) except: test_util.test_fail("management node does not recover after MN VM is running") for index in need_recover_mn_host_list: test_util.test_logger("recover host: %s" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=True) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm global mn_host_list global need_recover_mn_host_list global pub_mn_ip global mag_mn_ip test_stub.skip_if_scenario_not_multiple_networks() pub_mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] mag_mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = mag_mn_ip mn_host_list = test_stub.get_mn_host(test_lib.all_scenario_config, test_lib.scenario_file) mn_host_num = len(mn_host_list) test_mn_host_list = random.sample(range(mn_host_num), (mn_host_num + 1) / 2) for host in mn_host_list: test_util.test_logger("shutdown host's network [%s]" % (host.ip_)) test_stub.shutdown_host_network(host, test_lib.all_scenario_config, downMagt=False) need_recover_mn_host_list = range(mn_host_num) test_util.test_logger("wait 10s for MN VM to stop") time.sleep(10) mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(mn_host) == 0: test_util.test_fail('MN VM has been murdered, expected result should not be impacted when the separated network is down.') for index in test_mn_host_list: test_util.test_logger("recover host [%s]" % (mn_host_list[index].ip_)) test_stub.recover_host(mn_host_list[index], test_lib.all_scenario_config, test_lib.deploy_config) need_recover_mn_host_list.remove(index) test_util.test_logger("wait for 20 seconds to see if management node VM starts on any host") time.sleep(20) new_mn_host_ip = test_stub.get_host_by_consul_leader(test_lib.all_scenario_config, test_lib.scenario_file) if new_mn_host_ip == "": test_util.test_fail("management node VM not run correctly on [%s] after its former host [%s] down for 20s" % (new_mn_host_ip, mn_host_list[0].ip_)) count = 120 while count > 0: new_mn_host = test_stub.get_host_by_mn_vm(test_lib.all_scenario_config, test_lib.scenario_file) if len(new_mn_host) == 1: test_util.test_logger("management node VM run after its former host down for 120s") break elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") time.sleep(5) count -= 1 if len(new_mn_host) == 0: test_util.test_fail("management node VM does not run after its former host down for 120s") elif len(new_mn_host) > 1: test_util.test_fail("management node VM runs on more than one host after its former host down") #node_ops.wait_for_management_server_start(300) test_stub.wrapper_of_wait_for_management_server_start(600) test_stub.ensure_hosts_connected() test_stub.ensure_bss_host_connected_from_sep_net_down(test_lib.scenario_file, test_lib.all_scenario_config, downMagt=False) test_stub.ensure_bss_connected() test_stub.ensure_pss_connected() test_stub.return_pass_ahead_if_3sites("TEST PASS") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')