def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger( "wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_logger("recover node: %s" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) test_stub.exercise_connection(600) test_util.test_pass('Create VM Test Success')
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node network: %s" % (node_ip)) l2_network_interface = os.environ.get('l2ManagementNetworkInterface') cmd = "ifdown %s" % (l2_network_interface) host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_logger("recover node: %s, and create vm" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test with one node network down and recovery Success')
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm1 global vm2 global node_ip vm1 = test_stub.create_basic_vm() node_ip = os.environ.get('node1Ip') test_util.test_logger("stop rabbitmq on node: %s" % (node_ip)) cmd = "rabbitmqctl stop_app" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(120) test_stub.exercise_connection(600) test_util.test_logger("create vm to check if it still works") vm1.check() vm1.destroy() vm2 = test_stub.create_basic_vm() vm2.check() vm2.destroy() cmd = "rabbitmqctl start_app" rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(60) test_stub.exercise_connection(600) test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test(): global vm1 global vm2 global node_ip vm1 = test_stub.create_basic_vm() node_ip = os.environ.get('node1Ip') test_util.test_logger("stop rabbitmq on node: %s" % (node_ip)) cmd = "rabbitmqctl stop_app" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(120) test_stub.exercise_connection(1200) test_util.test_logger("create vm to check if it still works") vm1.check() vm1.destroy() vm2 = test_stub.create_basic_vm() vm2.check() vm2.destroy() cmd = "rabbitmqctl start_app" rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(60) test_stub.exercise_connection(1200) test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node network for a small while: %s" % (node_ip)) l2_network_interface = os.environ.get('l2ManagementNetworkInterface') cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface, l2_network_interface) host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover mysql") cmd = "service mysql start" rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_stub.exercise_connection(600) test_util.test_logger("check create vm") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass( 'Create VM Test with one node network down for a while Success')
def test(): global vm global node1_ip vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date + 60, 30) node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger( "wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): for j in range(0, 6): if test_lib.lib_find_in_remote_management_server_log( node1_ip, host_username, host_password, start_date + 60 + 30 * i + j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log( node2_ip, host_username, host_password, start_date + 60 + 30 * i + j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail( 'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) test_stub.exercise_connection(600) test_util.test_pass('Scheduler Test Success')
def error_cleanup(): global vm if vm: try: vm.destroy() except: pass test_util.test_logger("recover node: %s" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) test_stub.exercise_connection(1200)
def error_cleanup(): global vm if vm: try: vm.destroy() except: pass test_util.test_logger("recover node: %s" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) test_stub.exercise_connection(600)
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) test_util.test_dsc('Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail') cmd = "rm /var/lib/zstack/ha/ha.yaml" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) cmd = "zstack-ctl recover_ha" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) if rsp == False: test_util.test_logger("Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False") else: test_util.test_fail('Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha') test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass') cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Scheduler Test Success')
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node network for a small while: %s" % (node_ip)) l2_network_interface = os.environ.get('l2ManagementNetworkInterface') cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface, l2_network_interface) host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover mysql") cmd = "service mysql start" rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_stub.exercise_connection(600) test_util.test_logger("check create vm") vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test with one node network down for a while Success')
def error_cleanup(): global vm1 global vm2 global node_ip cmd = "rabbitmqctl start_app" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(60) test_stub.exercise_connection(1200) if vm1: try: vm1.destroy() except: pass if vm2: try: vm2.destroy() except: pass
def test(): global vm global node1_ip vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30) node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): for j in range(0, 6): if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) test_stub.exercise_connection(600) test_util.test_pass('Scheduler Test Success')
def error_cleanup(): global vm1 global vm2 global node_ip cmd = "rabbitmqctl start_app" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) time.sleep(60) test_stub.exercise_connection(600) if vm1: try: vm1.destroy() except: pass if vm2: try: vm2.destroy() except: pass
def test(): global vm global node_ip node_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(1200) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_logger("recover node: %s" % (node_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip)) time.sleep(180) test_stub.exercise_connection(1200) test_util.test_pass('Create VM Test Success')
def test(): global vm cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) test_util.test_dsc( 'Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail' ) cmd = "rm /var/lib/zstack/ha/ha.yaml" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) cmd = "zstack-ctl recover_ha" rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) if rsp == False: test_util.test_logger( "Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False" ) else: test_util.test_fail( 'Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha' ) test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass') cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) vm = test_stub.create_basic_vm() vm.check() vm.destroy() test_util.test_pass('Create VM Test Success')
def test(): global vm vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date + 60, 30) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): if test_lib.lib_find_in_remote_management_server_log( node1_ip, host_username, host_password, start_date + 60 + 30 * i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log( node2_ip, host_username, host_password, start_date + 60 + 30 * i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail( 'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Scheduler Test Success')