def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_util.test_logger(
"wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s" % (node_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(600)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network: %s" % (node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s" % (l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s, and create vm" % (node_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test with one node network down and recovery Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm1
global vm2
global node_ip
vm1 = test_stub.create_basic_vm()
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop rabbitmq on node: %s" % (node_ip))
cmd = "rabbitmqctl stop_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
time.sleep(120)
test_stub.exercise_connection(600)
test_util.test_logger("create vm to check if it still works")
vm1.check()
vm1.destroy()
vm2 = test_stub.create_basic_vm()
vm2.check()
vm2.destroy()
cmd = "rabbitmqctl start_app"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
time.sleep(60)
test_stub.exercise_connection(600)
test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test():
global vm1
global vm2
global node_ip
vm1 = test_stub.create_basic_vm()
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop rabbitmq on node: %s" % (node_ip))
cmd = "rabbitmqctl stop_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(120)
test_stub.exercise_connection(1200)
test_util.test_logger("create vm to check if it still works")
vm1.check()
vm1.destroy()
vm2 = test_stub.create_basic_vm()
vm2.check()
vm2.destroy()
cmd = "rabbitmqctl start_app"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(60)
test_stub.exercise_connection(1200)
test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network for a small while: %s" %
(node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface,
l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover mysql")
cmd = "service mysql start"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_stub.exercise_connection(600)
test_util.test_logger("check create vm")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass(
'Create VM Test with one node network down for a while Success')
def test():
global vm
global node1_ip
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple',
'simple_reboot_vm_scheduler',
start_date + 60, 30)
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
test_util.test_logger(
"wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
for j in range(0, 6):
if test_lib.lib_find_in_remote_management_server_log(
node1_ip, host_username, host_password,
start_date + 60 + 30 * i + j,
'[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg',
vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(
node2_ip, host_username, host_password,
start_date + 60 + 30 * i + j,
'[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg',
vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail(
'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times'
% (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
time.sleep(180)
test_stub.exercise_connection(600)
test_util.test_pass('Scheduler Test Success')
def error_cleanup():
global vm
if vm:
try:
vm.destroy()
except:
pass
test_util.test_logger("recover node: %s" % (node_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(1200)
def error_cleanup():
global vm
if vm:
try:
vm.destroy()
except:
pass
test_util.test_logger("recover node: %s" % (node_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(600)
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
test_util.test_dsc('Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail')
cmd = "rm /var/lib/zstack/ha/ha.yaml"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
cmd = "zstack-ctl recover_ha"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
if rsp == False:
test_util.test_logger("Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False")
else:
test_util.test_fail('Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha')
test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass')
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30)
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_pass('Scheduler Test Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network for a small while: %s" % (node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface, l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover mysql")
cmd = "service mysql start"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_stub.exercise_connection(600)
test_util.test_logger("check create vm")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test with one node network down for a while Success')
def error_cleanup():
global vm1
global vm2
global node_ip
cmd = "rabbitmqctl start_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(60)
test_stub.exercise_connection(1200)
if vm1:
try:
vm1.destroy()
except:
pass
if vm2:
try:
vm2.destroy()
except:
pass
def test():
global vm
global node1_ip
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30)
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
for j in range(0, 6):
if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
time.sleep(180)
test_stub.exercise_connection(600)
test_util.test_pass('Scheduler Test Success')
def error_cleanup():
global vm1
global vm2
global node_ip
cmd = "rabbitmqctl start_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
time.sleep(60)
test_stub.exercise_connection(600)
if vm1:
try:
vm1.destroy()
except:
pass
if vm2:
try:
vm2.destroy()
except:
pass
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(1200)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s" % (node_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(1200)
test_util.test_pass('Create VM Test Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node2_ip))
test_util.test_dsc(
'Delete /var/lib/zstack/ha/ha.yaml, recover ha with zstack-ctl recover_ha, expect to fail'
)
cmd = "rm /var/lib/zstack/ha/ha.yaml"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
cmd = "zstack-ctl recover_ha"
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
if rsp == False:
test_util.test_logger(
"Cannot recover ha without /var/lib/zstack/ha/ha.yaml when use zstack-ctl recover_ha, expect to False"
)
else:
test_util.test_fail(
'Expect to False, but get the different result when recover ha without /var/lib/zstack/ha/ha.yaml by using zstack-ctl recover_ha'
)
test_util.test_dsc('Recover with zstack-ctl install_ha, expect to pass')
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
global vm
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple',
'simple_reboot_vm_scheduler',
start_date + 60, 30)
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password,
cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username,
host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
if test_lib.lib_find_in_remote_management_server_log(
node1_ip, host_username, host_password,
start_date + 60 + 30 * i,
'[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg',
vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(
node2_ip, host_username, host_password,
start_date + 60 + 30 * i,
'[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg',
vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail(
'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times'
% (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_pass('Scheduler Test Success')
