def test(): global vm global schd vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) test_util.test_logger('Setup stop and start VM scheduler') schd = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', 0, 1) actual_startDate = time.mktime(time.strptime(schd.startTime, '%b %d, %Y %H:%M:%S %p')) if actual_startDate != start_date and actual_startDate != start_date + 1: test_util.test_fail('startDate is expectd to set to now, which should be around %s' % (start_date)) test_stub.sleep_util(start_date+58) start_msg_mismatch = 0 for i in range(1, 58): if not test_lib.lib_find_in_local_management_server_log(start_date+i, '[msg send]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn('StopVmInstanceMsg is expected to execute at %s' % (start_date+i)) if start_msg_mismatch > 5: test_util.test_fail('%s of 58 StartVmInstanceMsg not executed at expected timestamp' % (start_msg_mismatch)) schd_ops.delete_scheduler(schd.uuid) try: vm.destroy() except: test_util.test_logger('expected exception when destroy VM since too many queued task') test_util.test_pass('Create Simple VM Start Scheduler Success')
def error_cleanup(): global schd test_lib.lib_error_cleanup(test_obj_dict) if schd: schd_ops.delete_scheduler(schd.uuid)
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date + 60, 120) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date + 120, 120) test_stub.sleep_util(start_date + 60 + 120 * 2 + 5) schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) time.sleep(60) vm.update() vm_state = vm.get_vm().state for i in range(3, 5): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120 * i + 5) vm.update() test_util.test_logger( 'check VM status at %s, VM is expected to stay in state %s' % (start_date + 60 + 120 * i + 5, vm_state)) if vm.get_vm().state != vm_state: test_util.test_fail('VM is expected to stay in state %s' % (vm_state)) vm.destroy() test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date-60, 120) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date-120, 120) test_stub.sleep_util(start_date+58) vm.update() if not test_lib.lib_is_vm_running(vm.get_vm()): test_util.test_fail('VM is expected to run until stop vm scheduler start_date') for i in range(0, 4): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120*i + 5) test_util.test_logger('check VM status at %s, VM is expected to stop' % (start_date + 60 + 120*i + 5)) vm.update() if vm.get_vm().state != 'Stopping' and vm.get_vm().state != 'Stopped': test_util.test_fail('VM is expected to stop') test_stub.sleep_util(start_date + 60 + 120*i + 65) test_util.test_logger('check VM status at %s, VM is expected to start' % (start_date + 60 + 120*i + 65)) vm.update() if vm.get_vm().state != 'Starting' and vm.get_vm().state != 'Running': test_util.test_fail('VM is expected to start') schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.destroy() test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def test(): global vm global schd vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date + 60, 120) test_stub.sleep_util(start_date + 58) for i in range(0, 58): if test_lib.lib_find_in_local_management_server_log( start_date + i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail( 'VM is expected to reboot start from %s' (start_date + 60)) test_stub.sleep_util(start_date + 59) if not test_lib.lib_wait_target_down(vm.get_vm().vmNics[0].ip, '22', 120): test_util.test_fail( 'VM: %s is not reboot in 120 seconds. Fail to reboot it with scheduler. ' % vm.get_vm().uuid) if not test_lib.lib_find_in_local_management_server_log( start_date + 60, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail('VM is expected to reboot start from %s' % (start_date + 60)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Create Simple VM Reboot Scheduler Success')
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date+60, 120) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date+120, 120) test_stub.sleep_util(start_date + 60 + 120*2 + 5) schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) time.sleep(60) vm.update() vm_state = vm.get_vm().state for i in range(3, 5): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120*i + 5) vm.update() test_util.test_logger('check VM status at %s, VM is expected to stay in state %s' % (start_date + 60 + 120*i + 5, vm_state)) if vm.get_vm().state != vm_state: test_util.test_fail('VM is expected to stay in state %s' % (vm_state)) vm.destroy() test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def error_cleanup(): global vm global schd if vm: vm.destroy() if schd: schd_ops.delete_scheduler(schd.uuid)
def test(): global vm global node1_ip vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date + 60, 30) node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger( "wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): for j in range(0, 6): if test_lib.lib_find_in_remote_management_server_log( node1_ip, host_username, host_password, start_date + 60 + 30 * i + j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log( node2_ip, host_username, host_password, start_date + 60 + 30 * i + j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail( 'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) test_stub.exercise_connection(600) test_util.test_pass('Scheduler Test Success')
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date+10, 20) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date+20, 20) test_stub.sleep_util(start_date+45) test_util.test_dsc('check scheduler state after create scheduler') check_scheduler_state(schd1, 'Enabled') check_scheduler_state(schd2, 'Enabled') if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', start_date+10): test_util.test_fail('StopVmInstanceJob not executed at expected timestamp range') if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', start_date+20): test_util.test_fail('StartVmInstanceJob not executed at expected timestamp range') schd_ops.change_scheduler_state(schd1.uuid, 'disable') schd_ops.change_scheduler_state(schd2.uuid, 'disable') current_time = int(time.time()) except_start_time = start_date + 20 * (((current_time - start_date) % 20) + 1) test_stub.sleep_util(except_start_time+45) test_util.test_dsc('check scheduler state after pause scheduler') check_scheduler_state(schd1, 'Disabled') check_scheduler_state(schd2, 'Disabled') if check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time+10): test_util.test_fail('StopVmInstanceJob executed at unexpected timestamp range') if check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time+20): test_util.test_fail('StartVmInstanceJob executed at unexpected timestamp range') schd_ops.change_scheduler_state(schd1.uuid, 'enable') schd_ops.change_scheduler_state(schd2.uuid, 'enable') current_time = int(time.time()) except_start_time = start_date + 20 * (((current_time - start_date) % 20) + 1) test_stub.sleep_util(except_start_time+45) test_util.test_dsc('check scheduler state after resume scheduler') check_scheduler_state(schd1, 'Enabled') check_scheduler_state(schd2, 'Enabled') if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time+10): test_util.test_fail('StopVmInstanceJob not executed at expected timestamp range') if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time+20): test_util.test_fail('StartVmInstanceJob not executed at expected timestamp range') schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.destroy() test_util.test_pass('Check Scheduler State after Pause and Resume Scheduler Success')
def error_cleanup(): global vm global schd if vm: try: vm.destroy() except: test_util.test_logger('expected exception when destroy VM since too many queued task') if schd: schd_ops.delete_scheduler(schd.uuid)
def test(): global vm global schd vm = test_stub.create_vlan_vm() test_obj_dict.add_vm(vm) disk_offering = test_lib.lib_get_disk_offering_by_name( os.environ.get('smallDiskOfferingName')) volume_creation_option = test_util.VolumeOption() volume_creation_option.set_name('volume for snapshot scheduler testing') volume_creation_option.set_disk_offering_uuid(disk_offering.uuid) volume = test_stub.create_volume(volume_creation_option) test_obj_dict.add_volume(volume) volume.attach(vm) volume.detach() test_util.test_dsc('create snapshot and check') start_date = int(time.time()) sp_option = test_util.SnapshotOption() sp_option.set_name('simple_schduler_snapshot') sp_option.set_volume_uuid(volume.get_volume().uuid) schd = vol_ops.create_snapshot_scheduler( sp_option, 'simple', 'simple_create_snapshot_scheduler', start_date + 60, 120) snapshot_num = 0 for i in range(0, 3): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120 * i - 2) test_util.test_logger( 'check volume snapshot number at %s, there should be %s' % (start_date + 60 + 120 * i - 2, snapshot_num)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) snapshot_num += 1 test_stub.sleep_util(start_date + 60 + 120 * i + 60) test_util.test_logger( 'check volume snapshot number at %s, there should be %s' % (start_date + 60 + 120 * i + 65, snapshot_num + 1)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def error_cleanup(): global vm global schd global new_account test_lib.lib_error_cleanup(test_obj_dict) if schd: schd_ops.delete_scheduler(schd.uuid) if new_account: account_operations.delete_account(new_account.uuid)
def error_cleanup(): global vm global schd1 global schd2 if vm: vm.destroy() if schd1: schd_ops.delete_scheduler(schd1.uuid) if schd2: schd_ops.delete_scheduler(schd2.uuid)
def error_cleanup(): global vm global schd if vm: try: vm.destroy() except: test_util.test_logger( 'expected exception when destroy VM since too many queued task' ) if schd: schd_ops.delete_scheduler(schd.uuid)
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date + 60, 120, 3) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date + 120, 120, 2) test_stub.sleep_util(start_date + 58) vm.update() if not test_lib.lib_is_vm_running(vm.get_vm()): test_util.test_fail( 'VM is expected to run until stop vm scheduler start_date') for i in range(0, 5): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120 * i + 5) test_util.test_logger('check VM status at %s, VM is expected to stop' % (start_date + 60 + 120 * i + 5)) vm.update() if vm.get_vm().state != 'Stopping' and vm.get_vm().state != 'Stopped': test_util.test_fail('VM is expected to stop') test_stub.sleep_util(start_date + 60 + 120 * i + 65) if i >= 2: test_util.test_logger( 'check VM status at %s, VM is expected to stay stopped' % (start_date + 60 + 120 * i + 65)) vm.update() if vm.get_vm().state != 'Stopped': test_util.test_fail('VM is expected to stay stop') continue test_util.test_logger( 'check VM status at %s, VM is expected to start' % (start_date + 60 + 120 * i + 65)) vm.update() if vm.get_vm().state != 'Starting' and vm.get_vm().state != 'Running': test_util.test_fail('VM is expected to start') schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.destroy() test_util.test_pass( 'Create Simple VM Stop Start Scheduler Repeat Count Success')
def test(): global vm vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Scheduler Test Success')
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) minute = start_date / 60 % 60 % 2 if minute == 0: test_stub.sleep_util((start_date + 59) / 60 * 60) else: test_stub.sleep_util((start_date + 59) / 60 * 60 + 60) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'cron', 'cron_stop_vm_scheduler', None, None, None, '0 0/2 * * * ?') schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'cron', 'cron_start_vm_scheduler', None, None, None, '59 0/2 * * * ?') test_stub.sleep_util(start_date + 58) vm.update() if not test_lib.lib_is_vm_running(vm.get_vm()): test_util.test_fail( 'VM is expected to run until stop vm scheduler start_date') for i in range(0, 10): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120 * i + 5) test_util.test_logger('check VM status at %s, VM is expected to stop' % (start_date + 60 + 120 * i + 5)) vm.update() if vm.get_vm().state != 'Stopping' and vm.get_vm().state != 'Stopped': test_util.test_fail('VM is expected to stop') test_stub.sleep_util(start_date + 60 + 120 * i + 65) test_util.test_logger( 'check VM status at %s, VM is expected to start' % (start_date + 60 + 120 * i + 65)) vm.update() if vm.get_vm().state != 'Starting' and vm.get_vm().state != 'Running': test_util.test_fail('VM is expected to start') schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.destroy() test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def error_cleanup(): global vm global schd1 global schd2 global new_account if vm: vm.destroy() if schd1: schd_ops.delete_scheduler(schd1.uuid) if schd2: schd_ops.delete_scheduler(schd2.uuid) if new_account: account_operations.delete_account(new_account.uuid)
def test(): global vm global schd1 global schd2 global new_account vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date+30, 60) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date+60, 60) test_stub.sleep_util(start_date+130) test_util.test_dsc('check scheduler state after creating scheduler') check_scheduler_state(schd1, 'Enabled') check_scheduler_state(schd2, 'Enabled') if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', start_date+30): test_util.test_fail('StopVmInstanceJob not executed at expected timestamp range') if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', start_date+60): test_util.test_fail('StartVmInstanceJob not executed at expected timestamp range') # vm.check(); new_account = account_operations.create_account('new_account', 'password', 'Normal') res_ops.change_recource_owner(new_account.uuid, vm.vm.uuid) current_time = int(time.time()) except_start_time = start_date + 60 * (((current_time - start_date) % 60) + 1) test_stub.sleep_util(except_start_time + 130) test_util.test_dsc('check scheduler state after changing the owner of vm') check_scheduler_state(schd1, 'Disabled') check_scheduler_state(schd2, 'Disabled') if check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time+30): test_util.test_fail('StopVmInstanceJob executed at unexpected timestamp range') if check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time+60): test_util.test_fail('StartVmInstanceJob executed at unexpected timestamp range') schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.destroy() account_operations.delete_account(new_account.uuid) test_util.test_pass('Check Scheduler State after Changing VM Owner Success')
def test(): global vm global schd vm = test_stub.create_vlan_vm() test_obj_dict.add_vm(vm) disk_offering = test_lib.lib_get_disk_offering_by_name(os.environ.get('smallDiskOfferingName')) volume_creation_option = test_util.VolumeOption() volume_creation_option.set_name('volume for snapshot scheduler testing') volume_creation_option.set_disk_offering_uuid(disk_offering.uuid) volume = test_stub.create_volume(volume_creation_option) test_obj_dict.add_volume(volume) volume.attach(vm) volume.detach() test_util.test_dsc('create snapshot and check') start_date = int(time.time()) sp_option = test_util.SnapshotOption() sp_option.set_name('simple_schduler_snapshot') sp_option.set_volume_uuid(volume.get_volume().uuid) schd = vol_ops.create_snapshot_scheduler(sp_option, 'simple', 'simple_create_snapshot_scheduler', start_date+60, 120) snapshot_num = 0 for i in range(0, 3): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120*i - 2) test_util.test_logger('check volume snapshot number at %s, there should be %s' % (start_date + 60 + 120*i - 2, snapshot_num)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) snapshot_num += 1 test_stub.sleep_util(start_date + 60 + 120*i + 60) test_util.test_logger('check volume snapshot number at %s, there should be %s' % (start_date + 60 + 120*i + 65, snapshot_num+1)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def test(): global vm global node1_ip vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30) node1_ip = os.environ.get('node1Ip') node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) test_util.test_logger("wait for 2 minutes to see if http api still works well") time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): for j in range(0, 6): if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i+j, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) time.sleep(180) test_stub.exercise_connection(600) test_util.test_pass('Scheduler Test Success')
def test(): global vm global schd vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) test_util.test_logger('Setup stop and start VM scheduler') schd = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', 0, 1) actual_startDate = time.mktime( time.strptime(schd.startTime, '%b %d, %Y %I:%M:%S %p')) if actual_startDate != start_date and actual_startDate != start_date + 1.0: test_util.test_fail( 'startDate is expectd to set to now, which should be around %s' % (start_date)) test_stub.sleep_util(start_date + 58) start_msg_mismatch = 0 for i in range(1, 58): if not test_lib.lib_find_in_local_management_server_log( start_date + i, '[msg send]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn( 'StopVmInstanceMsg is expected to execute at %s' % (start_date + i)) if start_msg_mismatch > 5: test_util.test_fail( '%s of 58 StartVmInstanceMsg not executed at expected timestamp' % (start_msg_mismatch)) schd_ops.delete_scheduler(schd.uuid) try: vm.destroy() except: test_util.test_logger( 'expected exception when destroy VM since too many queued task') test_util.test_pass('Create Simple VM Start Scheduler Success')
def test(): global vm global schd vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 120) test_stub.sleep_util(start_date+58) for i in range(0, 58): if test_lib.lib_find_in_local_management_server_log(start_date+i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail('VM is expected to reboot start from %s' (start_date+60)) test_stub.sleep_util(start_date+59) if not test_lib.lib_wait_target_down(vm.get_vm().vmNics[0].ip, '22', 60): test_util.test_fail('VM: %s is not reboot in 60 seconds. Fail to reboot it with scheduler. ' % vm.get_vm().uuid) if not test_lib.lib_find_in_local_management_server_log(start_date+60, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail('VM is expected to reboot start from %s' % (start_date+60)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Create Simple VM Reboot Scheduler Success')
def delete_start_vm_scheduler(schd_uuid): global schds schd_ops.delete_scheduler(schd_uuid)
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) start_date = int(time.time()) test_util.test_logger('Setup stop and start VM scheduler') schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date, 1) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date, 2) test_stub.sleep_util(start_date + 58) stop_msg_mismatch = 0 start_msg_mismatch = 0 for i in range(0, 58): if not test_lib.lib_find_in_local_management_server_log( start_date + i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): stop_msg_mismatch += 1 test_util.test_warn( 'StopVmInstanceMsg is expected to execute at %s' % (start_date + i)) if i % 2 == 0: if not test_lib.lib_find_in_local_management_server_log( start_date + i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn( 'StartVmInstanceMsg is expected to execute at %s' % (start_date + i)) else: if test_lib.lib_find_in_local_management_server_log( start_date + i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn( 'StartVmInstanceMsg is not expected to execute at %s' % (start_date + i)) if stop_msg_mismatch > 10: test_util.test_fail( '%s of 58 StopVmInstanceMsg not executed at expected timestamp' % (stop_msg_mismatch)) if start_msg_mismatch > 5: test_util.test_fail( '%s of 58 StartVmInstanceMsg not executed at expected timestamp' % (start_msg_mismatch)) test_util.test_logger('Update stop and start VM scheduler to cron type') start_date = int(time.time()) test_stub.sleep_util((start_date + 59) / 60 * 60) schd_ops.update_scheduler(schd1.uuid, 'cron', 'cron_stop_vm_scheduler2', None, None, None, '0 * * * * ?') schd_ops.update_scheduler(schd2.uuid, 'cron', 'cron_start_vm_scheduler2', None, None, None, '0 * * * * ?') change_date = int(time.time()) test_stub.sleep_util(start_date + 59) for i in range(2, 58): if test_lib.lib_find_in_local_management_server_log( change_date + i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail( 'StopVmInstanceMsg is not expected to execute at %s' % (change_date + i)) if test_lib.lib_find_in_local_management_server_log( change_date + i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail( 'StartVmInstanceMsg is not expected to execute at %s' % (change_date + i)) schd_ops.update_scheduler(schd1.uuid, 'cron', 'cron_stop_vm_scheduler3', None, None, None, '0/2 * * * * ?') schd_ops.update_scheduler(schd2.uuid, 'cron', 'cron_start_vm_scheduler3', None, None, None, '* * * * * ?') change_date = int(time.time()) test_stub.sleep_util(change_date + 60) stop_msg_mismatch = 0 start_msg_mismatch = 0 for i in range(1, 58): if (change_date + i) % 2 == 0: if not test_lib.lib_find_in_local_management_server_log( change_date + i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): stop_msg_mismatch += 1 test_util.test_warn( 'StopVmInstanceMsg is expected to execute at %s' % (change_date + i)) else: if test_lib.lib_find_in_local_management_server_log( change_date + i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): stop_msg_mismatch += 1 test_util.test_warn( 'StopVmInstanceMsg is not expected to execute at %s' % (change_date + i)) if not test_lib.lib_find_in_local_management_server_log( change_date + i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn( 'StartVmInstanceMsg is expected to execute at %s' % (change_date + i)) if stop_msg_mismatch > 5: test_util.test_fail( '%s of 58 StopVmInstanceMsg not executed at expected timestamp' % (stop_msg_mismatch)) if start_msg_mismatch > 10: test_util.test_fail( '%s of 58 StartVmInstanceMsg not executed at expected timestamp' % (start_msg_mismatch)) schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) try: vm.destroy() except: test_util.test_logger( 'expected exception when destroy VM since too many queued task') test_util.test_pass('Create Simple VM Stop Start Scheduler Success')
def test(): global vm vm = test_stub.create_basic_vm() vm.check() start_date = int(time.time()) schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date + 60, 30) cmd = "init 0" host_username = os.environ.get('nodeUserName') host_password = os.environ.get('nodePassword') zstack_ha_vip = os.environ.get('zstackHaVip') node1_ip = os.environ.get('node1Ip') test_util.test_logger("shutdown node: %s" % (node1_ip)) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) node2_ip = os.environ.get('node2Ip') test_util.test_logger("shutdown node: %s" % (node2_ip)) rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) test_util.test_logger("recover node: %s" % (node1_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip)) test_util.test_logger("recover node: %s" % (node2_ip)) os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip)) cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \ (host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip) rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180) if not rsp: rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180) time.sleep(180) test_stub.exercise_connection(600) time.sleep(180) scheduler_execution_count = 0 for i in range(0, 30): if test_lib.lib_find_in_remote_management_server_log( node1_ip, host_username, host_password, start_date + 60 + 30 * i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count += 1 if test_lib.lib_find_in_remote_management_server_log( node2_ip, host_username, host_password, start_date + 60 + 30 * i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid): scheduler_execution_count -= 1 if abs(scheduler_execution_count) < 5: test_util.test_fail( 'VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() test_util.test_pass('Scheduler Test Success')
def test(): global vm global schd1 global schd2 delete_policy = test_lib.lib_get_delete_policy('vm') vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) vm.set_delete_policy('Delay') start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date + 10, 20) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date + 20, 20) test_stub.sleep_util(start_date + 45) test_util.test_dsc('check scheduler state after create scheduler') check_scheduler_state(schd1, 'Enabled') check_scheduler_state(schd2, 'Enabled') if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', start_date + 10): test_util.test_fail( 'StopVmInstanceJob not executed at expected timestamp range') if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', start_date + 20): test_util.test_fail( 'StartVmInstanceJob not executed at expected timestamp range') vm.destroy() current_time = int(time.time()) except_start_time = start_date + 20 * (( (current_time - start_date) % 20) + 1) test_stub.sleep_util(except_start_time + 45) test_util.test_dsc('check scheduler state after destroy vm') check_scheduler_state(schd1, 'Disabled') check_scheduler_state(schd2, 'Disabled') if check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time + 10): test_util.test_fail( 'StopVmInstanceJob executed at unexpected timestamp range') if check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time + 20): test_util.test_fail( 'StartVmInstanceJob executed at unexpected timestamp range') vm.recover() current_time = int(time.time()) except_start_time = start_date + 20 * (( (current_time - start_date) % 20) + 1) test_stub.sleep_util(except_start_time + 45) test_util.test_dsc('check scheduler state after recover vm') check_scheduler_state(schd1, 'Enabled') check_scheduler_state(schd2, 'Enabled') if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time + 10): test_util.test_fail( 'StopVmInstanceJob not executed at expected timestamp range') if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time + 20): test_util.test_fail( 'StartVmInstanceJob not executed at expected timestamp range') schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.set_delete_policy(delete_policy) vm.destroy() conditions = res_ops.gen_query_conditions('uuid', '=', schd1.uuid) if len(res_ops.query_resource(res_ops.SCHEDULER, conditions)) > 0: test_util.test_fail( 'check stop vm scheduler, it is expected to be destroied, but it still exists' ) conditions = res_ops.gen_query_conditions('uuid', '=', schd2.uuid) if len(res_ops.query_resource(res_ops.SCHEDULER, conditions)) > 0: test_util.test_fail( 'check start vm scheduler, it is expected to be destroied, but it still exists' ) test_util.test_pass( 'Check Scheduler State after Destroy and Recover VM Success')
def test(): global vm global schd1 global schd2 delete_policy = test_lib.lib_get_delete_policy('vm') vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) vm.set_delete_policy('Delay') start_date = int(time.time()) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_stop_vm_scheduler', start_date+10, 20) schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_start_vm_scheduler', start_date+20, 20) test_stub.sleep_util(start_date+45) test_util.test_dsc('check scheduler state after create scheduler') check_scheduler_state(schd1, 'Enabled') check_scheduler_state(schd2, 'Enabled') if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', start_date+10): test_util.test_fail('StopVmInstanceJob not executed at expected timestamp range') if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', start_date+20): test_util.test_fail('StartVmInstanceJob not executed at expected timestamp range') vm.destroy() current_time = int(time.time()) except_start_time = start_date + 20 * (((current_time - start_date) % 20) + 1) test_stub.sleep_util(except_start_time + 45) test_util.test_dsc('check scheduler state after destroy vm') check_scheduler_state(schd1, 'Disabled') check_scheduler_state(schd2, 'Disabled') if check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time+10): test_util.test_fail('StopVmInstanceJob executed at unexpected timestamp range') if check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time+20): test_util.test_fail('StartVmInstanceJob executed at unexpected timestamp range') vm.recover() current_time = int(time.time()) except_start_time = start_date + 20 * (((current_time - start_date) % 20) + 1) test_stub.sleep_util(except_start_time + 45) test_util.test_dsc('check scheduler state after recover vm') check_scheduler_state(schd1, 'Disabled') check_scheduler_state(schd2, 'Disabled') # if not check_scheduler_msg('run scheduler for job: StopVmInstanceJob', except_start_time+10): # test_util.test_fail('StopVmInstanceJob not executed at expected timestamp range') # if not check_scheduler_msg('run scheduler for job: StartVmInstanceJob', except_start_time+20): # test_util.test_fail('StartVmInstanceJob not executed at expected timestamp range' ) schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) vm.set_delete_policy(delete_policy) vm.destroy() conditions = res_ops.gen_query_conditions('uuid', '=', schd1.uuid) if len(res_ops.query_resource(res_ops.SCHEDULER, conditions)) > 0: test_util.test_fail('check stop vm scheduler, it is expected to be destroied, but it still exists') conditions = res_ops.gen_query_conditions('uuid', '=', schd2.uuid) if len(res_ops.query_resource(res_ops.SCHEDULER, conditions)) > 0: test_util.test_fail('check start vm scheduler, it is expected to be destroied, but it still exists') test_util.test_pass('Check Scheduler State after Destroy and Recover VM Success')
def test(): global vm global schd global new_account vm = test_stub.create_vlan_vm() test_obj_dict.add_vm(vm) disk_offering = test_lib.lib_get_disk_offering_by_name(os.environ.get('smallDiskOfferingName')) volume_creation_option = test_util.VolumeOption() volume_creation_option.set_name('volume for snapshot scheduler testing') volume_creation_option.set_disk_offering_uuid(disk_offering.uuid) volume = test_stub.create_volume(volume_creation_option) test_obj_dict.add_volume(volume) volume.attach(vm) volume.detach() test_util.test_dsc('create snapshot scheduler') start_date = int(time.time()) sp_option = test_util.SnapshotOption() sp_option.set_name('simple_schduler_snapshot') sp_option.set_volume_uuid(volume.get_volume().uuid) schd = vol_ops.create_snapshot_scheduler(sp_option, 'simple', 'simple_create_snapshot_scheduler', start_date+60, 120) check_scheduler_state(schd, 'Enabled') snapshot_num = 0 for i in range(0, 3): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(start_date + 60 + 120*i - 2) test_util.test_logger('check volume snapshot number at %s, there should be %s' % (start_date + 60 + 120*i - 2, snapshot_num)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) snapshot_num += 1 test_stub.sleep_util(start_date + 60 + 120*i + 60) test_util.test_logger('check volume snapshot number at %s, there should be %s' % (start_date + 60 + 120*i + 65, snapshot_num+1)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) new_account = account_operations.create_account('new_account', 'password', 'Normal') res_ops.change_recource_owner(new_account.uuid, volume.get_volume().uuid) test_util.test_dsc('check scheduler state after changing the owner of volume') check_scheduler_state(schd, 'Disabled') current_time = int(time.time()) except_start_time = start_date + 120 * (((current_time - start_date) % 120) + 1) for i in range(0, 3): test_util.test_logger('round %s' % (i)) test_stub.sleep_util(except_start_time + 60 + 120*i - 2) test_util.test_logger('check volume snapshot number at %s, there should be %s' % (except_start_time + 60 + 120*i - 2, snapshot_num)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) test_stub.sleep_util(except_start_time + 60 + 120*i + 60) test_util.test_logger('check volume snapshot number at %s, there should be %s' % (except_start_time + 60 + 120*i + 65, snapshot_num+1)) new_snapshot_num = query_snapshot_number('simple_schduler_snapshot') if snapshot_num != new_snapshot_num: test_util.test_fail('there sholuld be %s snapshots' % (snapshot_num)) schd_ops.delete_scheduler(schd.uuid) vm.destroy() account_operations.delete_account(new_account.uuid) test_util.test_pass('Check Scheduler State after Changing Volume Owner Success')
def test(): global vm global schd1 global schd2 vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1')) test_util.test_logger('Update stop and start VM scheduler to cron type') start_date = int(time.time()) test_stub.sleep_util((start_date+59)/60*60) schd1 = vm_ops.stop_vm_scheduler(vm.get_vm().uuid, 'cron', 'cron_stop_vm_scheduler', None, None, None, '0 * * * * ?') schd2 = vm_ops.start_vm_scheduler(vm.get_vm().uuid, 'cron', 'cron_start_vm_scheduler', None, None, None, '0 * * * * ?') change_date = int(time.time()) test_stub.sleep_util(start_date+30) for i in range(2, 30): if test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail('StopVmInstanceMsg is not expected to execute at %s' % (change_date+i)) if test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): test_util.test_fail('StartVmInstanceMsg is not expected to execute at %s' % (change_date+i)) schd_ops.update_scheduler(schd1.uuid, 'simple', 'simple_stop_vm_scheduler2', start_date, 2, 10) schd_ops.update_scheduler(schd2.uuid, 'simple', 'simple_start_vm_scheduler2', start_date, 1, 20) change_date = int(time.time()) test_stub.sleep_util(change_date+60) stop_msg_mismatch = 0 start_msg_mismatch = 0 for i in range(1, 15): if (change_date+i-start_date) % 2 == 0: if not test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): stop_msg_mismatch += 1 test_util.test_warn('StopVmInstanceMsg is expected to execute at %s' % (change_date+i)) else: if test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): stop_msg_mismatch += 1 test_util.test_warn('StopVmInstanceMsg is not expected to execute at %s' % (change_date+i)) if not test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn('StartVmInstanceMsg is expected to execute at %s' % (change_date+i)) for i in range(21, 60): if test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StopVmInstanceMsg', vm.get_vm().uuid): stop_msg_mismatch += 1 test_util.test_warn('StopVmInstanceMsg is not expected to execute at %s' % (change_date+i)) if test_lib.lib_find_in_local_management_server_log(change_date+i, '[msg received]: {"org.zstack.header.vm.StartVmInstanceMsg', vm.get_vm().uuid): start_msg_mismatch += 1 test_util.test_warn('StartVmInstanceMsg is not expected to execute at %s' % (change_date+i)) if stop_msg_mismatch > 5: test_util.test_fail('%s of 58 StopVmInstanceMsg not executed at expected timestamp' % (stop_msg_mismatch)) if start_msg_mismatch > 10: test_util.test_fail('%s of 58 StartVmInstanceMsg not executed at expected timestamp' % (start_msg_mismatch)) schd_ops.delete_scheduler(schd1.uuid) schd_ops.delete_scheduler(schd2.uuid) try: vm.destroy() except: test_util.test_logger('expected exception when destroy VM since too many queued task') test_util.test_pass('Create Simple VM Stop Start Scheduler Success')