def get_s_vm_cfg_lst_vip_bind(scenarioConfig, scenarioFile, retry_times=1):
"""
It gets host with vip bound, while returned a s_vm config
"""
mha_s_vm_list = get_mha_s_vm_list_from_scenario_file(scenarioConfig, scenarioFile)
if len(mha_s_vm_list) < 1:
return []
test_util.test_logger("@@DEBUG@@: mha_s_vm_list=<%s>" %(str(mha_s_vm_list)))
host_list = []
vip = os.environ['zstackHaVip']
for host in mha_s_vm_list:
host_config = sce_ops.get_scenario_config_vm(host.name_, scenarioConfig)
cmd = "ip a|grep " + vip
try:
for retry_cnt in range(retry_times):
if sce_is_sep_pub():
vm_list = test_lib.lib_execute_ssh_cmd(host.managementIp_, host_config.imageUsername_, host_config.imagePassword_,cmd)
else:
vm_list = test_lib.lib_execute_ssh_cmd(host.ip_, host_config.imageUsername_, host_config.imagePassword_,cmd)
if vm_list:
host_list.append(host)
break
time.sleep(1)
except Exception, e:
test_util.test_logger("@@get host exception@@:%s" %(str(e)))
continue
def test():
test_util.test_dsc("create vr vm and vpc vrouter")
vm = test_stub.create_vr_vm('vr_vm','imageName_s','l3NoVlanNetworkName2')
test_obj_dict.add_vm(vm)
if not test_lib.lib_find_vr_by_vm(vm.vm) or not test_lib.lib_find_vr_by_vm(vm.vm)[0]:
test_lib.lib_error_cleanup(test_obj_dict)
test_util.test_skip("skip the test for no vr found in the env.")
vpc_vr = test_stub.create_vpc_vrouter()
check_host_prometheus_conf()
check_vrouter_prometheus_conf()
check_prometheus_data()
hosts = test_lib.lib_get_all_hosts_from_plan()
for host in hosts:
host_ops.reconnect_host(host_uuid)
check_host_prometheus_conf()
check_vrouter_prometheus_conf()
check_prometheus_data()
test_lib.lib_execute_ssh_cmd(mn_ip,"root","password","zstack-ctl restart_node",timeout=300)
check_host_prometheus_conf()
check_vrouter_prometheus_conf()
check_prometheus_data()
test_lib.lib_error_cleanup(test_obj_dict)
test_util.test_pass('Test prometheus Success')
def test():
global vm1
global vm2
global node_ip
vm1 = test_stub.create_basic_vm()
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop rabbitmq on node: %s" % (node_ip))
cmd = "rabbitmqctl stop_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(120)
test_stub.exercise_connection(1200)
test_util.test_logger("create vm to check if it still works")
vm1.check()
vm1.destroy()
vm2 = test_stub.create_basic_vm()
vm2.check()
vm2.destroy()
cmd = "rabbitmqctl start_app"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
time.sleep(60)
test_stub.exercise_connection(1200)
test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test():
global vm
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test Success')
def test():
ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)
for i in ps:
if i.type == 'AliyunEBS':
test_util.test_skip('Skip vm reinit test on AliyunEBS')
vm = test_stub.create_user_vlan_vm()
test_obj_dict.add_vm(vm)
vm.check()
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
cmd = 'touch /root/test-file-for-reinit'
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp == False:
test_util.test_fail('Fail to create file in VM')
vm.stop()
vm.reinit()
vm.update()
vm.check()
vm.start()
cmd = '[ -e /root/test-file-for-reinit ] && echo yes || echo no'
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp == 'yes':
test_util.test_fail('VM does not be reverted to image used for creating the VM, the later file still exists')
vm.destroy()
test_util.test_pass('Re-init VM Test Success')
def test():
bs = res_ops.query_resource(res_ops.BACKUP_STORAGE)[0]
if bs.type == "Ceph":
test_util.test_skip('bs: %s is ceph backup storage. Will skip test.' % bs.uuid)
command = 'dd if=/dev/zero of=%s bs=1M count=1 seek=300' % test_image
test_lib.lib_execute_ssh_cmd(bs.hostname, 'root', 'password', command)
image_name = 'test-image-%s' % time.time()
image_option = test_util.ImageOption()
image_option.set_name(image_name)
image_option.set_description('test image which is upload from local filesystem.')
image_option.set_url('file://%s' % test_image)
image_option.set_backup_storage_uuid_list([bs.uuid])
image_option.set_format('raw')
image_option.set_mediaType('RootVolumeTemplate')
image_inv = img_ops.add_root_volume_template(image_option)
time.sleep(10)
image = zstack_image_header.ZstackTestImage()
image.set_creation_option(image_option)
image.set_image(image_inv)
test_obj_dict.add_image(image)
image.check()
vm = test_stub.create_vm(image_name = image_name)
vm.destroy()
image.delete()
command = 'ls %s' % test_image
if not test_lib.lib_execute_ssh_cmd(bs.hostname, 'root', 'password', command):
test_util.test_fail('test image disappeared, after add image.')
command = 'rm -f %s' % test_image
result = test_lib.lib_execute_ssh_cmd(bs.hostname, 'root', 'password', command)
test_util.test_pass('Test adding image from local storage pass.')
def test():
global test_host
global mn_ip
global host_list
allow_ps_list = [inventory.LOCAL_STORAGE_TYPE]
test_lib.skip_test_when_ps_type_not_in_list(allow_ps_list)
mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
host_list = test_stub.get_sce_hosts(test_lib.all_scenario_config, test_lib.scenario_file)
for host in host_list:
if host.ip_ != mn_ip:
test_host = host
break
if not test_host:
test_util.test_fail('there is no host with ip excluding mn_ip: %s in scenario file.' %(mn_ip))
host_username = os.environ.get('hostUsername')
host_password = os.environ.get('hostPassword')
cmd = "zstack-ctl stop"
if not test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd, timeout = 120):
test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))
t = threading.Thread(target=async_ifconfig_br_eth0_down_up_wrapper,
args=(240, test_host.ip_, host_username, host_password))
t.start()
cmd = "nohup zstack-ctl start &"
if not test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd, timeout = 120):
test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))
t.join()
test_util.test_pass('Test zstack-ctl start when host status is not conformance with zstack db Success')
def test():
vm = test_stub.create_user_vlan_vm()
test_obj_dict.add_vm(vm)
vm.check()
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
cmd = 'touch /root/test-file-for-reinit'
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp == False:
test_util.test_fail('Fail to create file in VM')
vm.stop()
vm.reinit()
vm.update()
vm.check()
vm.start()
cmd = '[ -e /root/test-file-for-reinit ] && echo yes || echo no'
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp == 'yes':
test_util.test_fail(
'VM does not be reverted to image used for creating the VM, the later file still exists'
)
vm.destroy()
test_util.test_pass('Re-init VM Test Success')
def test():
global vm1
global vm2
global node_ip
vm1 = test_stub.create_basic_vm()
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop rabbitmq on node: %s" % (node_ip))
cmd = "rabbitmqctl stop_app"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
time.sleep(120)
test_stub.exercise_connection(600)
test_util.test_logger("create vm to check if it still works")
vm1.check()
vm1.destroy()
vm2 = test_stub.create_basic_vm()
vm2.check()
vm2.destroy()
cmd = "rabbitmqctl start_app"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
time.sleep(60)
test_stub.exercise_connection(600)
test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test():
imagestore_backup_storage = test_lib.lib_get_image_store_backup_storage()
if not imagestore_backup_storage:
test_util.test_skip('Not find image store type backup storage.')
img_option = test_util.ImageOption()
img_option.set_name('iso')
root_disk_uuid = test_lib.lib_get_disk_offering_by_name(os.environ.get('rootDiskOfferingName')).uuid
bs_uuid = imagestore_backup_storage.uuid
img_option.set_backup_storage_uuid_list([bs_uuid])
command = "command -v genisoimage"
result = test_lib.lib_execute_ssh_cmd(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], 'root', 'password', command)
if not result:
command = "yum -y install genisoimage --disablerepo=* --enablerepo=zstack-local"
test_lib.lib_execute_ssh_cmd(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], 'root', 'password', command)
command = "genisoimage -o %s/apache-tomcat/webapps/zstack/static/zstack-repo/7/x86_64/os/test.iso /tmp/" % os.environ.get('zstackInstallPath')
test_lib.lib_execute_ssh_cmd(os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'], 'root', 'password', command)
img_option.set_url('http://%s:8080/zstack/static/zstack-repo/7/x86_64/os/test.iso' % (os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']))
image_inv = img_ops.add_iso_template(img_option)
image = test_image.ZstackTestImage()
image.set_image(image_inv)
image.set_creation_option(img_option)
test_obj_dict.add_image(image)
image_url = image.export()
image.delete_exported_image()
test_lib.lib_robot_cleanup(test_obj_dict)
if image_url.endswith('.iso'):
test_util.test_pass('Export ISO Image Test Success')
else:
test_util.test_fail('Export ISO Image Test Fail, exported ISO image ends with %s' % (image_url.split('.')[-1]))
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network for a small while: %s" %
(node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s && sleep 10 && ifup %s" % (l2_network_interface,
l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_util.test_logger("recover mysql")
cmd = "service mysql start"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_stub.exercise_connection(600)
test_util.test_logger("check create vm")
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass(
'Create VM Test with one node network down for a while Success')
def test():
vm = test_stub.create_user_vlan_vm()
test_obj_dict.add_vm(vm)
vm.check()
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
cmd = 'touch /root/test-file-for-reinit'
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp == False:
test_util.test_fail('Fail to create file in VM')
vm.stop()
vm.reinit()
vm.update()
vm.check()
vm.start()
cmd = '[ -e /root/test-file-for-reinit ] && echo yes || echo no'
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp == 'yes':
test_util.test_fail('VM does not be reverted to image used for creating the VM, the later file still exists')
vm.destroy()
test_util.test_pass('Re-init VM Test Success')
def check_host_prometheus_conf():
global host_ip
global host_uuid
global mn_ip
mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
# host_list = test_stub.get_sce_hosts(test_lib.all_scenario_config, test_lib.scenario_file)
host_list = test_lib.lib_get_all_hosts_from_plan()
cmd = 'yum install -y jq --nogpgcheck'
test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
for host in host_list:
host_ip = host.managementIp_
host_hostname = host_ip.replace('.', '-')
host_uuid = test_lib.lib_get_host_by_ip(host_ip).uuid
if host_ip == mn_ip:
cmd = "jq -r '.[].targets[]' /usr/local/zstacktest/prometheus/discovery/management-node/management-server-exporter.json"
cmd_out = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
expect_result = mn_ip + ':' + str(8081)
if cmd_out.strip() != expect_result.strip():
test_util.test_fail('targets in management-node-exporter.json is not right.')
else:
cmd = "jq -r '.[].targets[]' /usr/local/zstacktest/prometheus/discovery/hosts/%s-%s.json" % (host_uuid, host_hostname)
cmd_out = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
expect_result1 = host_ip + ':' + str(9103)
expect_result2 = host_ip + ':' + str(9100)
expect_result3 = host_ip + ':' + str(7069)
if expect_result1 not in cmd_out or expect_result2 not in cmd_out or expect_result3 not in cmd_out:
test_util.test_fail('targets in hosts/%s-%s.json is not right.') % (host_uuid, host_hostname)
cmd = "jq -r '.[].labels[]' /usr/local/zstacktest/prometheus/discovery/hosts/%s-%s.json" % (host_uuid, host_hostname)
cmd_out = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
if host_uuid.strip() != cmd_out.strip():
test_util.test_fail('labels in hosts/%s-%s.json is not right.') % (host_uuid, host_hostname)
def upgrade_zsha(scenarioConfig, scenarioFile):
host_list = get_mn_host(scenarioConfig, scenarioFile)
test_host = host_list[0]
test_host_ip = test_host.ip_
zsha_path = "/home/%s/zs-ha" % test_host_ip
config_path = "/home/%s/config.json" % test_host_ip
current_zsha_path = "/tmp/zstack-ha-installer/zsha"
check_cmd = "ls -l %s" % current_zsha_path
host_config = sce_ops.get_scenario_config_vm(test_host.name_,
scenarioConfig)
former_time = []
for host in host_list:
former_zsha = test_lib.lib_execute_ssh_cmd(host.ip_,
host_config.imageUsername_,
host_config.imagePassword_,
check_cmd)
former_time.append(former_zsha.split()[7])
upgrade_zsha_cmd = "%s install -p %s -c %s" % (
zsha_path, host_config.imagePassword_, config_path)
test_lib.lib_execute_ssh_cmd(test_host_ip, host_config.imageUsername_,
host_config.imagePassword_, upgrade_zsha_cmd)
current_time = []
for host in host_list:
current_zsha = test_lib.lib_execute_ssh_cmd(host.ip_,
host_config.imageUsername_,
host_config.imagePassword_,
check_cmd)
current_time.append(current_zsha.split()[7])
for i in range(len(former_time)):
if current_time[i] == former_time[i]:
return False
return True
def test():
global vm, kvm_host
ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)
for i in ps:
if i.type == inventory.LOCAL_STORAGE_TYPE:
break
else:
test_util.test_skip('Skip test on non-localstoreate PS')
#ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)[0]
#if ps.type != inventory.LOCAL_STORAGE_TYPE:
# test_util.test_skip('Skip test on non-localstorage')
#if "test-config-local-ps.xml" != os.path.basename(os.environ.get('WOODPECKER_TEST_CONFIG_FILE')).strip():
# test_util.test_skip('Skip test on non-localstoreage')
#query all hosts and change password
conditions = res_ops.gen_query_conditions('state', '=', 'Enabled')
if res_ops.query_resource(res_ops.HOST, conditions):
kvm_host = res_ops.query_resource(res_ops.HOST, conditions)
for i in kvm_host:
host_ops.update_kvm_host(i.uuid, 'password', 'password*()')
cmd = 'echo "password*()"| passwd --stdin root'
test_lib.lib_execute_ssh_cmd(i.managementIp, "root", "password",
cmd)
host_ops.reconnect_host(i.uuid)
else:
test_util.test_skip("There is no host. Skip test")
test_util.test_dsc('Test KVM Host Infomation: password')
#create vm and stop and migrate
vm = test_stub.create_vr_vm('migrate_stopped_vm', 'imageName_s',
'l3VlanNetwork2')
vm.check()
target_host = test_lib.lib_find_random_host(vm.vm)
vm.stop()
vol_ops.migrate_volume(vm.get_vm().allVolumes[0].uuid, target_host.uuid)
vm.check()
vm.start()
vm.check()
vm.destroy()
vm.expunge()
################################# recover KVM HOST Password #################################
conditions = res_ops.gen_query_conditions('state', '=', 'Enabled')
if res_ops.query_resource(res_ops.HOST, conditions):
kvm_host = res_ops.query_resource(res_ops.HOST, conditions)
for i in kvm_host:
host_ops.update_kvm_host(i.uuid, 'password', 'password')
cmd = 'echo "password"| passwd --stdin root'
test_lib.lib_execute_ssh_cmd(i.managementIp, "root", "password*()",
cmd)
host_ops.reconnect_host(i.uuid)
else:
test_util.test_skip("There is no host. Skip test")
test_util.test_dsc('Test KVM Host Infomation: password')
test_util.test_pass(
'Migrate Stopped VM with special_password Test Success')
def async_ifconfig_br_eth0_down_up_wrapper(sleep_time, ip, host_username,
host_password):
cmd = "ifconfig br_eth0 down; sleep %s; ifconfig br_eth0 up" % (sleep_time)
test_lib.lib_execute_ssh_cmd(ip,
host_username,
host_password,
cmd,
timeout=sleep_time + 20)
def error_cleanup():
mn_ip = os.environ["ZSTACK_BUILT_IN_HTTP_SERVER_IP"]
test_lib.lib_execute_ssh_cmd(mn_ip,
"root",
"password",
"zstack-ctl restart_node",
timeout=300)
test_lib.lib_error_cleanup(test_obj_dict)
def test():
if test_lib.scenario_config != None and test_lib.scenario_file != None and not os.path.exists(test_lib.scenario_file):
scenario_operations.deploy_scenario(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config)
test_util.test_skip('Suite Setup Success')
if test_lib.scenario_config != None and test_lib.scenario_destroy != None:
scenario_operations.destroy_scenario(test_lib.all_scenario_config, test_lib.scenario_destroy)
nic_name = "eth0"
if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file):
nic_name = "zsn0"
#This vlan creation is not a must, if testing is under nested virt env. But it is required on physical host without enough physcial network devices and your test execution machine is not the same one as Host machine.
#no matter if current host is a ZStest host, we need to create 2 vlan devs for future testing connection for novlan test cases.
#linux.create_vlan_eth(nic_name, 10)
#linux.create_vlan_eth(nic_name, 11)
#If test execution machine is not the same one as Host machine, deploy work is needed to separated to 2 steps(deploy_test_agent, execute_plan_without_deploy_test_agent). And it can not directly call SetupAction.run()
test_lib.setup_plan.deploy_test_agent()
#cmd = host_plugin.CreateVlanDeviceCmd()
#cmd.ethname = nic_name
#cmd.vlan = 10
#cmd2 = host_plugin.CreateVlanDeviceCmd()
#cmd2.ethname = nic_name
#cmd2.vlan = 11
testHosts = test_lib.lib_get_all_hosts_from_plan()
if type(testHosts) != type([]):
testHosts = [testHosts]
#for host in testHosts:
# http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd)
# http.json_dump_post(testagent.build_http_path(host.managementIp_, host_plugin.CREATE_VLAN_DEVICE_PATH), cmd2)
test_lib.setup_plan.execute_plan_without_deploy_test_agent()
if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file):
mn_ips = deploy_operations.get_nodes_from_scenario_file(test_lib.all_scenario_config, test_lib.scenario_file, test_lib.deploy_config)
if os.path.exists(EXTRA_SUITE_SETUP_SCRIPT):
os.system("bash %s '%s' %s" % (EXTRA_SUITE_SETUP_SCRIPT, mn_ips, 'disaster-recovery'))
elif os.path.exists(EXTRA_SUITE_SETUP_SCRIPT):
os.system("bash %s '' '%s'" % (EXTRA_SUITE_SETUP_SCRIPT, 'disaster-recovery'))
dicts = {}
for host in testHosts:
cmd = 'virsh --version'
dicts[host.managementIp_] = test_lib.lib_execute_ssh_cmd(host.managementIp_, host.username_, host.password_, cmd, 180)
deploy_operations.deploy_initial_database(test_lib.deploy_config, test_lib.all_scenario_config, test_lib.scenario_file)
for host in testHosts:
os.system("bash %s %s" % (EXTRA_HOST_SETUP_SCRIPT, host.managementIp_))
version = test_lib.lib_execute_ssh_cmd(host.managementIp_, host.username_, host.password_, cmd, 180)
# Some host are installed in expert mode without virsh
if dicts[host.managementIp_]:
assert version == dicts[host.managementIp_]
if test_lib.lib_get_ha_selffencer_maxattempts() != None:
test_lib.lib_set_ha_selffencer_maxattempts('60')
test_lib.lib_set_ha_selffencer_storagechecker_timeout('60')
test_lib.lib_set_primary_storage_imagecache_gc_interval(1)
test_util.test_pass('Suite Setup Success')
def add_default_route(ip):
#this function should not be invoked to recover the env.
host_username = os.environ.get('hostUsername')
host_password = os.environ.get('hostPassword')
cmd = "ip r add default dev br_eth0 via 172.20.0.1"
test_lib.lib_execute_ssh_cmd(ip,
host_username,
host_password,
cmd,
timeout=20)
def migrate_mn_vm(origin_host, target_host, scenarioConfig):
if test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-flat-dhcp-nfs-sep-pub-man.xml"], ["scenario-config-nfs-sep-pub.xml"]) or \
test_lib.lib_cur_cfg_is_a_and_b(["test-config-vyos-ceph-3-nets-sep.xml"], ["scenario-config-ceph-sep-pub.xml"]):
cmd = 'zsha migrate %s' % (target_host.managementIp_)
host_config = sce_ops.get_scenario_config_vm(origin_host.name_, scenarioConfig)
test_lib.lib_execute_ssh_cmd(origin_host.managementIp_, host_config.imageUsername_, host_config.imagePassword_,cmd)
else:
cmd = 'zsha migrate %s' % (target_host.ip_)
host_config = sce_ops.get_scenario_config_vm(origin_host.name_, scenarioConfig)
test_lib.lib_execute_ssh_cmd(origin_host.ip_, host_config.imageUsername_, host_config.imagePassword_,cmd)
def verify_config(management_ip):
tmp_file = '/tmp/%s' % uuid.uuid1().get_hex()
test_util.test_logger("%s" %(tmp_file))
test_lib.lib_execute_ssh_cmd(management_ip, 'root', 'password', 'zstack-cli LogInByAccount accountName=admin password=password', 180)
test_lib.lib_execute_ssh_cmd(management_ip,'root','password','zstack-cli QueryGlobalConfig sortBy=name > %s' %(tmp_file), 180)
default_path=environment(management_ip)
test_util.test_logger("default global config path:%s" %(default_path))
test_util.test_logger("check two file diff:%s, %s" %(default_path,tmp_file))
check(default_path,tmp_file)
def check_prometheus_data():
test_util.test_dsc("Check prometheus data.")
cmd = "ps -ef|grep [t]ools/prometheus |awk '{for(i=7+1;i<=NF;i++)printf $i \" \";printf\"\\n\"}'"
cmd_out = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
if cmd_out == None:
test_util.test_fail('prometheus agent is not enabled.')
cmd = 'ls -A /var/lib/zstack/prometheus/data/ |wc -w'
cmd_out = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
if cmd_out <= 10:
test_util.test_fail('prometheus data is null.')
def test():
global test_obj_dict
global origin_ip
global test_config_des
global deploy_tmpt_src
global deploy_tmpt_des
origin_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
test_util.test_dsc(
'Create test vm to test zstack all installation in CentOS7.')
# image_name = os.environ.get('imageName_i_c7')
# image_name = "zstack_iso_centos7_141"
image_name = os.environ.get('imageName_i_offline')
vm = test_stub.create_vlan_vm(image_name)
test_obj_dict.add_vm(vm)
if os.environ.get('zstackManagementIp') == None:
vm.check()
else:
time.sleep(60)
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
target_file = '/root/zstack-all-in-one.tgz'
test_stub.prepare_test_env(vm_inv, target_file)
ssh_cmd = 'ssh -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
test_stub.execute_all_install(ssh_cmd, target_file, tmp_file)
# test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)
test_util.test_dsc("Prepare Config Files")
cmd = "cp %s %s" % (deploy_tmpt_src, deploy_tmpt_des)
os.system(cmd)
cmd = "sed -i \"s/templateIP/%s/g\" %s" % (vm_ip, deploy_tmpt_des)
os.system(cmd)
test_config_obj = test_util.TestConfig(test_config_des)
test_config = test_config_obj.get_test_config()
all_config = test_config_obj.get_deploy_config()
deploy_config = all_config.deployerConfig
cmd = "vconfig add eth0 213"
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
cmd = "vconfig add eth0 212"
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = vm_ip
deploy_operations.deploy_initial_database(deploy_config)
test_util.test_dsc("Clean up")
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = origin_ip
os.system('rm -f %s' % deploy_tmpt_des)
os.system('rm -f %s' % tmp_file)
vm.destroy()
test_obj_dict.rm_vm(vm)
test_util.test_pass('ZStack installation Test Success')
def test():
global vm, kvm_host
ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)
for i in ps:
if i.type == inventory.LOCAL_STORAGE_TYPE:
break
else:
test_util.test_skip('Skip test on non-localstoreate PS')
#ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)[0]
#if ps.type != inventory.LOCAL_STORAGE_TYPE:
# test_util.test_skip('Skip test on non-localstorage')
#if "test-config-local-ps.xml" != os.path.basename(os.environ.get('WOODPECKER_TEST_CONFIG_FILE')).strip():
# test_util.test_skip('Skip test on non-localstoreage')
#query all hosts and change password
conditions = res_ops.gen_query_conditions('state', '=', 'Enabled')
if res_ops.query_resource(res_ops.HOST, conditions):
kvm_host = res_ops.query_resource(res_ops.HOST, conditions)
for i in kvm_host:
host_ops.update_kvm_host(i.uuid, 'password', 'password*()')
cmd = 'echo "password*()"| passwd --stdin root'
test_lib.lib_execute_ssh_cmd(i.managementIp,"root","password",cmd)
host_ops.reconnect_host(i.uuid)
else:
test_util.test_skip("There is no host. Skip test")
test_util.test_dsc('Test KVM Host Infomation: password')
#create vm and stop and migrate
vm = test_stub.create_vr_vm('migrate_stopped_vm', 'imageName_s', 'l3VlanNetwork2')
vm.check()
target_host = test_lib.lib_find_random_host(vm.vm)
vm.stop()
vol_ops.migrate_volume(vm.get_vm().allVolumes[0].uuid, target_host.uuid)
vm.check()
vm.start()
vm.check()
vm.destroy()
vm.expunge()
################################# recover KVM HOST Password #################################
conditions = res_ops.gen_query_conditions('state', '=', 'Enabled')
if res_ops.query_resource(res_ops.HOST, conditions):
kvm_host = res_ops.query_resource(res_ops.HOST, conditions)
for i in kvm_host:
host_ops.update_kvm_host(i.uuid, 'password', 'password')
cmd = 'echo "password"| passwd --stdin root'
test_lib.lib_execute_ssh_cmd(i.managementIp,"root","password*()",cmd)
host_ops.reconnect_host(i.uuid)
else:
test_util.test_skip("There is no host. Skip test")
test_util.test_dsc('Test KVM Host Infomation: password')
test_util.test_pass('Migrate Stopped VM with special_password Test Success')
def test():
global test_obj_dict
global origin_ip
global test_config_des
global deploy_tmpt_src
global deploy_tmpt_des
origin_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
test_util.test_dsc('Create test vm to test zstack all installation in CentOS7.')
# image_name = os.environ.get('imageName_i_c7')
# image_name = "zstack_iso_centos7_141"
image_name = os.environ.get('imageName_i_offline')
vm = test_stub.create_vlan_vm(image_name)
test_obj_dict.add_vm(vm)
if os.environ.get('zstackManagementIp') == None:
vm.check()
else:
time.sleep(60)
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
target_file = '/root/zstack-all-in-one.tgz'
test_stub.prepare_test_env(vm_inv, target_file)
ssh_cmd = 'ssh -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
test_stub.execute_all_install(ssh_cmd, target_file, tmp_file)
# test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)
test_util.test_dsc("Prepare Config Files")
cmd = "cp %s %s" % (deploy_tmpt_src, deploy_tmpt_des)
os.system(cmd)
cmd = "sed -i \"s/templateIP/%s/g\" %s" % (vm_ip, deploy_tmpt_des)
os.system(cmd)
test_config_obj = test_util.TestConfig(test_config_des)
test_config = test_config_obj.get_test_config()
all_config = test_config_obj.get_deploy_config()
deploy_config = all_config.deployerConfig
cmd = "vconfig add eth0 213"
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
cmd = "vconfig add eth0 212"
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = vm_ip
deploy_operations.deploy_initial_database(deploy_config)
test_util.test_dsc("Clean up")
os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = origin_ip
os.system('rm -f %s' % deploy_tmpt_des)
os.system('rm -f %s' % tmp_file)
vm.destroy()
test_obj_dict.rm_vm(vm)
test_util.test_pass('ZStack installation Test Success')
def test():
global test_obj_dict
global test_file_src
global test_file_des
global ct_original
if con_ops.get_global_config_value('vm', 'cleanTraffic') == 'false':
ct_original = 'false'
con_ops.change_global_config('vm', 'cleanTraffic', 'true')
else:
ct_original = 'true'
vm = test_stub.create_basic_vm()
test_obj_dict.add_vm(vm)
vm.check()
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
new_vm_ip = '172.20.1.1'
if new_vm_ip == vm_ip:
new_vm_ip = '172.20.1.2'
test_util.test_dsc("Prepare Test File")
cmd = "cp %s %s" % (test_file_src, test_file_des)
os.system(cmd)
cmd = "sed -i \"s/TemplateNodeIP/%s/g\" %s" % (node_ip, test_file_des)
os.system(cmd)
cmd = "sed -i \"s/TemplateOriginalIP/%s/g\" %s" % (vm_ip, test_file_des)
os.system(cmd)
cmd = "sed -i \"s/TemplateTestIP/%s/g\" %s" % (new_vm_ip, test_file_des)
os.system(cmd)
target_file = "/home/change_ip_test.sh"
test_stub.scp_file_to_vm(vm_inv, test_file_des, target_file)
cmd = "chmod +x %s" % target_file
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', target_file,
180)
time.sleep(60)
cmd = "cat /home/ip_spoofing_result"
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp[0] != "1":
test_util.test_fail(rsp)
vm.destroy()
test_obj_dict.rm_vm(vm)
con_ops.change_global_config('vm', 'cleanTraffic', ct_original)
os.system('rm -f %s' % test_file_des)
test_util.test_pass('IP Spoofing Test Success')
def set_host_cpu_model(ip,model=None):
i = 0
_model = get_host_cpu_model(ip)
test_util.test_logger(_model)
while _model != model:
i += 1
if i > 5:
test_util.test_fail("set host cpu model faild")
cmd = '''sed -i "s/'{}'/'{}'/g" /usr/share/libvirt/cpu_map.xml'''.format(_model,model)
test_lib.lib_execute_ssh_cmd(ip, 'root', 'password',cmd)
cmd = "systemctl restart libvirtd"
test_lib.lib_execute_ssh_cmd(ip, 'root', 'password',cmd)
_model = get_host_cpu_model(ip)
def test():
global test_obj_dict
global test_file_src
global test_file_des
global ct_original
if con_ops.get_global_config_value('vm', 'cleanTraffic') == 'false' :
ct_original='false'
con_ops.change_global_config('vm', 'cleanTraffic', 'true')
else:
ct_original='true'
vm = test_stub.create_basic_vm()
test_obj_dict.add_vm(vm)
vm.check()
vm_inv = vm.get_vm()
vm_ip = vm_inv.vmNics[0].ip
new_vm_ip = '172.20.1.1'
if new_vm_ip == vm_ip:
new_vm_ip = '172.20.1.2'
test_util.test_dsc("Prepare Test File")
cmd = "cp %s %s" % (test_file_src, test_file_des)
os.system(cmd)
cmd = "sed -i \"s/TemplateNodeIP/%s/g\" %s" % (node_ip, test_file_des)
os.system(cmd)
cmd = "sed -i \"s/TemplateOriginalIP/%s/g\" %s" % (vm_ip, test_file_des)
os.system(cmd)
cmd = "sed -i \"s/TemplateTestIP/%s/g\" %s" % (new_vm_ip, test_file_des)
os.system(cmd)
target_file = "/home/change_ip_test.sh"
test_stub.scp_file_to_vm(vm_inv, test_file_des, target_file)
cmd = "chmod +x %s" % target_file
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', target_file, 180)
time.sleep(60)
cmd = "cat /home/ip_spoofing_result"
rsp = test_lib.lib_execute_ssh_cmd(vm_ip, 'root', 'password', cmd, 180)
if rsp[0] != "1":
test_util.test_fail(rsp)
vm.destroy()
test_obj_dict.rm_vm(vm)
con_ops.change_global_config('vm', 'cleanTraffic', ct_original)
os.system('rm -f %s' % test_file_des)
test_util.test_pass('IP Spoofing Test Success')
def test():
global vm
vm = test_stub.create_spice_vm()
vm.check()
vm_inv = vm.get_vm()
vm_uuid = vm_inv.uuid
vm_state = vm.get_vm().state
host_inv = test_lib.lib_find_host_by_vm(vm_inv)
host_ip = host_inv.managementIp
host_username = host_inv.username
host_password = host_inv.password
test_util.test_dsc('check the vm console protocol is spice via read xml')
cmd = '''virsh dumpxml %s |grep spice|awk '{print $2}' |awk -F= '{print $2}' ''' % (
vm_uuid)
result = test_lib.lib_execute_ssh_cmd(host_ip, host_username,
host_password, cmd, 180)
if eval(result) == "spice":
print "Vm console protocol is spice, test success"
else:
print "Vm console protocol is %s " % eval(result)
print "test is fail"
test_util.test_fail('Create VM with spice Test Failed')
test_util.test_dsc('check the spiceStreamingMode is off via read xml')
cmd = '''virsh dumpxml %s |grep streaming|awk -F/ '{print $1}' |awk -F= '{print $2}' ''' % (
vm_uuid)
result = test_lib.lib_execute_ssh_cmd(host_ip, host_username,
host_password, cmd, 180)
if eval(result) == "off":
print "Vm spiceStreamingMode is off, test success"
else:
print "Vm spiceStreamingModeis %s " % eval(result)
print "test is fail"
test_util.test_fail(
'Create VM with spice and spiceStreamingMode is off Test Failed')
test_util.test_dsc(
'check the vm console protocol is spice via GetVmConsoleAddressAction')
if test_stub.get_vm_console_protocol(vm.vm.uuid).protocol == "spice":
print "Vm console protocol is spice, test success"
else:
print "Vm console protocol is %s " % test_stub.get_vm_console_protocol(
vm.vm.uuid).protocol
print "test is fail"
test_util.test_fail('Create VM with spice Test Failed')
vm.destroy()
vm.check()
test_util.test_pass('Create VM with spice Test Success')
def update_mn_vm_config(mn_vm_host,
option,
content,
scenarioConfig,
new_config="/tmp/zstack-ha-installer/config.json"):
cmd1 = 'sed -i "s/\\\"%s\\\": \\\".*\\\"/\\\"%s\\\": \\\"%s\\\"/g" %s' % (
option, option, content, new_config)
cmd2 = "zsha import-config %s" % new_config
host_config = sce_ops.get_scenario_config_vm(mn_vm_host.name_,
scenarioConfig)
test_lib.lib_execute_ssh_cmd(mn_vm_host.ip_, host_config.imageUsername_,
host_config.imagePassword_, cmd1)
test_lib.lib_execute_ssh_cmd(mn_vm_host.ip_, host_config.imageUsername_,
host_config.imagePassword_, cmd2)
def set_host_cpu_model(ip, model=None):
i = 0
_model = get_host_cpu_model(ip)
test_util.test_logger(_model)
while _model != model:
i += 1
if i > 5:
test_util.test_fail("set host cpu model faild")
cmd = '''sed -i "s/'{}'/'{}'/g" /usr/share/libvirt/cpu_map.xml'''.format(
_model, model)
test_lib.lib_execute_ssh_cmd(ip, 'root', 'password', cmd)
cmd = "systemctl restart libvirtd"
test_lib.lib_execute_ssh_cmd(ip, 'root', 'password', cmd)
_model = get_host_cpu_model(ip)
def recover_host_vlan(host_vm, scenarioConfig, deploy_config):
host_ip = host_vm.ip_
test_lib.lib_wait_target_up(host_ip, '22', 120)
host_config = sce_ops.get_scenario_config_vm(host_vm.name_,scenarioConfig)
for l3network in xmlobject.safe_list(host_config.l3Networks.l3Network):
if hasattr(l3network, 'l2NetworkRef'):
for l2networkref in xmlobject.safe_list(l3network.l2NetworkRef):
nic_name = sce_ops.get_ref_l2_nic_name(l2networkref.text_, deploy_config)
if nic_name.find('.') >= 0 :
vlan = nic_name.split('.')[1]
test_util.test_logger('[vm:] %s %s is created.' % (host_ip, nic_name.replace("eth", "zsn")))
cmd = 'vconfig add %s %s' % (nic_name.split('.')[0].replace("eth", "zsn"), vlan)
test_lib.lib_execute_ssh_cmd(host_ip, host_config.imageUsername_, host_config.imagePassword_, cmd)
return True
def test():
global vm
global default_mode
global default_mode1
default_mode = conf_ops.change_global_config('mevoco', 'vm.consoleMode',
'spice')
default_mode1 = conf_ops.change_global_config('vm', 'spiceStreamingMode',
'filter')
vm = test_stub.create_spice_vm()
vm.check()
vm_inv = vm.get_vm()
vm_uuid = vm_inv.uuid
vm_state = vm.get_vm().state
host_inv = test_lib.lib_find_host_by_vm(vm_inv)
host_ip = host_inv.managementIp
host_username = host_inv.username
host_password = host_inv.password
test_util.test_dsc('check the vm console protocol is spice via read xml')
cmd = '''virsh dumpxml %s |grep spice|awk '{print $2}' |awk -F= '{print $2}' ''' % (
vm_uuid)
result = test_lib.lib_execute_ssh_cmd(host_ip, host_username,
host_password, cmd, 180)
if eval(result) == "spice":
print "Vm console protocol is spice, test success"
else:
print "Vm console protocol is %s " % eval(result)
print "test is fail"
test_util.test_fail('Create VM with spice Test Failed')
test_util.test_dsc('check the spiceStreamingMode is filter via read xml')
cmd = '''virsh dumpxml %s |grep streaming|awk -F/ '{print $1}' |awk -F= '{print $2}' ''' % (
vm_uuid)
result = test_lib.lib_execute_ssh_cmd(host_ip, host_username,
host_password, cmd, 180)
if eval(result) == "filter":
print "Vm spiceStreamingMode is filter, test success"
else:
print "Vm spiceStreamingModeis %s " % eval(result)
print "test is fail"
test_util.test_fail(
'Create VM with spice and spiceStreamingMode is filter Test Failed'
)
vm.destroy()
vm.check()
test_util.test_pass(
'Create VM with spice and spiceStreamingMode is filter Test Success')
def compare(ps, vm, backup):
test_util.test_logger("-----------------compare----------------")
# find vm_host
host = test_lib.lib_find_host_by_vm(vm.vm)
cond = res_ops.gen_query_conditions("type", '=', "ImageStoreBackupStorage")
bs = res_ops.query_resource(res_ops.BACKUP_STORAGE, cond)[0]
root_volume = test_lib.lib_get_root_volume(vm.get_vm())
for i in test_lib.lib_get_data_volumes(vm.get_vm()):
if i.name == "DATA-for-test_vm":
data_volume = i
vm_path = root_volume.installPath
data_path = data_volume.installPath
if ps.type == "SharedBlock":
vm_path = "/dev/" + root_volume.installPath.split("/")[2] + "/" + root_volume.installPath.split("/")[3]
data_path = "/dev/" + data_volume.installPath.split("/")[2] + "/" + data_volume.installPath.split("/")[3]
test_util.test_logger(vm_path)
test_util.test_logger(data_path)
for i in backup:
if i.type == "Root":
name = i.backupStorageRefs[0].installPath.split("/")[2]
id = i.backupStorageRefs[0].installPath.split("/")[3]
if i.type == "Data" and "DATA-for-test_vm" in i.metadata:
name1 = i.backupStorageRefs[0].installPath.split("/")[2]
id1 = i.backupStorageRefs[0].installPath.split("/")[3]
# compare vm_root_volume & image
cmd = "mkdir /root/%s;" \
"/usr/local/zstack/imagestore/bin/zstcli " \
"-rootca=/var/lib/zstack/imagestorebackupstorage/package/certs/ca.pem " \
"-url=%s:8000 " \
"pull -installpath /root/%s/old.qcow2 %s:%s;" \
"qemu-img compare %s /root/%s/old.qcow2;" % (id, bs.hostname, id, name, id, vm_path, id)
cmd1 = "mkdir /root/%s;" \
"/usr/local/zstack/imagestore/bin/zstcli " \
"-rootca=/var/lib/zstack/imagestorebackupstorage/package/certs/ca.pem " \
"-url=%s:8000 " \
"pull -installpath /root/%s/old.qcow2 %s:%s;" \
"qemu-img compare %s /root/%s/old.qcow2;" % (id1, bs.hostname, id1, name1, id1, data_path, id1)
# clean image
result = test_lib.lib_execute_ssh_cmd(host.managementIp, "root", "password", cmd, timeout=300)
if result != "Images are identical.\n":
test_util.test_fail("compare vm_root_volume & image created by backup")
result = test_lib.lib_execute_ssh_cmd(host.managementIp, "root", "password", cmd1, timeout=300)
if result != "Images are identical.\n":
test_util.test_fail("compare vm_data_volume & image created by backup")
def test():
test_util.test_dsc('Test Host ntp')
cmd = 'hostname'
mn = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0]
cmd = 'ntpq -p'
for host in test_lib.lib_get_all_hosts_from_plan():
test_util.test_logger("host.managementIp_: %s" % (host.managementIp_))
test_util.test_logger("mn.hostName: %s" % (mn.hostName))
test_util.test_logger(
"anotherIp: %s" % (test_stub.get_another_ip_of_host(
host.managementIp_, host.username_, host.password_)))
if host.managementIp_ == mn.hostName or test_stub.get_another_ip_of_host(
host.managementIp_, host.username_,
host.password_) == mn.hostName:
output = test_lib.lib_execute_ssh_cmd(host.managementIp_,
host.username_,
host.password_,
cmd,
timeout=30)
if output.find(mn.hostName.replace('.', '-')) >= 0:
test_util.test_fail(
'if host and MN are same host, its not expected to use itself'
)
else:
output = test_lib.lib_execute_ssh_cmd(host.managementIp_,
host.username_,
host.password_,
cmd,
timeout=30)
test_util.test_logger("output: %s" % (output))
if not output:
test_util.test_dsc('try ssh port 2222')
cmd = '/usr/sbin/ntpq -p'
output = test_lib.lib_execute_ssh_cmd(host.managementIp_,
host.username_,
host.password_,
cmd,
timeout=30,
port=2222)
if not output:
test_util.test_fail('can not ssh in vm[%s]' %
host.managementIp_)
if output.find(mn.hostName.replace('.', '-')) < 0 and output.find(
mn.hostName) < 0:
test_util.test_fail('all host expect to use MN ntp service')
test_util.test_pass('Test Host ntp Pass')
def test():
global vm
global node1_ip
global node2_ip
node1_ip = os.environ.get('node1Ip')
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
end_time = int(time.time())
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
time.sleep(180)
host_uuid = vm.get_vm().hostUuid
data1 = test_lib.lib_get_host_cpu_prometheus_data(node1_ip, end_time, 200, host_uuid)
data2 = test_lib.lib_get_host_cpu_prometheus_data(node2_ip, end_time, 200, host_uuid)
if cmp(data1,data2) != 0:
test_util.test_fail('Prometheus Sync Test Fail, data on node:%s is [%s], data on node:%s is [%s]' % (node1_ip, data1, node2_ip, data2))
test_util.test_pass('Prometheus Sync Test Success')
def test():
#skip ceph in c74
cmd = "cat /etc/redhat-release | grep '7.4'"
mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
rsp = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
if rsp != False:
ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)
for i in ps:
if i.type == 'Ceph':
test_util.test_skip('cannot hotplug iso to the vm in ceph,it is a libvirt bug:https://bugzilla.redhat.com/show_bug.cgi?id=1541702.')
multi_iso.add_iso_image()
multi_iso.create_windows_vm()
test_obj_dict.add_vm(multi_iso.vm1)
multi_iso.get_all_iso_uuids()
multi_iso.attach_iso(multi_iso.iso_uuids[0])
multi_iso.attach_iso(multi_iso.iso_uuids[1])
multi_iso.attach_iso(multi_iso.iso_uuids[2])
multi_iso.check_windows_vm_cdrom(3)
multi_iso.detach_iso(multi_iso.iso_uuids[1])
multi_iso.check_windows_vm_cdrom(2)
# multi_iso.vm1.reboot()
multi_iso.detach_iso(multi_iso.iso_uuids[0])
multi_iso.check_windows_vm_cdrom(1)
multi_iso.detach_iso(multi_iso.iso_uuids[2])
multi_iso.check_windows_vm_cdrom(0)
test_lib.lib_robot_cleanup(test_obj_dict)
test_util.test_pass('Attach 3 ISO Test Success')
def env_recover():
test_lib.lib_error_cleanup(test_obj_dict)
for delete_ps in delete_ps_list:
ps_config = test_util.PrimaryStorageOption()
ps_config.set_name(delete_ps.name)
ps_config.set_description(delete_ps.description)
ps_config.set_zone_uuid(delete_ps.zoneUuid)
ps_config.set_type(delete_ps.type)
ps_config.set_url(delete_ps.url)
if delete_ps.type == inventory.LOCAL_STORAGE_TYPE:
new_ps = ps_ops.create_local_primary_storage(ps_config)
elif delete_ps.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
new_ps = ps_ops.create_nfs_primary_storage(ps_config)
elif delete_ps.type == "SharedBlock":
host = random.choice(res_ops.query_resource(res_ops.HOST))
cmd = "vgchange --lock-start %s && vgremove %s -y" % (
delete_ps.uuid, delete_ps.uuid)
host_username = os.environ.get('hostUsername')
host_password = os.environ.get('hostPassword')
rsp = test_lib.lib_execute_ssh_cmd(host.managementIp,
host_username, host_password,
cmd, 240)
if not rsp:
test_util.test_logger("vgremove failed")
new_ps = ps_ops.create_sharedblock_primary_storage(
ps_config, disk_uuid)
else:
new_ps = None
ps_ops.attach_primary_storage(
new_ps.uuid,
res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
def test():
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop kairosdb on node: %s" % (node_ip))
cmd = "zstack-ctl kairosdb stop"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("check if it still works")
zstack_ha_vip = os.environ.get('zstackHaVip')
if not test_lib.lib_network_check(zstack_ha_vip, 58080):
test_util.test_fail('Could not access UI through VIP: %s, port: 58080' % (zstack_ha_vip))
cmd = "zstack-ctl kairosdb start"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_pass('Create VM Test Rabbitmq Stop on one node Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node_ip))
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_util.test_logger(
"wait for 2 minutes to see if http api still works well")
time.sleep(180)
test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s" % (node_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
test_stub.exercise_connection(600)
test_util.test_pass('Create VM Test Success')
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network: %s" % (node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s" % (l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("wait for 2 minutes to see if http api still works well")
time.sleep(180)
#test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s, and create vm" % (node_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
#test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass('Create VM Test with one node network down and recovery Success')
def check(self):
super(zstack_kvm_image_file_checker, self).check()
image = self.test_obj.image
backupStorages = image.backupStorageRefs
bs_one = backupStorages[0]
bs = test_lib.lib_get_backup_storage_by_uuid(bs_one.backupStorageUuid)
if bs.type == inventory.SFTP_BACKUP_STORAGE_TYPE:
host = test_lib.lib_get_backup_storage_host(bs_one.backupStorageUuid)
image_url = backupStorages[0].installPath
self.judge(test_lib.lib_check_file_exist(host, image_url))
elif bs.type == inventory.CEPH_BACKUP_STORAGE_TYPE:
if self.test_obj.state == image_header.DELETED:
#https://github.com/zstackorg/zstack/issues/93#issuecomment-130935998
test_util.test_logger("skip ceph image delete check, since the image won't be deleted until no vms refer to it.")
return self.judge(self.exp_result)
ceph_host, username, password = test_lib.lib_get_ceph_info(os.environ.get('cephBackupStorageMonUrls'))
image_installPath = bs_one.installPath.split('ceph://')[1]
command = 'rbd info %s' % image_installPath
if test_lib.lib_execute_ssh_cmd(ceph_host, username, password, command, 10):
test_util.test_logger('Check result: [image:] %s [file:] %s exist on ceph [host name:] %s .' % (image.uuid, image_installPath, ceph_host))
return self.judge(True)
else:
test_util.test_logger('Check result: [image:] %s [file:] %s does not exist on ceph [host name:] %s .' % (image.uuid, image_installPath, ceph_host))
return self.judge(False)
def check(self):
super(zstack_kvm_image_file_checker, self).check()
image = self.test_obj.image
backupStorages = image.backupStorageRefs
bs_one = backupStorages[0]
bs = test_lib.lib_get_backup_storage_by_uuid(bs_one.backupStorageUuid)
if bs.type == inventory.SFTP_BACKUP_STORAGE_TYPE:
self.judge(test_lib.lib_check_backup_storage_image_file(image))
elif hasattr(inventory, 'IMAGE_STORE_BACKUP_STORAGE_TYPE') and bs.type == inventory.IMAGE_STORE_BACKUP_STORAGE_TYPE:
if self.test_obj.state == image_header.DELETED:
test_util.test_logger("skip image store image delete check, since the image won't be deleted until no vms refer to it.")
return self.judge(self.exp_result)
self.judge(test_lib.lib_check_backup_storage_image_file(image))
elif bs.type == inventory.CEPH_BACKUP_STORAGE_TYPE:
if self.test_obj.state == image_header.DELETED:
#https://github.com/zstackorg/zstack/issues/93#issuecomment-130935998
test_util.test_logger("skip ceph image delete check, since the image won't be deleted until no vms refer to it.")
return self.judge(self.exp_result)
if test_lib.scenario_config != None and test_lib.scenario_file != None and os.path.exists(test_lib.scenario_file):
import zstackwoodpecker.operations.scenario_operations as sce_ops
sce_ops.replace_env_params_if_scenario()
ceph_host, username, password = test_lib.lib_get_ceph_info(os.environ.get('cephBackupStorageMonUrls'))
image_installPath = bs_one.installPath.split('ceph://')[1]
command = 'rbd info %s' % image_installPath
if test_lib.lib_execute_ssh_cmd(ceph_host, username, password, command, 10):
test_util.test_logger('Check result: [image:] %s [file:] %s exist on ceph [host name:] %s .' % (image.uuid, image_installPath, ceph_host))
return self.judge(True)
else:
test_util.test_logger('Check result: [image:] %s [file:] %s does not exist on ceph [host name:] %s .' % (image.uuid, image_installPath, ceph_host))
return self.judge(False)
def test():
global vm
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node network: %s" % (node_ip))
l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
cmd = "ifdown %s" % (l2_network_interface)
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
test_util.test_logger(
"wait for 2 minutes to see if http api still works well")
time.sleep(180)
#test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_logger("recover node: %s, and create vm" % (node_ip))
os.system('bash -ex %s %s' %
(os.environ.get('nodeRecoverScript'), node_ip))
time.sleep(180)
#test_stub.exercise_connection(600)
vm = test_stub.create_basic_vm()
vm.check()
vm.destroy()
test_util.test_pass(
'Create VM Test with one node network down and recovery Success')
def test():
global vm
loop = 20
count = 0
vm_creation_option = test_util.VmOption()
image_name = os.environ.get('imageName_net')
l3_name = os.environ.get('l3VlanNetworkName1')
vm = test_stub.create_vm("test_libvirt_snapshot_vm", image_name, l3_name)
test_obj_dict.add_vm(vm)
vm.check()
volume_uuid = test_lib.lib_get_root_volume(vm.get_vm()).uuid
snapshots = test_obj_dict.get_volume_snapshot(volume_uuid)
snapshots.set_utility_vm(vm)
while count <= loop:
sp_name = "create_snapshot" + str(count)
snapshots.create_snapshot(sp_name)
snapshots.check()
count += 1
host = test_lib.lib_find_host_by_vm(vm.get_vm())
result = test_lib.lib_execute_ssh_cmd(host.managementIp, 'root', 'password', 'service libvirtd restart')
time.sleep(600)
vm.check()
snapshots.check()
snapshots.delete()
test_obj_dict.rm_volume_snapshot(snapshots)
test_lib.lib_error_cleanup(test_obj_dict)
test_util.test_pass('Libvirt restart with snapshot passed')
def compare(ps, vm, dvol, backup):
test_util.test_logger("-----------------compare----------------")
# find vm_host
host = test_lib.lib_find_host_by_vm(vm.vm)
bs = res_ops.query_resource(res_ops.BACKUP_STORAGE)[0]
cond = res_ops.gen_query_conditions("uuid", '=', dvol.volume.uuid)
current_volume = res_ops.query_resource(res_ops.VOLUME, cond)[0]
vol_path = current_volume.installPath
if ps.type == "SharedBlock":
vol_path = "/dev/" + current_volume.installPath.split("/")[2] + "/" + current_volume.installPath.split("/")[3]
test_util.test_logger(vol_path)
name = backup.backupStorageRefs[0].installPath.split("/")[2]
id = backup.backupStorageRefs[0].installPath.split("/")[3]
# compare vm_root_volume & image
cmd = "mkdir /root/%s;" \
"/usr/local/zstack/imagestore/bin/zstcli " \
"-rootca=/var/lib/zstack/imagestorebackupstorage/package/certs/ca.pem " \
"-url=%s:8000 " \
"pull -installpath /root/%s/old.qcow2 %s:%s;" \
"qemu-img compare %s /root/%s/old.qcow2;" % (id, bs.hostname, id, name, id, vol_path, id)
# clean image
result = test_lib.lib_execute_ssh_cmd(host.managementIp, "root", "password", cmd, timeout=300)
if result != "Images are identical.\n":
test_util.test_fail("compare vm_root_volume & image created by backup")
def get_qcow_backing_file_by_ip(installPath, hostIp):
'''
get backing file with given install Path
'''
cmd = 'qemu-img info %s | grep "backing file:" | awk \'{print $3}\'' % installPath
return test_lib.lib_execute_ssh_cmd(hostIp, 'root', 'password', cmd)
def check_ceph(self, volume, volume_installPath, ps):
monHost = ps.mons[0].hostname
for key in os.environ.keys():
if monHost in os.environ.get(key):
ceph_host, username, password = \
test_lib.lib_get_ceph_info(os.environ.get(key))
break
else:
ceph_host = monHost
username = '******'
password = '******'
volume_installPath = volume_installPath.split('ceph://')[1]
command = 'rbd info %s' % volume_installPath
if test_lib.lib_execute_ssh_cmd(ceph_host, username, password, command,
10):
test_util.test_logger(
'Check result: [volume:] %s [file:] %s exist on ceph [host name:] %s .'
% (volume.uuid, volume_installPath, ceph_host))
return self.judge(True)
else:
test_util.test_logger(
'Check result: [volume:] %s [file:] %s does NOT exist on ceph [host name:] %s .'
% (volume.uuid, volume_installPath, ceph_host))
return self.judge(False)
def error_cleanup():
global node_ip
cmd = "zstack-ctl start_ui"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password,
cmd, 180)
def test():
host = test_lib.lib_get_all_hosts_from_plan()[0]
cmd = 'cat /sys/module/kvm_intel/parameters/nested'
cmd_out = test_lib.lib_execute_ssh_cmd(host.managementIp_, host.username_,
host.password_, cmd, 180)
if 'Y' not in cmd_out:
test_util.test_fail('nested virt not enabled')
cmd = 'cat /etc/modprobe.d/kvm-nested.conf'
cmd_out = test_lib.lib_execute_ssh_cmd(host.managementIp_, host.username_,
host.password_, cmd, 180)
if 'options kvm_intel nested=1' not in cmd_out:
test_util.test_fail(
'nested virt not enabled in /etc/modprobe.d/kvm_nested.conf')
test_util.test_pass('Enable nested virt Test Success')
def wait_for_mn_ha_ready(scenarioConfig, scenarioFile):
mn_host_list = get_mn_host(scenarioConfig, scenarioFile)
if len(mn_host_list) < 1:
return []
for i in range(0, 5):
for host in mn_host_list:
host_config = sce_ops.get_scenario_config_vm(host.name_, scenarioConfig)
try:
cmd = "zsha status"
zsha_status = test_lib.lib_execute_ssh_cmd(host.ip_, host_config.imageUsername_, host_config.imagePassword_,cmd)
if zsha_status.find('3 osds: 3 up, 3 in') <= 0:
continue
if zsha_status.count('alive') < 3:
continue
if zsha_status.count(': ceph') < 3:
continue
if zsha_status.count(': running') < 1:
continue
if zsha_status.count('3 mons at') < 1:
continue
return True
except:
continue
time.sleep(10)
return False
def compare(ps, vm, dvol, backup):
test_util.test_logger("-----------------compare----------------")
# find vm_host
host = test_lib.lib_find_host_by_vm(vm.vm)
bs = res_ops.query_resource(res_ops.BACKUP_STORAGE)[0]
cond = res_ops.gen_query_conditions("uuid", '=', dvol.volume.uuid)
current_volume = res_ops.query_resource(res_ops.VOLUME, cond)[0]
vol_path = current_volume.installPath
if ps.type == "SharedBlock":
vol_path = "/dev/" + current_volume.installPath.split(
"/")[2] + "/" + current_volume.installPath.split("/")[3]
test_util.test_logger(vol_path)
name = backup.backupStorageRefs[0].installPath.split("/")[2]
id = backup.backupStorageRefs[0].installPath.split("/")[3]
# compare vm_root_volume & image
cmd = "mkdir /root/%s;" \
"/usr/local/zstack/imagestore/bin/zstcli " \
"-rootca=/var/lib/zstack/imagestorebackupstorage/package/certs/ca.pem " \
"-url=%s:8000 " \
"pull -installpath /root/%s/old.qcow2 %s:%s;" \
"qemu-img compare %s /root/%s/old.qcow2;" % (id, bs.hostname, id, name, id, vol_path, id)
# clean image
result = test_lib.lib_execute_ssh_cmd(host.managementIp,
"root",
"password",
cmd,
timeout=300)
if result != "Images are identical.\n":
test_util.test_fail("compare vm_root_volume & image created by backup")
def test():
global node_ip
node_ip = os.environ.get('node1Ip')
test_util.test_logger("stop ui on node: %s" % (node_ip))
cmd = "zstack-ctl stop_ui"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_logger("check if it still works")
zstack_ha_vip = os.environ.get('zstackHaVip')
if not test_lib.lib_network_check(zstack_ha_vip, 8888):
test_util.test_fail('Could not access UI through VIP: %s, port: 8888' % (zstack_ha_vip))
cmd = "zstack-ctl start_ui"
rsp = test_lib.lib_execute_ssh_cmd(node_ip, host_username, host_password, cmd, 180)
test_util.test_pass('Create VM Test UI Stop on one node Success')
def get_ssh_ip_result(self):
vm = self.vm_list[0]
host = test_lib.lib_get_vm_host(vm)
port = self.lbl.get_creation_option().get_load_balancer_port()
iport = self.lbl.get_creation_option().get_instance_port()
if iport == 22:
vm_command = '/sbin/ip a|grep inet'
vm_cmd_result = test_lib.lib_execute_ssh_cmd(self.vip_ip, \
test_lib.lib_get_vm_username(vm), \
test_lib.lib_get_vm_password(vm), \
vm_command,\
port = port)
if iport == 80:
vm_command = 'curl %s:%s' % (self.vip_ip, port)
vm_cmd_result = shell.call('%s' % vm_command)
if not vm_cmd_result:
test_util.test_logger(
'Checker result: FAIL to execute test ssh command in vip: %s for lb: %s.'
% (self.vip_ip, self.lbl_uuid))
return False
for key, values in self.vm_ip_test_dict.iteritems():
if key in vm_cmd_result:
self.vm_ip_test_dict[key] += 1
break
return True
def env_recover():
test_lib.lib_error_cleanup(test_obj_dict)
for delete_ps in delete_ps_list:
ps_config = test_util.PrimaryStorageOption()
ps_config.set_name(delete_ps.name)
ps_config.set_description(delete_ps.description)
ps_config.set_zone_uuid(delete_ps.zoneUuid)
ps_config.set_type(delete_ps.type)
ps_config.set_url(delete_ps.url)
if delete_ps.type == inventory.LOCAL_STORAGE_TYPE:
new_ps = ps_ops.create_local_primary_storage(ps_config)
elif delete_ps.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
new_ps = ps_ops.create_nfs_primary_storage(ps_config)
elif delete_ps.type == "SharedBlock":
host = random.choice(res_ops.query_resource(res_ops.HOST))
cmd = "vgchange --lock-start %s && vgremove %s -y" % (delete_ps.uuid, delete_ps.uuid)
host_username = os.environ.get('hostUsername')
host_password = os.environ.get('hostPassword')
rsp = test_lib.lib_execute_ssh_cmd(host.managementIp, host_username, host_password, cmd, 240)
if not rsp:
test_util.test_logger("vgremove failed")
new_ps = ps_ops.create_sharedblock_primary_storage(ps_config, disk_uuid)
else:
new_ps = None
ps_ops.attach_primary_storage(new_ps.uuid, res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
def test():
global vm
vm = test_stub.create_basic_vm()
vm.check()
start_date = int(time.time())
schd = vm_ops.reboot_vm_scheduler(vm.get_vm().uuid, 'simple', 'simple_reboot_vm_scheduler', start_date+60, 30)
cmd = "init 0"
host_username = os.environ.get('nodeUserName')
host_password = os.environ.get('nodePassword')
zstack_ha_vip = os.environ.get('zstackHaVip')
node1_ip = os.environ.get('node1Ip')
test_util.test_logger("shutdown node: %s" % (node1_ip))
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
node2_ip = os.environ.get('node2Ip')
test_util.test_logger("shutdown node: %s" % (node2_ip))
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
test_util.test_logger("recover node: %s" % (node1_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node1_ip))
test_util.test_logger("recover node: %s" % (node2_ip))
os.system('bash -ex %s %s' % (os.environ.get('nodeRecoverScript'), node2_ip))
cmd = "zstack-ctl install_ha --host1-info %s:%s@%s --host2-info %s:%s@%s --vip %s --recovery-from-this-host" % \
(host_username, host_password, node1_ip, host_username, host_password, node2_ip, zstack_ha_vip)
rsp = test_lib.lib_execute_ssh_cmd(node1_ip, host_username, host_password, cmd, 180)
if not rsp:
rsp = test_lib.lib_execute_ssh_cmd(node2_ip, host_username, host_password, cmd, 180)
time.sleep(180)
test_stub.exercise_connection(600)
time.sleep(180)
scheduler_execution_count = 0
for i in range(0, 30):
if test_lib.lib_find_in_remote_management_server_log(node1_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count += 1
if test_lib.lib_find_in_remote_management_server_log(node2_ip, host_username, host_password, start_date+60+30*i, '[msg received]: {"org.zstack.header.vm.RebootVmInstanceMsg', vm.get_vm().uuid):
scheduler_execution_count -= 1
if abs(scheduler_execution_count) < 5:
test_util.test_fail('VM reboot scheduler is expected to executed for more than 5 times, while it only execute %s times' % (scheduler_execution_count))
schd_ops.delete_scheduler(schd.uuid)
vm.destroy()
test_util.test_pass('Scheduler Test Success')
def check_vm_running_exist_on_host(vm_uuid, host_ip, scenarioConfig, scenarioFile):
mn_host_list = get_mn_host(scenarioConfig, scenarioFile)
cmd = "virsh list|grep %s|awk '{print $3}'" %(vm_uuid)
host = mn_host_list[0]
host_config = sce_ops.get_scenario_config_vm(host.name_, scenarioConfig)
vm_is_exist = True if test_lib.lib_execute_ssh_cmd(host_ip, host_config.imageUsername_, host_config.imagePassword_,cmd) else False
return vm_is_exist
def test():
mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
conditions = res_ops.gen_query_conditions('state', '=', 'Enabled')
conditions = res_ops.gen_query_conditions('status', '=', 'Connected',
conditions)
conditions = res_ops.gen_query_conditions('managementIp', '!=', mn_ip,
conditions)
host = random.choice(res_ops.query_resource(res_ops.HOST, conditions))
record['host_ip'] = host.managementIp
record['host_port'] = host.sshPort
record['host_uuid'] = host.uuid
test_util.test_dsc("Create {0} vm each with {1} datavolume".format(
VM_COUNT, DATA_VOLUME_NUMBER))
vm_list = test_stub.create_multi_vms(name_prefix='test-',
count=VM_COUNT,
data_volume_number=DATA_VOLUME_NUMBER,
host_uuid=host.uuid)
for vm in vm_list:
test_obj_dict.add_vm(vm)
test_util.test_logger("host %s is disconnecting" % host.managementIp)
for vm in vm_list:
ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop")
#l2_network_interface = os.environ.get('l2ManagementNetworkInterface')
l2interface = test_lib.lib_get_l2s_by_vm(vm.get_vm())[0].physicalInterface
l2_network_interface = test_stub.get_host_l2_nic_name(l2interface)
cmd = "ifdown %s && sleep 180 && ifup %s" % (l2_network_interface,
l2_network_interface)
host_username = os.environ.get('hostUsername')
host_password = os.environ.get('hostPassword')
rsp = test_lib.lib_execute_ssh_cmd(host.managementIp, host_username,
host_password, cmd, 240)
if not rsp:
test_util.test_logger(
"host is expected to shutdown after its network down for a while")
test_util.test_logger("wait for 180 seconds")
time.sleep(180)
for vm in vm_list:
vm.update()
if test_lib.lib_find_host_by_vm(
vm.get_vm()).managementIp == host.managementIp:
test_util.test_fail(
"VM is expected to start running on another host")
vm.set_state(vm_header.RUNNING)
vm.check()
cmd = 'PORT=%s bash -ex %s %s' % (
host.sshPort, os.environ.get('hostRecoverScript'), host.managementIp)
test_util.test_logger(cmd)
os.system(cmd)
host_ops.reconnect_host(host.uuid)
test_util.test_pass('Multi PrimaryStorage Vm HA Test Pass')
def recover_host(host_vm, scenarioConfig, deploy_config):
stop_host(host_vm, scenarioConfig)
host_inv = start_host(host_vm, scenarioConfig)
if not host_inv:
return False
host_ip = host_vm.ip_
test_lib.lib_wait_target_up(host_ip, '22', 120)
host_config = sce_ops.get_scenario_config_vm(host_inv.name,scenarioConfig)
for l3network in xmlobject.safe_list(host_config.l3Networks.l3Network):
if hasattr(l3network, 'l2NetworkRef'):
for l2networkref in xmlobject.safe_list(l3network.l2NetworkRef):
nic_name = sce_ops.get_ref_l2_nic_name(l2networkref.text_, deploy_config)
if nic_name.find('.') >= 0 :
vlan = nic_name.split('.')[1]
test_util.test_logger('[vm:] %s %s is created.' % (host_ip, nic_name.replace("eth","zsn")))
cmd = 'vconfig add %s %s' % (nic_name.split('.')[0].replace("eth","zsn"), vlan)
test_lib.lib_execute_ssh_cmd(host_ip, host_config.imageUsername_, host_config.imagePassword_, cmd)
return True
