def env_recover():
    test_util.test_dsc("Destroy test object")
    test_lib.lib_error_cleanup(test_obj_dict)
    if new_ps_list:
        for new_ps in new_ps_list:
            ps_ops.detach_primary_storage(new_ps.uuid, new_ps.attachedClusterUuids[0])
            ps_ops.delete_primary_storage(new_ps.uuid)
Ejemplo n.º 2
0
def test():
    global host_config
    global ps_inv
    curr_deploy_conf = exp_ops.export_zstack_deployment_config(
        test_lib.deploy_config)

    host1 = res_ops.get_resource(res_ops.HOST, name=host1_name)[0]

    host_config.set_name(host1_name)
    host_config.set_cluster_uuid(host1.clusterUuid)
    host_config.set_management_ip(host1.managementIp)
    host_config.set_username(os.environ.get('hostUsername'))
    host_config.set_password(os.environ.get('hostPassword'))

    test_util.test_dsc('delete host')
    host_ops.delete_host(host1.uuid)

    test_util.test_dsc('delete primary storage')
    ps_name = os.environ.get('nfsPrimaryStorageName1')
    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name=ps_name)[0]
    ps_ops.delete_primary_storage(ps_inv.uuid)

    test_util.test_dsc("Recover Primary Storage")
    recover_ps()
    test_util.test_dsc("Recover Host")
    host_ops.add_kvm_host(host_config)

    host1 = res_ops.get_resource(res_ops.HOST, name=host1_name)[0]
    ps1 = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name=ps_name)[0]

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Delete Host and Primary Storage Test Success')
def test():
    global host_config
    global ps_inv
    curr_deploy_conf = exp_ops.export_zstack_deployment_config(test_lib.deploy_config)

    host1 = res_ops.get_resource(res_ops.HOST, name = host1_name)[0]

    host_config.set_name(host1_name)
    host_config.set_cluster_uuid(host1.clusterUuid)
    host_config.set_management_ip(host1.managementIp)
    host_config.set_username(os.environ.get('hostUsername'))
    host_config.set_password(os.environ.get('hostPassword'))

    test_util.test_dsc('delete host')
    host_ops.delete_host(host1.uuid)

    test_util.test_dsc('delete primary storage')
    ps_name = os.environ.get('nfsPrimaryStorageName1')
    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name)[0]
    ps_ops.delete_primary_storage(ps_inv.uuid)

    test_util.test_dsc("Recover Primary Storage")
    recover_ps()
    test_util.test_dsc("Recover Host")
    host_ops.add_kvm_host(host_config)

    host1 = res_ops.get_resource(res_ops.HOST, name = host1_name)[0]
    ps1 = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name)[0]

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Delete Host and Primary Storage Test Success')
def error_cleanup():
    global email_endpoint_uuid,email_platform_uuid,my_sns_topic_uuid,host_sns_topic_uuid,ps_uuid,hostname,host_management_ip,host_status,bs_type,bs_status,event_list
    if host_status == 'Disconnected':
        host_ops.update_host(host_uuid, 'managementIp', host_management_ip)
        host_ops.reconnect_host(host_uuid)
    if bs_status == 'Disconnected':
        if bs_type == res_ops.IMAGE_STORE_BACKUP_STORAGE:
            bs_ops.update_image_store_backup_storage_info(bs_uuid, 'hostname', hostname)
            bs_ops.reconnect_backup_storage(bs_uuid)
        elif bs_type == res_ops.SFTP_BACKUP_STORAGE:
            bs_ops.update_sftp_backup_storage_info(bs_uuid, 'hostname', hostname)
            bs_ops.reconnect_backup_storage(bs_uuid)
    if ps_uuid:
        ps_ops.delete_primary_storage(ps_uuid)
    if host_sns_topic_uuid:
        zwt_ops.delete_sns_topic(host_sns_topic_uuid)
    if my_sns_topic_uuid:
        zwt_ops.delete_sns_topic(my_sns_topic_uuid)
    if event_list:
        for event_uuid in event_list:
            zwt_ops.unsubscribe_event(event_uuid)
    if email_endpoint_uuid:
        zwt_ops.delete_sns_application_endpoint(email_endpoint_uuid)
    if email_platform_uuid:
        zwt_ops.delete_sns_application_platform(email_platform_uuid)
def env_recover():
    for ps in detached_ps_list:
        ps_ops.attach_primary_storage(ps.uuid, res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
    test_lib.lib_error_cleanup(test_obj_dict)
    if new_ps_list:
        for new_ps in new_ps_list:
            ps_ops.detach_primary_storage(new_ps.uuid, new_ps.attachedClusterUuids[0])
            ps_ops.delete_primary_storage(new_ps.uuid)
def env_recover():
    test_lib.lib_error_cleanup(test_obj_dict)
    for disabled_ps in disabled_ps_list:
        ps_ops.change_primary_storage_state(disabled_ps.uuid, state='enable')
    if new_ps_list:
        for new_ps in new_ps_list:
            ps_ops.detach_primary_storage(new_ps.uuid, new_ps.attachedClusterUuids[0])
            ps_ops.delete_primary_storage(new_ps.uuid)
Ejemplo n.º 7
0
def env_recover():
    test_lib.lib_error_cleanup(test_obj_dict)
    for disabled_ps in disabled_ps_list:
        ps_ops.change_primary_storage_state(disabled_ps.uuid, state='enable')
    if new_ps_list:
        for new_ps in new_ps_list:
            ps_ops.detach_primary_storage(new_ps.uuid, new_ps.attachedClusterUuids[0])
            ps_ops.delete_primary_storage(new_ps.uuid)
def env_recover():
    for ps in detached_ps_list:
        ps_ops.attach_primary_storage(
            ps.uuid,
            res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
    test_lib.lib_error_cleanup(test_obj_dict)
    if new_ps_list:
        for new_ps in new_ps_list:
            ps_ops.detach_primary_storage(new_ps.uuid,
                                          new_ps.attachedClusterUuids[0])
            ps_ops.delete_primary_storage(new_ps.uuid)
Ejemplo n.º 9
0
def test():
    ps_env = test_stub.PSEnvChecker()

    ps1, ps2 = ps_env.get_two_ps()

    for _ in xrange(5):
        test_util.test_dsc('Remove ps2')
        ps_ops.detach_primary_storage(
            ps2.uuid,
            res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
        ps_ops.delete_primary_storage(ps2.uuid)
        delete_ps_list.append(ps2)
        test_util.test_dsc('Add ps2 back')
        ps_config = test_util.PrimaryStorageOption()
        ps_config.set_name(ps2.name)
        ps_config.set_description(ps2.description)
        ps_config.set_zone_uuid(ps2.zoneUuid)
        ps_config.set_type(ps2.type)
        ps_config.set_url(ps2.url)
        if ps2.type == inventory.LOCAL_STORAGE_TYPE:
            ps2 = ps_ops.create_local_primary_storage(ps_config)
        elif ps2.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
            ps2 = ps_ops.create_nfs_primary_storage(ps_config)
        else:
            ps2 = None
        time.sleep(5)
        ps_ops.attach_primary_storage(
            ps2.uuid,
            res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
        time.sleep(5)
        delete_ps_list.pop()

    test_util.test_dsc('create VM by default para')
    vm1 = test_stub.create_multi_vms(name_prefix='vm1',
                                     count=1,
                                     data_volume_number=VOLUME_NUMBER)[0]
    test_obj_dict.add_vm(vm1)

    if ps_env.is_local_nfs_env:
        test_util.test_dsc('create date volume in ps2')
        volume = test_stub.create_multi_volumes(count=VOLUME_NUMBER, ps=ps2)
        test_obj_dict.add_volume(volume)
    else:
        test_util.test_dsc('create VM in ps2')
        vm2 = test_stub.create_multi_vms(name_prefix='vm2',
                                         count=1,
                                         ps_uuid=ps2.uuid,
                                         data_volume_number=VOLUME_NUMBER)[0]
        test_obj_dict.add_vm(vm2)

    test_util.test_pass('Multi PrimaryStorage Test Pass')
Ejemplo n.º 10
0
def test():
    global host_config
    global ps_inv
    curr_deploy_conf = exp_ops.export_zstack_deployment_config(
        test_lib.deploy_config)

    host1 = res_ops.get_resource(res_ops.HOST, name=host1_name)[0]

    host_config.set_name(host1_name)
    host_config.set_cluster_uuid(host1.clusterUuid)
    host_config.set_management_ip(host1.managementIp)
    host_config.set_username(os.environ.get('hostUsername'))
    host_config.set_password(os.environ.get('hostPassword'))

    test_util.test_dsc('delete host')
    host_ops.delete_host(host1.uuid)

    test_util.test_dsc('delete primary storage')
    zone_name = os.environ.get('zoneName1')
    zone_uuid = res_ops.get_resource(res_ops.ZONE, name=zone_name)[0].uuid
    cond = res_ops.gen_query_conditions('zoneUuid', '=', zone_uuid)
    ps_inv = res_ops.query_resource(res_ops.PRIMARY_STORAGE, cond)[0]
    if ps_inv.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
        ps_name = os.environ.get('nfsPrimaryStorageName1')
    elif ps_inv.type == inventory.CEPH_PRIMARY_STORAGE_TYPE:
        ps_name = os.environ.get('cephPrimaryStorageName1')

    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name=ps_name)[0]
    ps_ops.delete_primary_storage(ps_inv.uuid)

    test_util.test_dsc("Recover Primary Storage")
    test_stub.recover_ps(ps_inv)
    test_util.test_dsc("Recover Host")
    host_ops.add_kvm_host(host_config)

    host1 = res_ops.get_resource(res_ops.HOST, name=host1_name)[0]
    ps1 = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name=ps_name)[0]

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Delete Host and Primary Storage Test Success')
def test():
    global host_config
    global ps_inv
    curr_deploy_conf = exp_ops.export_zstack_deployment_config(test_lib.deploy_config)

    host1 = res_ops.get_resource(res_ops.HOST, name = host1_name)[0]

    host_config.set_name(host1_name)
    host_config.set_cluster_uuid(host1.clusterUuid)
    host_config.set_management_ip(host1.managementIp)
    host_config.set_username(os.environ.get('hostUsername'))
    host_config.set_password(os.environ.get('hostPassword'))

    test_util.test_dsc('delete host')
    host_ops.delete_host(host1.uuid)

    test_util.test_dsc('delete primary storage')
    zone_name = os.environ.get('zoneName1')
    zone_uuid = res_ops.get_resource(res_ops.ZONE, name = zone_name)[0].uuid
    cond = res_ops.gen_query_conditions('zoneUuid', '=', zone_uuid)
    ps_inv = res_ops.query_resource(res_ops.PRIMARY_STORAGE, cond)[0]
    if ps_inv.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
        ps_name = os.environ.get('nfsPrimaryStorageName1')
    elif ps_inv.type == inventory.CEPH_PRIMARY_STORAGE_TYPE:
        ps_name = os.environ.get('cephPrimaryStorageName1')

    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name)[0]
    ps_ops.delete_primary_storage(ps_inv.uuid)

    test_util.test_dsc("Recover Primary Storage")
    test_stub.recover_ps(ps_inv)
    test_util.test_dsc("Recover Host")
    host_ops.add_kvm_host(host_config)

    host1 = res_ops.get_resource(res_ops.HOST, name = host1_name)[0]
    ps1 = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name)[0]

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Delete Host and Primary Storage Test Success')
def test():
    global ps_inv
    global tag
    curr_deploy_conf = exp_ops.export_zstack_deployment_config(test_lib.deploy_config)

    vm_creation_option = test_util.VmOption()
    image_name = os.environ.get('imageName_net')
    image_uuid = test_lib.lib_get_image_by_name(image_name).uuid
    zone1_name = os.environ.get('zoneName1')
    zone1 = res_ops.get_resource(res_ops.ZONE, name = zone1_name)[0]
    #pick up primary storage 1 and set system tag for instance offering.
    zone_name = os.environ.get('zoneName1')
    zone_uuid = res_ops.get_resource(res_ops.ZONE, name = zone_name)[0].uuid
    cond = res_ops.gen_query_conditions('zoneUuid', '=', zone_uuid)
    ps_inv = res_ops.query_resource(res_ops.PRIMARY_STORAGE, cond)[0]
    if ps_inv.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
        ps_name1 = os.environ.get('nfsPrimaryStorageName1')
    elif ps_inv.type == inventory.CEPH_PRIMARY_STORAGE_TYPE:
        ps_name1 = os.environ.get('cephPrimaryStorageName1')

    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name1)[0]

    conditions = res_ops.gen_query_conditions('type', '=', 'UserVm')
    instance_offering_uuid = res_ops.query_resource(res_ops.INSTANCE_OFFERING, conditions)[0].uuid
    vm_creation_option.set_image_uuid(image_uuid)
    vm_creation_option.set_instance_offering_uuid(instance_offering_uuid)
    vm_creation_option.set_name('multizones_vm_detach_ps')
    vm_creation_option.set_zone_uuid(zone1.uuid)

    tag = tag_ops.create_system_tag('InstanceOfferingVO', \
            instance_offering_uuid, \
            'primaryStorage::allocator::uuid::%s' % ps_inv.uuid)

    l3_name = os.environ.get('l3VlanNetworkName1')
    l3 = res_ops.get_resource(res_ops.L3_NETWORK, name = l3_name)[0]
    vm_creation_option.set_l3_uuids([l3.uuid])

    vm1 = test_lib.lib_create_vm(vm_creation_option)
    test_obj_dict.add_vm(vm1)

    volume1 = test_stub.create_volume()
    test_obj_dict.add_volume(volume1)
    volume1.attach(vm1)

    test_util.test_dsc("Delete Primary Storage")
    #need to delete tag as well
    tag_ops.delete_tag(tag.uuid)
    ps_ops.delete_primary_storage(ps_inv.uuid)

    test_obj_dict.mv_vm(vm1, vm_header.RUNNING, vm_header.DESTROYED)
    vm1.update()

    test_obj_dict.rm_volume(volume1)
    volume1.update()

    test_lib.lib_robot_status_check(test_obj_dict)

    test_util.test_dsc("Recover Primary Storage")
    test_stub.recover_ps(ps_inv)
    test_lib.lib_robot_status_check(test_obj_dict)

    #update tag
    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name1)[0]

    tag = tag_ops.create_system_tag('InstanceOfferingVO', \
            instance_offering_uuid, \
            'primaryStorage::allocator::uuid::%s' % ps_inv.uuid)

    test_util.test_dsc("Create new VM and Volume")
    vm2 = test_lib.lib_create_vm(vm_creation_option)
    test_obj_dict.add_vm(vm2)

    volume2 = test_stub.create_volume()
    test_obj_dict.add_volume(volume2)
    volume2.attach(vm2)
    vm2.check()
    volume2.check()

    tag_ops.delete_tag(tag.uuid)
    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Test deleting primary storage Success')
def test():
    global ps_inv
    global ps_uuid
    global cluster_uuid
    global tag
    curr_deploy_conf = exp_ops.export_zstack_deployment_config(test_lib.deploy_config)

    vm_creation_option = test_util.VmOption()
    image_name = os.environ.get('imageName_net')
    image_uuid = test_lib.lib_get_image_by_name(image_name).uuid
    #pick up primary storage 1 and set system tag for instance offering.
    zone_name = os.environ.get('zoneName1')
    zone_uuid = res_ops.get_resource(res_ops.ZONE, name = zone_name)[0].uuid
    cond = res_ops.gen_query_conditions('zoneUuid', '=', zone_uuid)
    ps_inv = res_ops.query_resource(res_ops.PRIMARY_STORAGE, cond)[0]
    if ps_inv.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
        ps_name1 = os.environ.get('nfsPrimaryStorageName1')
    elif ps_inv.type == inventory.CEPH_PRIMARY_STORAGE_TYPE:
        ps_name1 = os.environ.get('cephPrimaryStorageName1')

    ps_inv = res_ops.get_resource(res_ops.PRIMARY_STORAGE, name = ps_name1)[0]
    ps_uuid = ps_inv.uuid

    conditions = res_ops.gen_query_conditions('type', '=', 'UserVm')
    instance_offering_uuid = res_ops.query_resource(res_ops.INSTANCE_OFFERING, \
            conditions)[0].uuid
    vm_creation_option.set_image_uuid(image_uuid)
    vm_creation_option.set_instance_offering_uuid(instance_offering_uuid)
    vm_creation_option.set_name('multizones_vm_ps_ops')

    tag = tag_ops.create_system_tag('InstanceOfferingVO', \
            instance_offering_uuid, \
            'primaryStorage::allocator::uuid::%s' % ps_uuid)

    l3_name = os.environ.get('l3VlanNetworkName1')
    l3 = res_ops.get_resource(res_ops.L3_NETWORK, name = l3_name)[0]
    vm_creation_option.set_l3_uuids([l3.uuid])

    vm1 = test_lib.lib_create_vm(vm_creation_option)
    test_obj_dict.add_vm(vm1)

    cluster_uuid = vm1.get_vm().clusterUuid

    test_util.test_dsc("Detach Primary Storage")
    ps_ops.detach_primary_storage(ps_uuid, cluster_uuid)

    test_obj_dict.mv_vm(vm1, vm_header.RUNNING, vm_header.STOPPED)
    vm1.update()
    vm1.set_state(vm_header.STOPPED)

    vm1.check()

    vm1.start()

    vm2 = test_lib.lib_create_vm(vm_creation_option)
    test_obj_dict.add_vm(vm2)
    
    test_util.test_dsc("Delete Primary Storage")
    tag_ops.delete_tag(tag.uuid)
    ps_ops.delete_primary_storage(ps_inv.uuid)

    test_obj_dict.mv_vm(vm1, vm_header.RUNNING, vm_header.DESTROYED)
    vm1.set_state(vm_header.DESTROYED)
    vm1.check()

    test_obj_dict.mv_vm(vm2, vm_header.RUNNING, vm_header.DESTROYED)
    vm2.set_state(vm_header.DESTROYED)
    vm2.check()

    try:
        vm3 = test_lib.lib_create_vm(vm_creation_option)
    except:
        test_util.test_logger('Catch expected vm creation exception, since primary storage has been deleted. ')
    else:
        test_util.test_fail('Fail: Primary Storage has been deleted. But vm is still created with it.')

    test_stub.recover_ps(ps_inv)
    test_util.test_dsc("Attach Primary Storage")

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Test primary storage operations Success')
Ejemplo n.º 14
0
def test():
    ps_env = test_stub.PSEnvChecker()

    ps1, ps2 = ps_env.get_two_ps()
    if ps2.type == 'SharedBlock':
        volumegroup_uuid = ps2.sharedBlocks[0].sharedBlockGroupUuid
        disk_uuid.append(ps2.sharedBlocks[0].diskUuid)

    for _ in xrange(5):
        test_util.test_dsc('Remove ps2')
        ps_ops.detach_primary_storage(
            ps2.uuid,
            res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
        ps_ops.delete_primary_storage(ps2.uuid)
        delete_ps_list.append(ps2)
        test_util.test_dsc('Add ps2 back')
        ps_config = test_util.PrimaryStorageOption()
        ps_config.set_name(ps2.name)
        ps_config.set_description(ps2.description)
        ps_config.set_zone_uuid(ps2.zoneUuid)
        ps_config.set_type(ps2.type)
        ps_config.set_url(ps2.url)
        if ps2.type == inventory.LOCAL_STORAGE_TYPE:
            ps2 = ps_ops.create_local_primary_storage(ps_config)
        elif ps2.type == inventory.NFS_PRIMARY_STORAGE_TYPE:
            ps2 = ps_ops.create_nfs_primary_storage(ps_config)
        elif ps2.type == "SharedBlock":
            host = random.choice(res_ops.query_resource(res_ops.HOST))
            cmd = "vgchange --lock-start %s && vgremove %s -y" % (
                volumegroup_uuid, volumegroup_uuid)
            host_username = os.environ.get('hostUsername')
            host_password = os.environ.get('hostPassword')
            rsp = test_lib.lib_execute_ssh_cmd(host.managementIp,
                                               host_username, host_password,
                                               cmd, 240)
            if not rsp:
                cmd = "vgs"
                rsp = test_lib.lib_execute_ssh_cmd(host.managementIp,
                                                   host_username,
                                                   host_password, cmd, 240)
                test_util.test_logger(rsp)
                test_util.test_fail("vgremove failed")
            ps2 = ps_ops.create_sharedblock_primary_storage(
                ps_config, disk_uuid)
            volumegroup_uuid = ps2.uuid
        else:
            ps2 = None
        time.sleep(5)
        ps_ops.attach_primary_storage(
            ps2.uuid,
            res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
        time.sleep(5)
        delete_ps_list.pop()
    test_util.test_dsc('create VM by default para')
    vm1 = test_stub.create_multi_vms(name_prefix='vm1',
                                     count=1,
                                     data_volume_number=VOLUME_NUMBER,
                                     timeout=1200000)[0]
    test_obj_dict.add_vm(vm1)

    if ps_env.is_local_nfs_env:
        test_util.test_dsc('create date volume in ps2')
        volume = test_stub.create_multi_volumes(count=VOLUME_NUMBER, ps=ps2)
        test_obj_dict.add_volume(volume)
    else:
        test_util.test_dsc('create VM in ps2')
        vm2 = test_stub.create_multi_vms(name_prefix='vm2',
                                         count=1,
                                         ps_uuid=ps2.uuid,
                                         data_volume_number=VOLUME_NUMBER,
                                         timeout=1200000)[0]
        test_obj_dict.add_vm(vm2)

    test_util.test_pass('Multi PrimaryStorage Test Pass')
Ejemplo n.º 15
0
def test():
    ps_env = test_stub.PSEnvChecker()

    ps1, ps2 = ps_env.get_two_ps()
    if ps_env.is_sb_ceph_env:
        disk_uuid.append(ps1.sharedBlocks[0].diskUuid)
    else:
        if ps2.type == 'SharedBlock':
            disk_uuid.append(ps2.sharedBlocks[0].diskUuid)

    vm_list = []
    for root_vol_ps in [ps1, ps2]:
        for data_vol_ps in [ps1, ps2]:
            if ps_env.is_sb_ceph_env:
                vm = test_stub.create_multi_vms(name_prefix='test_vm', count=1,
                                                ps_uuid=root_vol_ps.uuid, data_volume_number=VOLUME_NUMBER,
                                                ps_uuid_for_data_vol=data_vol_ps.uuid, timeout=1200000,
                                                bs_type="ImageStoreBackupStorage" if root_vol_ps.type == "SharedBlock" else "Ceph")[0]
            else:
                vm = test_stub.create_multi_vms(name_prefix='test_vm', count=1,
                                                ps_uuid=root_vol_ps.uuid, data_volume_number=VOLUME_NUMBER,
                                                ps_uuid_for_data_vol=data_vol_ps.uuid, timeout=1200000)[0]
            test_obj_dict.add_vm(vm)
            vm_list.append(vm)

    vm1, vm2, vm3, vm4 = vm_list

    if ps_env.is_sb_ceph_env:
        ps_ops.detach_primary_storage(ps1.uuid, res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
        delete_ps_list.append(ps1)
    else:
        ps_ops.detach_primary_storage(ps2.uuid, res_ops.get_resource(res_ops.CLUSTER)[0].uuid)
        delete_ps_list.append(ps2)
    time.sleep(30)
    for vm in vm_list:
        vm.update()

    if ps_env.is_sb_ceph_env:
        assert vm1.get_vm().state == inventory.STOPPED
        assert vm2.get_vm().state == inventory.STOPPED
        assert vm3.get_vm().state == inventory.STOPPED
        assert vm4.get_vm().state == inventory.RUNNING
        ps_ops.delete_primary_storage(ps1.uuid)
    else:
        assert vm1.get_vm().state == inventory.RUNNING
        assert vm2.get_vm().state == inventory.STOPPED
        assert vm3.get_vm().state == inventory.STOPPED
        assert vm4.get_vm().state == inventory.STOPPED
        ps_ops.delete_primary_storage(ps2.uuid)

    
    time.sleep(10)
    conf = res_ops.gen_query_conditions('type', '=', 'UserVM')
    left_vm_list = res_ops.query_resource(res_ops.VM_INSTANCE, conf)
    assert len(left_vm_list) == 2
    left_vm_uuid_list = [vm.uuid for vm in left_vm_list]
    if ps_env.is_sb_ceph_env:
        assert vm3.get_vm().uuid in left_vm_uuid_list
        assert vm4.get_vm().uuid in left_vm_uuid_list
    else:
        assert vm1.get_vm().uuid in left_vm_uuid_list
        assert vm2.get_vm().uuid in left_vm_uuid_list

    assert len(res_ops.query_resource(res_ops.VOLUME)) == \
           VOLUME_NUMBER * 2 + len(res_ops.query_resource(res_ops.VM_INSTANCE))

    test_util.test_pass('Multi PrimaryStorage Test Pass')
def test():
    global email_endpoint_uuid,email_platform_uuid,my_sns_topic_uuid,host_sns_topic_uuid,ps_uuid,hostname,host_management_ip,host_uuid,host_status,bs_uuid,bs_type,bs_status
    bs_cond = res_ops.gen_query_conditions('status', '=', 'Connected')
    bs_list = res_ops.query_resource(res_ops.BACKUP_STORAGE, bs_cond)
    for bss in bs_list:
        if bss.type ==res_ops.SFTP_BACKUP_STORAGE:
            bs_uuid =bss.uuid
            bs_type =bss.type
            hostname = bss.hostname
            break
        elif bss.type == res_ops.IMAGE_STORE_BACKUP_STORAGE:
            bs_uuid=bss.uuid
            bs_type=bss.type
            hostname = bss.hostname
            break
        else:
            test_util.test_skip('No match backupStorage,test skip')

    smtp_server = os.environ.get('smtpServer')
    pop_server = os.environ.get('popServer')
    smtp_port = os.environ.get('smtpPort')
    username = os.environ.get('mailUsername')
    password = os.environ.get('mailPassword')
    email_platform_name='Alarm_email'
    email_platform = zwt_ops.create_sns_email_platform(smtp_server, smtp_port,email_platform_name , username, password)
    email_platform_uuid = email_platform.uuid
    try:
        zwt_ops.validate_sns_email_platform(email_platform_uuid)
    except:
        test_util.test_fail('Validate SNS Email Platform Failed, Email Plarform: %s' % email_platform_uuid)
    email_endpoint_uuid = zwt_ops.create_sns_email_endpoint(username, 'test_qa', email_platform_uuid).uuid

    my_sns_topic = zwt_ops.create_sns_topic('my_sns_topic')
    my_sns_topic_uuid = my_sns_topic.uuid
    zwt_ops.subscribe_sns_topic(my_sns_topic_uuid, email_endpoint_uuid)

    host_sns_topic = zwt_ops.create_sns_topic('host_topic')
    host_sns_topic_uuid = host_sns_topic.uuid
    zwt_ops.subscribe_sns_topic(host_sns_topic_uuid, email_endpoint_uuid)

    ps_actions = [{"actionUuid": my_sns_topic_uuid, "actionType": "sns"}]
    ps_namespace = 'ZStack/PrimaryStorage'
    ps_disconnected = 'PrimaryStorageDisconnected'
    ps_event_sub_uuid = zwt_ops.subscribe_event(ps_namespace, ps_disconnected, ps_actions).uuid
    event_list.append(ps_event_sub_uuid)

    bs_actions = [{"actionUuid": my_sns_topic_uuid, "actionType": "sns"}]
    bs_namespace = 'ZStack/BackupStorage'
    bs_disconnected = 'BackupStorageDisconnected'
    bs_event_sub_uuid = zwt_ops.subscribe_event(bs_namespace, bs_disconnected, bs_actions).uuid
    event_list.append(bs_event_sub_uuid)

    host_actions = [{"actionUuid": host_sns_topic_uuid, "actionType": "sns"}]
    host_namespace = 'ZStack/Host'
    host_status_changed = 'HostStatusChanged'
    host_status_labels = [{"key": "NewStatus", "op": "Equal", "value": "Disconnected"}]
    host_status_event_sub_uuid = zwt_ops.subscribe_event(host_namespace, host_status_changed, host_actions, host_status_labels).uuid
    event_list.append(host_status_event_sub_uuid)

    host_disconnected = 'HostDisconnected'
    host_disconnected_event_sub_uuid = zwt_ops.subscribe_event(host_namespace, host_disconnected, host_actions).uuid
    event_list.append(host_disconnected_event_sub_uuid)

    if zwt_ops.check_sns_email(pop_server, username, password, ps_disconnected, ps_event_sub_uuid):
        test_util.test_fail('email already exsist before test')
    if zwt_ops.check_sns_email(pop_server, username, password, bs_disconnected, bs_event_sub_uuid):
        test_util.test_fail('email already exsist before test')
    if zwt_ops.check_sns_email(pop_server, username, password, host_status_changed, host_status_event_sub_uuid):
        test_util.test_fail('email already exsist before test')
    if zwt_ops.check_sns_email(pop_server, username, password, host_disconnected, host_disconnected_event_sub_uuid):
        test_util.test_fail('email already exsist before test')

    # Disconnected ps ,bs and host
    zone_uuid = res_ops.query_resource(res_ops.ZONE)[0].uuid
    primary_storage_option = test_util.PrimaryStorageOption()
    primary_storage_option.set_type('nfs')
    primary_storage_option.set_zone_uuid(zone_uuid)
    primary_storage_option.set_name('test_nfs_ps')
    primary_storage_option.set_url('222.222.222.222/nfs/')
    try:
        ps_uuid = ps_ops.create_nfs_primary_storage(primary_storage_option).uuid
        ps_ops.reconnect_primary_storage(ps_uuid)
    except:
        pass

    if bs_type == res_ops.IMAGE_STORE_BACKUP_STORAGE:
        bs_ops.update_image_store_backup_storage_info(bs_uuid, 'hostname', '222.222.222.222')
        try:
            bs_ops.reconnect_backup_storage(bs_uuid)
        except:
            cond = res_ops.gen_query_conditions('uuid', '=', bs_uuid)
            bs_status = res_ops.query_resource(res_ops.IMAGE_STORE_BACKUP_STORAGE, cond)[0].status
    elif bs_type == res_ops.SFTP_BACKUP_STORAGE:
        bs_ops.update_sftp_backup_storage_info(bs_uuid,'hostname','222.222.222.222')
        try:
            bs_ops.reconnect_backup_storage(bs_uuid)
        except:
            cond=res_ops.gen_query_conditions('uuid','=',bs_uuid)
            bs_status=res_ops.query_resource(res_ops.SFTP_BACKUP_STORAGE,cond)[0].status

    host_cond = res_ops.gen_query_conditions('status', '=', 'Connected')
    host = res_ops.query_resource_with_num(res_ops.HOST, host_cond, start=0, limit=1)[0]
    host_uuid = host.uuid
    host_management_ip = host.managementIp
    host_ops.update_host(host_uuid, 'managementIp', '222.222.222.222')
    try:
        host_ops.reconnect_host(host_uuid)
    except:
        cond = res_ops.gen_query_conditions('uuid', '=', host_uuid)
        bs_status = res_ops.query_resource(res_ops.HOST, cond)[0].status

    # wait for send email
    time.sleep(120)

    ps_ops.delete_primary_storage(ps_uuid)
    if hostname:
        if bs_type == res_ops.IMAGE_STORE_BACKUP_STORAGE:
            bs_ops.update_image_store_backup_storage_info(bs_uuid, 'hostname', hostname)
            bs_ops.reconnect_backup_storage(bs_uuid)
        elif bs_type == res_ops.SFTP_BACKUP_STORAGE:
            bs_ops.update_sftp_backup_storage_info(bs_uuid, 'hostname', hostname)
            bs_ops.reconnect_backup_storage(bs_uuid)
    host_ops.update_host(host_uuid, 'managementIp', host_management_ip)
    host_ops.reconnect_host(host_uuid)
    zwt_ops.delete_sns_topic(host_sns_topic_uuid)
    zwt_ops.delete_sns_topic(my_sns_topic_uuid)
    for event_uuid in event_list:
        zwt_ops.unsubscribe_event(event_uuid)
    zwt_ops.delete_sns_application_endpoint(email_endpoint_uuid)
    zwt_ops.delete_sns_application_platform(email_platform_uuid)

    check_1 = zwt_ops.check_sns_email(pop_server, username, password, ps_disconnected, ps_event_sub_uuid)
    check_2 = zwt_ops.check_sns_email(pop_server, username, password, bs_disconnected, bs_event_sub_uuid)
    check_3 = zwt_ops.check_sns_email(pop_server, username, password, host_status_changed, host_status_event_sub_uuid)
    check_4 = zwt_ops.check_sns_email(pop_server, username, password, host_disconnected, host_disconnected_event_sub_uuid)

    if check_1 and check_2 and check_3 and check_4:
        test_util.test_pass('test host ,bs, ps disconnected event with email success!')
    else:
        test_util.test_fail('cannt receive all event mail')