def check(self):
        '''
        will check if backuped snapshot existed.
        '''
        super(zstack_kvm_backuped_snapshot_checker, self).check()
        backuped_snapshots = self.test_obj.get_backuped_snapshots()
        for sp in backuped_snapshots:
            sp_bs_refs = sp.get_snapshot().backupStorageRefs
            for sp_bs_ref in sp_bs_refs:
                bs_uuid = sp_bs_ref.backupStorageUuid
                sp_path = sp_bs_ref.installPath
                sp_uuid = sp_bs_ref.volumeSnapshotUuid
                bs_host = test_lib.lib_get_backup_storage_host(bs_uuid)
                if test_lib.lib_check_file_exist(bs_host, sp_path):
                    test_util.test_logger(
                        'Checker result: backuped snapshot:%s is found in backup storage:%s in path: %s'
                        % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == False:
                        return self.judge(True)
                else:
                    test_util.test_logger(
                        'Checker result: backuped snapshot:%s is NOT found in backup storage:%s in path: %s'
                        % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == True:
                        return self.judge(False)

        test_util.test_logger(
            'Checker result: Finish backuped snapshot checking')
        return self.judge(self.exp_result)
    def check(self):
        super(zstack_kvm_image_file_checker, self).check()
        image = self.test_obj.image
        backupStorages = image.backupStorageRefs
        bs_one = backupStorages[0]
        bs = test_lib.lib_get_backup_storage_by_uuid(bs_one.backupStorageUuid)
        if bs.type == inventory.SFTP_BACKUP_STORAGE_TYPE:
            host = test_lib.lib_get_backup_storage_host(bs_one.backupStorageUuid)
            image_url = backupStorages[0].installPath
            self.judge(test_lib.lib_check_file_exist(host, image_url))
        elif bs.type == inventory.CEPH_BACKUP_STORAGE_TYPE:
            if self.test_obj.state == image_header.DELETED:
                #https://github.com/zstackorg/zstack/issues/93#issuecomment-130935998
                test_util.test_logger("skip ceph image delete check, since the image won't be deleted until no vms refer to it.")
                return self.judge(self.exp_result)

            ceph_host, username, password = test_lib.lib_get_ceph_info(os.environ.get('cephBackupStorageMonUrls'))
            image_installPath = bs_one.installPath.split('ceph://')[1]

            command = 'rbd info %s' % image_installPath
            if test_lib.lib_execute_ssh_cmd(ceph_host, username, password, command, 10):
                test_util.test_logger('Check result: [image:] %s [file:] %s exist on ceph [host name:] %s .' % (image.uuid, image_installPath, ceph_host))
                return self.judge(True)
            else:
                test_util.test_logger('Check result: [image:] %s [file:] %s does not exist on ceph [host name:] %s .' % (image.uuid, image_installPath, ceph_host))
                return self.judge(False)
    def check(self):
        '''
        will check if backuped snapshot existed.
        '''
        super(zstack_kvm_backuped_snapshot_checker, self).check()
        backuped_snapshots = self.test_obj.get_backuped_snapshots()
        for sp in backuped_snapshots:
            sp_bs_refs = sp.get_snapshot().backupStorageRefs
            for sp_bs_ref in sp_bs_refs:
                bs_uuid = sp_bs_ref.backupStorageUuid
                sp_path = sp_bs_ref.installPath
                sp_uuid = sp_bs_ref.volumeSnapshotUuid
                bs_host = test_lib.lib_get_backup_storage_host(bs_uuid)
                bs = test_lib.lib_get_backup_storage_by_uuid(sp_bs_ref.backupStorageUuid)
                if hasattr(inventory, 'IMAGE_STORE_BACKUP_STORAGE_TYPE') and bs.type == inventory.IMAGE_STORE_BACKUP_STORAGE_TYPE:
                    sp_info = sp_path.split('://')[1].split('/')
                    sp_path = '%s/registry/v1/repos/public/%s/manifests/revisions/%s' % (bs.url, sp_info[0], sp_info[1])
                if test_lib.lib_check_file_exist(bs_host, sp_path):
                    test_util.test_logger('Checker result: backuped snapshot:%s is found in backup storage:%s in path: %s' % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == False:
                        return self.judge(True)
                else:
                    test_util.test_logger('Checker result: backuped snapshot:%s is NOT found in backup storage:%s in path: %s' % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == True:
                        return self.judge(False)

        test_util.test_logger('Checker result: Finish backuped snapshot checking')
        return self.judge(self.exp_result)
예제 #4
0
 def check(self):
     super(zstack_kvm_image_file_checker, self).check()
     image = self.test_obj.image
     backupStorages = image.backupStorageRefs
     image_url = backupStorages[0].installPath
     host = test_lib.lib_get_backup_storage_host(image.backupStorageUuid)
     self.judge(test_lib.lib_check_file_exist(host, image_url))
def test():
    test_util.test_dsc('create image check timeout test')
    if res_ops.query_resource(res_ops.SFTP_BACKUP_STORAGE):
        bs = res_ops.query_resource(res_ops.SFTP_BACKUP_STORAGE)[0]
    else:
        test_util.test_skip("No sftp backupstorage for test. Skip test")
    cond = res_ops.gen_query_conditions('backupStorageRef.backupStorage.uuid',
                                        '=', bs.uuid)
    images = res_ops.query_resource(res_ops.IMAGE, cond)

    #create backup storage on previous vm
    backup_storage_option = test_util.SftpBackupStorageOption()
    backup_storage_option.name = bs.name
    backup_storage_option.username = bs.username
    backup_storage_option.hostname = bs.hostname
    backup_storage_option.password = test_lib.lib_get_backup_storage_host(
        bs.uuid).password
    backup_storage_option.sshPort = bs.sshPort
    backup_storage_option.importImages = "true"
    backup_storage_option.url = bs.url

    bs_ops.delete_backup_storage(bs.uuid)
    backup_storage_inventory = bs_ops.create_backup_storage(
        backup_storage_option)
    time.sleep(5)
    cond = res_ops.gen_query_conditions('backupStorageRef.backupStorage.uuid',
                                        '=', backup_storage_inventory.uuid)
    images_imported = res_ops.query_resource(res_ops.IMAGE, cond)
    if len(images_imported) != len(images):
        test_util.test_fail(
            "imported images has different number(%s) than original images(%s)"
            % (len(images_imported), len(images)))
    for image in images:
        image_match = False
        for ii in images_imported:
            if compare_image(image, ii):
                image_match = True
        if not image_match:
            test_util.test_fail("images(%s) not imported correctly" %
                                (image.uuid))

    test_lib.lib_error_cleanup(test_obj_dict)
    def check(self):
        '''
        will check if backuped snapshot existed.
        '''
        super(zstack_vcenter_backuped_snapshot_checker, self).check()
        backuped_snapshots = self.test_obj.get_backuped_snapshots()
        for sp in backuped_snapshots:
            sp_bs_refs = sp.get_snapshot().backupStorageRefs
            for sp_bs_ref in sp_bs_refs:
                bs_uuid = sp_bs_ref.backupStorageUuid
                sp_path = sp_bs_ref.installPath
                sp_uuid = sp_bs_ref.volumeSnapshotUuid
                bs_host = test_lib.lib_get_backup_storage_host(bs_uuid)
                bs = test_lib.lib_get_backup_storage_by_uuid(
                    sp_bs_ref.backupStorageUuid)
                if hasattr(
                        inventory, 'IMAGE_STORE_BACKUP_STORAGE_TYPE'
                ) and bs.type == inventory.IMAGE_STORE_BACKUP_STORAGE_TYPE:
                    sp_info = sp_path.split('://')[1].split('/')
                    sp_path = '%s/registry/v1/repos/public/%s/manifests/revisions/%s' % (
                        bs.url, sp_info[0], sp_info[1])
                if test_lib.lib_check_file_exist(bs_host, sp_path):
                    test_util.test_logger(
                        'Checker result: backuped snapshot:%s is found in backup storage:%s in path: %s'
                        % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == False:
                        return self.judge(True)
                else:
                    test_util.test_logger(
                        'Checker result: backuped snapshot:%s is NOT found in backup storage:%s in path: %s'
                        % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == True:
                        return self.judge(False)

        test_util.test_logger(
            'Checker result: Finish backuped snapshot checking')
        return self.judge(self.exp_result)
    def check(self):
        '''
        will check if backuped snapshot existed.
        '''
        super(zstack_kvm_backuped_snapshot_checker, self).check()
        backuped_snapshots = self.test_obj.get_backuped_snapshots()
        for sp in backuped_snapshots:
            sp_bs_refs = sp.get_snapshot().backupStorageRefs
            for sp_bs_ref in sp_bs_refs:
                bs_uuid = sp_bs_ref.backupStorageUuid
                sp_path = sp_bs_ref.installPath
                sp_uuid = sp_bs_ref.volumeSnapshotUuid
                bs_host = test_lib.lib_get_backup_storage_host(bs_uuid)
                if test_lib.lib_check_file_exist(bs_host, sp_path):
                    test_util.test_logger('Checker result: backuped snapshot:%s is found in backup storage:%s in path: %s' % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == False:
                        return self.judge(True)
                else:
                    test_util.test_logger('Checker result: backuped snapshot:%s is NOT found in backup storage:%s in path: %s' % (sp_uuid, bs_uuid, sp_path))
                    if self.exp_result == True:
                        return self.judge(False)

        test_util.test_logger('Checker result: Finish backuped snapshot checking')
        return self.judge(self.exp_result)
def test():
    test_util.test_dsc('create image check timeout test')
    if res_ops.query_resource(res_ops.SFTP_BACKUP_STORAGE):
        bs = res_ops.query_resource(res_ops.SFTP_BACKUP_STORAGE)[0]
    else:
        test_util.test_skip("No sftp backupstorage for test. Skip test")
    cond = res_ops.gen_query_conditions('backupStorageRef.backupStorage.uuid', '=', bs.uuid)
    images =  res_ops.query_resource(res_ops.IMAGE, cond)

    #create backup storage on previous vm
    backup_storage_option = test_util.SftpBackupStorageOption()
    backup_storage_option.name = bs.name
    backup_storage_option.username = bs.username
    backup_storage_option.hostname = bs.hostname
    backup_storage_option.password = test_lib.lib_get_backup_storage_host(bs.uuid).password
    backup_storage_option.sshPort = bs.sshPort
    backup_storage_option.importImages = "true"
    backup_storage_option.url = bs.url

    bs_ops.delete_backup_storage(bs.uuid)
    backup_storage_inventory = bs_ops.create_backup_storage(backup_storage_option)
    time.sleep(5)
    cond = res_ops.gen_query_conditions('backupStorageRef.backupStorage.uuid', '=', backup_storage_inventory.uuid)
    images_imported =  res_ops.query_resource(res_ops.IMAGE, cond)
    if len(images_imported) != len(images):
        test_util.test_fail("imported images has different number(%s) than original images(%s)" % (len(images_imported), len(images)))
    for image in images:
        image_match = False
        for ii in images_imported:
	    if compare_image(image, ii):
                image_match = True
        if not image_match:
            test_util.test_fail("images(%s) not imported correctly" % (image.uuid))


    test_lib.lib_error_cleanup(test_obj_dict)
def test():
    #str = str(os.listdir('/home'))
    #if 'scenario' in str:
    #    test_util.test_skip('Not available environment to run this case')

    path = "/home"
    pathlist = os.listdir(path)
    for path in pathlist:
        if re.findall(r'\d+.\d+.\d+.\d+', path):
            target_path = path

    if os.path.exists('/home/' + target_path + '/scenario-file.xml'):
        test_util.test_skip('Not available environment to run this case')

    global MEVOCO_LOG_FOLDER, MEVOCO_LOG_PATH, MEVOCO_LOG_FOLDER_PATTERN
    #if not test_lib.lib_check_version_is_mevoco():
    #    MEVOCO_LOG_FOLDER = r"collect-log-zstack*"
    #	MEVOCO_LOG_FOLDER_PATTERN = "collect-log-zstack"

    #Step1: clean env below TEMPT_FOLDER and run collect log
    retVal = os.system(" cd " + TEMPT_FOLDER  + "; rm -rf " + MEVOCO_LOG_FOLDER + "; zstack-ctl collect_log")

    if retVal != 0:
        test_util.test_logger("os.system return value: %d" % (retVal))
        test_util.test_fail('run zstack-ctl collect_log failed.')


    #Step2: refresh MEVOCO_LOG_FOLDER value to real one below TEMPT_FOLDER
    find_mevoco_log_folder_name()


    #Step3: verify sftpbackupstorage logs are saved
    bs_cond = res_ops.gen_query_conditions("status", '=', "Connected")
    bss = res_ops.query_resource_fields(res_ops.BACKUP_STORAGE, bs_cond, \
            None, fields=['uuid', 'type'])
    if not bss:
        test_util.test_skip("not find available backup storage. Skip test")


    for bs in bss:
        #test_util.test_logger(bs.dump())
        if bs.type == "SftpBackupStorage":
            #bs_sftp_cond = res_ops.gen_query_conditions("type", "=", "SftpBackupStorage")
            #hostIP = res_ops.query_resource_fields(res_ops.BACKUP_STORAGE, bs_sftp_cond, None, fields=["hostname"])
            hostIP = test_lib.lib_get_backup_storage_host(bs.uuid) 

            sftpLogFolderName = "sftp_bs-" + hostIP.managementIp 
            print "logPath=%s" %(MEVOCO_LOG_PATH)
            print "sftpLogFolderName=%s" %(sftpLogFolderName)
            dmesgFilePath             = MEVOCO_LOG_PATH + sftpLogFolderName + "/dmesg"
            hostInfoFilePath          = MEVOCO_LOG_PATH + sftpLogFolderName + "/host_info"
            messagesFilePath          = MEVOCO_LOG_PATH + sftpLogFolderName + "/messages"
            sftpbackupstorageFilePath = MEVOCO_LOG_PATH + sftpLogFolderName + "/zstack-sftpbackupstorage.log"

            if not os.path.exists(dmesgFilePath):
                test_util.test_fail( dmesgFilePath             + ' is not exist.')
            if not os.path.exists(hostInfoFilePath):
                test_util.test_fail( hostInfoFilePath          + ' is not exist.')
            if not os.path.exists(messagesFilePath):
                test_util.test_fail( messagesFilePath          + ' is not exist.')
            if not os.path.exists(sftpbackupstorageFilePath):
                test_util.test_fail( sftpbackupstorageFilePath + ' is not exist.')
        #elif bs.type == "ImageStoreBackupStorage":
        #    pass
        else:
            test_util.test_skip("backup storage is not sftp, skip")


    #Step4: verify hosts logs are saved
    #conditions = res_ops.gen_query_conditions('state', '=', host_header.ENABLED)
    #conditions = res_ops.gen_query_conditions('status', '=', host_header.CONNECTED, conditions)
    #all_hosts = res_ops.query_resource(res_ops.HOST, conditions)
    #if len(all_hosts) < 1:
    #    test_util.test_skip('Not available host to check')

    #for host in all_hosts:
    #    hostFolderName = host.managementIp
    #    dmesgFilePath    = MEVOCO_LOG_PATH + hostFolderName + "/dmesg"
    #    hostInfoFilePath = MEVOCO_LOG_PATH + hostFolderName + "/host_info"
    #    messagesFilePath = MEVOCO_LOG_PATH + hostFolderName + "/messages"
    #    kvmagentFilePath = MEVOCO_LOG_PATH + hostFolderName + "/zstack-kvmagent.log"
    #    zstackFilePath   = MEVOCO_LOG_PATH + hostFolderName + "/zstack.log"

    #    if not os.path.exists(dmesgFilePath):
    #        test_util.test_fail( dmesgFilePath    + ' is not exist.')
    #    if not os.path.exists(hostInfoFilePath):
    #        test_util.test_fail( hostInfoFilePath + ' is not exist.')
    #    if not os.path.exists(messagesFilePath):
    #        test_util.test_fail( messagesFilePath + ' is not exist.')
    #    if not os.path.exists(kvmagentFilePath):
    #        test_util.test_fail( kvmagentFilePath + ' is not exist.')
    #    if not os.path.exists(zstackFilePath):
    #        test_util.test_fail( zstackFilePath   + 'is not exist.')


    #Step5: verify management node logs are saved
    #conditions = res_ops.gen_query_conditions("status", '=', "Connected")
    all_mn = res_ops.query_resource(res_ops.MANAGEMENT_NODE)
    if len(all_mn) < 1:
        test_util.test_skip('Not available mn to check')

    for mn in all_mn:
        mnHostIP = mn.hostName
        mnFolderName = "management-node-" + mnHostIP
        dmesgFilePath    = MEVOCO_LOG_PATH + mnFolderName + "/dmesg"
        hostInfoFilePath = MEVOCO_LOG_PATH + mnFolderName + "/host_info"
        messagesFilePath = MEVOCO_LOG_PATH + mnFolderName + "/messages"
        manageServerPath = MEVOCO_LOG_PATH + mnFolderName + "/management-server.log"

        if not os.path.exists(dmesgFilePath):
            test_util.test_fail( dmesgFilePath    + ' is not exist.')
        if not os.path.exists(hostInfoFilePath):
            test_util.test_fail( hostInfoFilePath + ' is not exist.')
        if not os.path.exists(messagesFilePath):
            test_util.test_fail( messagesFilePath + ' is not exist.')
        if not os.path.exists(manageServerPath):
            test_util.test_fail( manageServerPath + ' is not exist.')
def test():
    global MEVOCO_LOG_FOLDER, MEVOCO_LOG_PATH
    if not test_lib.lib_check_version_is_mevoco():
        MEVOCO_LOG_FOLDER = r"collect-log-zstack_*"
        MEVOCO_LOG_FOLDER_PATTERN = "collect-log-zstack"

    #Step1: clean env below TEMPT_FOLDER and run collect log
    retVal = os.system(" cd " + TEMPT_FOLDER + "; rm -rf " +
                       MEVOCO_LOG_FOLDER + "; zstack-ctl collect_log")

    if retVal != 0:
        test_util.test_logger("os.system return value: %d" % (retVal))
        test_util.test_fail('run zstack-ctl collect_log failed.')

    #Step2: refresh MEVOCO_LOG_FOLDER value to real one below TEMPT_FOLDER
    find_mevoco_log_folder_name()

    #Step3: verify sftpbackupstorage logs are saved
    bs_cond = res_ops.gen_query_conditions("status", '=', "Connected")
    bss = res_ops.query_resource_fields(res_ops.BACKUP_STORAGE, bs_cond, \
            None, fields=['uuid', 'type'])
    if not bss:
        test_util.test_skip("not find available backup storage. Skip test")

    for bs in bss:
        #test_util.test_logger(bs.dump())
        if bs.type == "SftpBackupStorage":
            #bs_sftp_cond = res_ops.gen_query_conditions("type", "=", "SftpBackupStorage")
            #hostIP = res_ops.query_resource_fields(res_ops.BACKUP_STORAGE, bs_sftp_cond, None, fields=["hostname"])
            hostIP = test_lib.lib_get_backup_storage_host(bs.uuid)

            sftpLogFolderName = "sftp_bs-" + hostIP.managementIp
            dmesgFilePath = MEVOCO_LOG_PATH + sftpLogFolderName + "/dmesg"
            hostInfoFilePath = MEVOCO_LOG_PATH + sftpLogFolderName + "/host_info"
            messagesFilePath = MEVOCO_LOG_PATH + sftpLogFolderName + "/messages"
            sftpbackupstorageFilePath = MEVOCO_LOG_PATH + sftpLogFolderName + "/zstack-sftpbackupstorage.log"

            if not os.path.exists(dmesgFilePath):
                test_util.test_fail(dmesgFilePath + ' is not exist.')
            if not os.path.exists(hostInfoFilePath):
                test_util.test_fail(hostInfoFilePath + ' is not exist.')
            if not os.path.exists(messagesFilePath):
                test_util.test_fail(messagesFilePath + ' is not exist.')
            if not os.path.exists(sftpbackupstorageFilePath):
                test_util.test_fail(sftpbackupstorageFilePath +
                                    ' is not exist.')
        elif bs.type == "ImageStoreBackupStorage":
            pass

    #Step4: verify hosts logs are saved
    #conditions = res_ops.gen_query_conditions('state', '=', host_header.ENABLED)
    #conditions = res_ops.gen_query_conditions('status', '=', host_header.CONNECTED, conditions)
    #all_hosts = res_ops.query_resource(res_ops.HOST, conditions)
    #if len(all_hosts) < 1:
    #    test_util.test_skip('Not available host to check')

    #for host in all_hosts:
    #    hostFolderName = host.managementIp
    #    dmesgFilePath    = MEVOCO_LOG_PATH + hostFolderName + "/dmesg"
    #    hostInfoFilePath = MEVOCO_LOG_PATH + hostFolderName + "/host_info"
    #    messagesFilePath = MEVOCO_LOG_PATH + hostFolderName + "/messages"
    #    kvmagentFilePath = MEVOCO_LOG_PATH + hostFolderName + "/zstack-kvmagent.log"
    #    zstackFilePath   = MEVOCO_LOG_PATH + hostFolderName + "/zstack.log"

    #    if not os.path.exists(dmesgFilePath):
    #        test_util.test_fail( dmesgFilePath    + ' is not exist.')
    #    if not os.path.exists(hostInfoFilePath):
    #        test_util.test_fail( hostInfoFilePath + ' is not exist.')
    #    if not os.path.exists(messagesFilePath):
    #        test_util.test_fail( messagesFilePath + ' is not exist.')
    #    if not os.path.exists(kvmagentFilePath):
    #        test_util.test_fail( kvmagentFilePath + ' is not exist.')
    #    if not os.path.exists(zstackFilePath):
    #        test_util.test_fail( zstackFilePath   + 'is not exist.')

    #Step5: verify management node logs are saved
    #conditions = res_ops.gen_query_conditions("status", '=', "Connected")
    all_mn = res_ops.query_resource(res_ops.MANAGEMENT_NODE)
    if len(all_mn) < 1:
        test_util.test_skip('Not available mn to check')

    for mn in all_mn:
        mnHostIP = mn.hostName
        mnFolderName = "management-node-" + mnHostIP
        dmesgFilePath = MEVOCO_LOG_PATH + mnFolderName + "/dmesg"
        hostInfoFilePath = MEVOCO_LOG_PATH + mnFolderName + "/host_info"
        messagesFilePath = MEVOCO_LOG_PATH + mnFolderName + "/messages"
        manageServerPath = MEVOCO_LOG_PATH + mnFolderName + "/management-server.log"

        if not os.path.exists(dmesgFilePath):
            test_util.test_fail(dmesgFilePath + ' is not exist.')
        if not os.path.exists(hostInfoFilePath):
            test_util.test_fail(hostInfoFilePath + ' is not exist.')
        if not os.path.exists(messagesFilePath):
            test_util.test_fail(messagesFilePath + ' is not exist.')
        if not os.path.exists(manageServerPath):
            test_util.test_fail(manageServerPath + ' is not exist.')
예제 #11
0
def test():
    flavor = case_flavor[os.environ.get('CASE_FLAVOR')]
    test_util.test_dsc('bs auto reconnection check test')
    if flavor['sftp']:
        conditions = res_ops.gen_query_conditions('type', '=', 'SftpBackupStorage')
    if flavor['imagestore']:
        conditions = res_ops.gen_query_conditions('type', '=', 'ImageStoreBackupStorage')

    resource_type = res_ops.BACKUP_STORAGE
    bss = res_ops.query_resource(resource_type, conditions)
    if len(bss) == 0:
        test_util.test_skip('Skip due to no backup storage available')

    test_util.test_logger("kill bs agent on storage %s" % (bss[0].uuid))
    if flavor['sftp']:
        if flavor['kill']:
            cmd = "pkill -9 -f 'from sftpbackupstorage import sftpbackupstoragedaemon'"
        else:
            cmd = "service zstack-sftpbackupstorage stop"
    elif flavor['imagestore']:
        if flavor['kill']:
            cmd = "pkill -9 -f '/usr/local/zstack/imagestore/bin/zstore -conf'"
        else:
            cmd = "service zstack-imagestorebackupstorage stop"

    bs_host = test_lib.lib_get_backup_storage_host(bss[0].uuid)
    if bs_host.username != 'root':
        cmd = "sudo %s" % (cmd)
    if not bs_host.sshPort:
        bs_host.sshPort = 22
    if test_lib.lib_execute_ssh_cmd(bs_host.managementIp, bs_host.username, bs_host.password, cmd,  timeout = 120, port = int(bs_host.sshPort)) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, bs_host.managementIp))

    test_util.test_logger("bs %s is expected to disconnect and start reconnect automatically" % (bss[0].uuid))
    conditions = res_ops.gen_query_conditions('uuid', '=', bss[0].uuid)
    count = 0
    while count < 24:
        bs = res_ops.query_resource(resource_type, conditions)[0]
        if bs.status == "Connecting":
            break

        time.sleep(5)
        count += 1

    if bs.status != "Connecting":
        test_util.test_fail("bs %s is not disconnect and start reconnect automatically in 120 seconds" % (bss[0].uuid))

    test_util.test_logger("bs %s is expected to reconnect success automatically" % (bss[0].uuid))
    count = 0
    while count < 24:
        bs = res_ops.query_resource(resource_type, conditions)[0]
        if bs.status == "Connected":
            break

        time.sleep(5)
        count += 1

    if bs.status != "Connected":
        test_util.test_fail("bs %s not reconnect success automatically in 120 seconds" % (bss[0].uuid))

    test_util.test_pass("Auto reconnect backup storage pass")