Esempio n. 1
0
 def rebuld_loop(no_of_iterations):
     p2 = multiprocessing.current_process()
     logger.info("Starting : %s : %s" % (p2.name, p2.pid))
     print "Starting", p2.name, p2.pid
     for i in range(no_of_iterations):
         mediaList = media_group.get_media_group_disk()
         media = [mediaList[0], mediaList[1]]
         status = power_off_drive(media, pvlLibHandle)
         time.sleep(30)
         status = power_on_drive(media, pvlLibHandle)
         time.sleep(120)
         status = media_group.rebuild_media_group()
         logger.info(status)
         time.sleep(60)
Esempio n. 2
0
def pds_rollback_with_rebuild_20():
        #objects initialization & Authentication
    log = PvltLogger(inspect.stack()[0][3],'INFO')
    logger = log.logger
    host = HostOps(HOST[0])
    fio = FioUtils()
    pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
    result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)

        #calling base function to populate the objects
    media_group,vol = base_function(pvlLibHandle,logger)

        # assign the vol
    ports= get_volume_network_map(pvlLibHandle, ZONE)
    logger.info("Assigning vols ")
    status  = vol.assign_volume(ports,hostnqn=[])
    assert(status == True)

        # connet to the host
    status = host.connect_volume(vol)
    assert(status == 0)

        # powring off two drives 
    mediaList = media_group.get_media_group_disk()
    media = [mediaList[0]] + [mediaList[1]]
    logger.info("Powering OFF drive : %s"%media)
    status = power_off_drive([media_group.md_disk[1],media_group.md_disk[0]],pvlLibHandle)

        # write on the voluem
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
    status,response=fio.seq_write([vol], host, kwargs)
    assert (status==0)

        #Taking snaphost
    logger.info("Taking snapshot :%s"%SNAP_PREFIX)
    snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
    assert(snapshot != None)

        # poweron the drives 
    logger.info("Powering ON drive : %s"%media)
    status = power_on_drive(media,pvlLibHandle)
    time.sleep(60)

        # start rebuild 
    logger.info("Start rebuild 2>0")
    status=media_group.synchronous_rebuild_media_group()

        # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
    status,response=fio.seq_write([vol], host, kwargs)
    assert (status==0)

        # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)

        # unassign  the vol
    status = vol.unassign_volume()
    assert(status == True)

        # restore the volumes using s2
    status = vol.vol_rollback(snapshot,backup="true")
    assert(status == True)

        # connect the vol & verify the data
    logger.info("Assigning vols ")
    status  = vol.assign_volume(ports,hostnqn=[])
    assert(status == True)

    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    assert(status==0)
    logger.info("Reading from the raw device")
    kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
    status,response=fio.seq_read([vol], host, kwargs)
    assert (status==0)

         # delete forensic snapshot
    status = snapshot.delete_all_forensic_snap()
    assert(status == True)

        # disconnect the vol to run teardown
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
Esempio n. 3
0
def pds_vol_restore_sanity():
    host = HostOps(HOST[0])
    fio = FioUtils()
    # give function name as log file name
    log = PvltLogger("Volume_restore", 'INFO')
    logger = log.logger
    name = MEDIA_GROUP_NAME
    pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
    result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
    ## velidation for prerequisits
    drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
    logger.info(ZONE)
    logger.info(drv_cnt)
    if drv_cnt < 9:
        logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
        sys.exit()
    if VOL_CNT > 1:
        logger.error(
            'Update the configuration As this test is currently written for 1 vol only'
        )
        sys.exit()
    md_type = MEDIA_GROUP_TYPE
    ## creating mediaGroup
    logger.info("Creating a MediaGroup")
    media_group = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=True)
    assert (media_group != None)
    ## creaing volumes
    logger.info("Creating voluems")
    #vol_list = []
    volname = VOL_PREFIX
    vol = VolumeOps(md=media_group,
                    name=volname,
                    size=VOL_SIZE,
                    vol_res=VOL_RES,
                    flavor=FLAVOR,
                    create=True)
    assert (vol != None)

    #Creating a snapshot
    snap_name = "s1"
    logger.info("Creating a snapshot : %s" % snap_name)
    snapshot = SnapShotOps(snap_name, volobj=vol, create=True)
    assert (snapshot != None)

    ################################ 1 Vol restore for unassigned volume
    # Restore the vol using its snapshot
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    #print("1 Vol restore for unassigned volume: PASS")

    ################################ 2 Vol restore after assign volume
    ports = get_two_port_diff_ctlr(ZONE)
    logger.info("Assigning vols ")
    status = vol.assign_volume(ports, hostnqn=[])
    assert (status == True)
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    #print("2 Vol restore after assign volume:PASS")

    ################################## delete s1 and its child snapshot
    #  Need to write forensic snapshots deletion code
    ################################## 3. Vol restore after 5% IO on the vol
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    #start fio on all the volumes
    logger.info("Starting write on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # Taking snaphost
    snap_name = "s2"
    logger.info("Taking snapshot :%s" % snap_name)
    snapshot = SnapShotOps(snap_name, volobj=vol, create=True)
    assert (snapshot != None)
    # Write on remaning space of the volumes
    logger.info("Starting write on other space of the vol ")
    kwargs = {
        'offset': "45g",
        'size': '5g',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.rand_read_write([vol], host, kwargs)
    assert (status == 0)
    # disconnect the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore using latest snapshot
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # cheking data integrity by rading the volumes data
    # connecting the vol
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    assert (status == 0)
    print("3. Vol restore after 5% IO on the vol:PASS")

    ################################ 4. Vol restore for degraded volumes
    # Power off one drive
    mediaList = media_group.get_media_group_disk()
    media = [mediaList[0]]
    logger.info("Powering off drive : %s" % media)
    status = power_off_drive(media, pvlLibHandle)
    time.sleep(30)
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore the volumes using s2
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    print("4. Vol restore for degraded volumes:PASS")

    ################################ 5. Vol restore for critical voluems
    # poweroff one more drive
    media = [mediaList[1]]
    logger.info("Powering off drive : %s" % media)
    status = power_off_drive(media, pvlLibHandle)
    time.sleep(30)
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore the volumes using s2
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    print("5. Vol restore for critical voluems:PASS")

    ################################ 6. Vol restore with rebuild (2>1)
    # Poweron one drive and start the rebuild
    media = [mediaList[0]]
    logger.info("Powering ON drive : %s" % media)
    status = power_on_drive(media, pvlLibHandle)
    time.sleep(120)
    logger.info("Start rebuild 2>1")
    status = media_group.rebuild_media_group()
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore the volumes using s2
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    print("6. Vol restore with rebuild (2>1):PASS")

    ################################ 7. Vol restore with rebuild (1>0)
    media = [mediaList[1]]
    logger.info("Powering ON drive : %s" % media)
    status = power_on_drive(media, pvlLibHandle)
    time.sleep(120)
    logger.info("Start rebuild 1>0")
    status = media_group.rebuild_media_group()
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore the volumes using s2
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    print("7. Vol restore with rebuild (1>0):PASS")

    ################################ 8. Vol restore with rebuild (2>0)
    media = [media_group.md_disk[1], media_group.md_disk[0]]
    logger.info("Powering OFF drive : %s" % media)
    status = power_off_drive([media_group.md_disk[1], media_group.md_disk[0]],
                             pvlLibHandle)
    time.sleep(60)
    logger.info("Powering ON drive : %s" % media)
    status = power_on_drive([media_group.md_disk[1], media_group.md_disk[0]],
                            pvlLibHandle)
    time.sleep(60)
    # start rebuild
    logger.info("Start rebuild 2>0")
    status = media_group.rebuild_media_group()
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore the volumes using s2
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    print("8. Vol restore with rebuild (2>0):PASS")

    ################################ 9. Vol restore with clone is connected to the host
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write([vol], host, kwargs)
    assert (status == 0)
    # create clone
    clone_list = []
    clone_name = str(snap_name) + str("_c1")
    logger.info("Creating clone name : %s" % clone_name)
    cloneObj = CloneOps(clone_name, snapshot, create=True)
    clone_list.append(cloneObj)
    # assign clone
    logger.info("Assigning clone %s to contollers" % clone_name)
    assign = cloneObj.assign_clone(ports, hostnqn=[])
    # connect clone to the host
    logger.info("Connecting clone %s to the host" % clone_name)
    status = host.connect_volume(cloneObj)
    time.sleep(10)
    assert (status == 0)
    # start IO on the clone
    logger.info("Staring write IO to the clone %s" % clone_name)
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(clone_list, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = host.disconnect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    # restore the volumes using s2
    #status = snapshot.vol_restore()
    #assert(status == True)
    #time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = host.connect_volume(vol)
    time.sleep(10)
    assert (status == 0)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read([vol], host, kwargs)
    print("9. Vol restore with clone is connected to the host:PASS")
Esempio n. 4
0
################################ 1. FIO on online volume
        #start fio on all the volumes with BS:4k/IODEPTH:128
     for blk_size in blk_list:
         for io_type in ioitems:
             logger.info("Starting IO Online volumes : IO_TYPE=%s :BS=%s"%(io_type,blk_size))
             kwargs = {'offset':"0",'bs':blk_size,'size':'5%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
             status,response=fio.generic_fio(io_type,[vol], host, kwargs)
             assert (status == 0)
             logger.info("Completed IO for Online volume with:IO_TYPE=%s||BS=%s"%(io_type,blk_size))
    
################################ 2. IO on degraded volumes 
        # Power off one drive 
    media = media_group.md_disk[0]
    #media = [mediaList[0]]
    logger.info("Powering off drive : %s"%media)
    status = power_off_drive([media],pvlLibHandle)
    time.sleep(30)
        # overwrite on the volume 
    for blk_size in blk_list:
        for io_type in ioitems:
            logger.info("Starting IO Online volumes : IO_TYPE=%s :BS=%s"%(io_type,blk_size))
            kwargs = {'offset':"0",'bs':blk_size,'size':'5%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
            status,response=fio.generic_fio(io_type,[vol], host, kwargs)
            assert (status == 0)
            logger.info("Completed IO for degraded volume with:IO_TYPE=%s||BS=%s"%(io_type,blk_size))

################################ 3. IO on critical voluems 
        # poweroff one more drive 
    #media = [mediaList[1]]
    media = media_group.md_disk[1]
    logger.info("Powering off drive : %s"%media)
Esempio n. 5
0
def pds_vol_restore_sanity():
    host = HostOps(HOST[0])
    fio = FioUtils()
    # give function name as log file name
    log = PvltLogger("Volume_restore", 'INFO')
    logger = log.logger
    name = MEDIA_GROUP_NAME
    pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
    result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
    ## creating mediaGroup
    drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
    logger.info(ZONE)
    logger.info(drv_cnt)
    if drv_cnt < 9:
        logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
        #assert(drv_cnt > 8 )

    # take it from config
    md_type = 'RAID-0 (18+0)' if drv_cnt == 18 else 'RAID-6 (7+2)'
    logger.info("Creating a MediaGroup")
    md = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=False)
    media_group = md
    assert (md != None)
    ## creaing volumes
    logger.info("Creating voluems")
    vol_list = []
    for i in range(1):
        # dont use + fro contact use %
        volname = str(VOL_PREFIX) + str("_") + str(i)
        volname = '%s_%s' % (VOL_PREFIX, i)
        vol = VolumeOps(md=media_group,
                        name=volname,
                        size=VOL_SIZE,
                        vol_res=VOL_RES,
                        flavor=FLAVOR,
                        create=False)
        vol_list.append(vol)
    #Creating a snapshot
    snap_name = "s1"
    logger.info("Creating a snapshot : %s" % snap_name)
    snapshot = SnapShotOps(snap_name, volobj=vol_list[0], create=False)
    assert (snapshot != None)

    ################################ 1 Vol restore for unassigned volume
    # Restore the vol using its snapshot
    snap_id = snapshot.snap_id
    print snap_id
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    print("1 Vol restore for unassigned volume: PASS")

    ################################ 2 Vol restore after assign volume
    ports = get_two_port_diff_ctlr(ZONE)
    logger.info("Assigning vols ")
    assign = [v.assign_volume(ports, hostnqn=[]) for v in vol_list]
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    print("2 Vol restore after assign volume:PASS")

    ################################## delete s1 and its child snapshot
    #vol,snap_list,clone_list = list_existing_obj(md.name)
    #for snap in snap_list:
    #snap_to_delete = SnapShotOps(snap,volobj=vol_list[0])
    #if snap != snap_name:
    #status = snap_to_delete.delete_snapshot
    #assert(status != None)
    #result = pvlLibHandle.storage.get_snapshot
    #print result
    #print json.dumps(result,indent=4)
    ################################## 3. Vol restore after 5% IO on the vol
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    #stert fio on all the volumes
    logger.info("Starting write on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    # Taking snaphost
    snap_name = "s2"
    logger.info("Taking snapshot :%s" % snap_name)
    snapshot = SnapShotOps(snap_name, volobj=vol_list[0], create=True)
    assert (snapshot != None)
    # Write on remaning space of the volumes
    logger.info("Starting write on other space of the vol ")
    kwargs = {
        'offset': "45g",
        'size': '5g',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.rand_read_write(vol_list, host, kwargs)
    assert (status == 0)
    # disconnect the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    #assert(status==0)
    # restore using latest snapshot
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    snap_id = snapshot.snap_id
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # cheking data integrity by rading the volumes data
    # connecting the vol
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    assert (status == 0)
    print("3. Vol restore after 5% IO on the vol:PASS")

    ################################ 4. Vol restore for degraded volumes
    # Power off one drive
    mediaList = media_group.get_media_group_disk()
    media = [mediaList[0]]
    logger.info("Powering off drive : %s" % media)
    status = power_off_drive(media, pvlLibHandle)
    time.sleep(30)
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    # restore the volumes using s2
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    print("4. Vol restore for degraded volumes:PASS")

    ################################ 5. Vol restore for critical voluems
    # poweroff one more drive
    media = [mediaList[1]]
    logger.info("Powering off drive : %s" % media)
    status = power_off_drive(media, pvlLibHandle)
    time.sleep(30)
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    # restore the volumes using s2
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    print("5. Vol restore for critical voluems:PASS")

    ################################ 6. Vol restore with rebuild (2>1)
    # Poweron one drive and start the rebuild
    media = [mediaList[0]]
    logger.info("Powering ON drive : %s" % media)
    status = power_on_drive(media, pvlLibHandle)
    time.sleep(120)
    logger.info("Start rebuild 2>1")
    status = media_group.rebuild_media_group()
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != None)
    time.sleep(5)
    # restore the volumes using s2
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    print("6. Vol restore with rebuild (2>1):PASS")

    ################################ 7. Vol restore with rebuild (1>0)
    media = [mediaList[1]]
    logger.info("Powering ON drive : %s" % media)
    status = power_on_drive(media, pvlLibHandle)
    time.sleep(120)
    logger.info("Start rebuild 1>0")
    status = media_group.rebuild_media_group()
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    # restore the volumes using s2
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    print("7. Vol restore with rebuild (1>0):PASS")

    ################################ 8. Vol restore with rebuild (2>0)
    media = [mediaList[0], mediaList[1]]
    logger.info("Powering OFF drive : %s" % media)
    status = power_off_drive(media, pvlLibHandle)
    time.sleep(120)
    logger.info("Powering ON drive : %s" % media)
    status = power_on_drive(media, pvlLibHandle)
    time.sleep(120)
    # start rebuild
    logger.info("Start rebuild 2>0")
    status = media_group.rebuild_media_group()
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    # restore the volumes using s2
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    print("8. Vol restore with rebuild (2>0):PASS")

    ################################ 9. Vol restore with clone is connected to the host
    # overwrite on the volume
    logger.info("Starting overwrite on the vol-seq_write")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(vol_list, host, kwargs)
    assert (status == 0)
    # create clone
    clone_name = str(snap_name) + str("_c1")
    cloneObj = CloneOps(clone_name, snapshot, create=True)
    # assign clone
    assign = cloneobj.assign_clone(ports, hostnqn=[])
    # connect clone to the host
    status = host.connect_volume(cloneObj)
    assert (status != 0)
    time.sleep(5)
    # start IO on the clone
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_write(cloneObj, host, kwargs)
    assert (status == 0)
    # disconenct the vol
    logger.info("disconnecting the vol from the host")
    status = [host.disconnect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    # restore the volumes using s2
    logger.info("Running vol_restore using following snaphost : %s" %
                snap_name)
    status = pvlLibHandle.system.generic_api(
        'POST', '/storage/snapshots/%s/restore' % snap_id)
    assert (status != 200)
    time.sleep(60)
    # connect the vol & verify the data
    logger.info("connecting the vol")
    status = [host.connect_volume(v) for v in vol_list]
    assert (status != 0)
    time.sleep(5)
    logger.info("Reading from the raw device")
    kwargs = {
        'offset': "0",
        'size': '5%',
        "verify_pattern": "0xABCD",
        "verify_interval": 4096,
        "do_verify": 1
    }
    status, response = fio.seq_read(vol_list, host, kwargs)
    print("9. Vol restore with clone is connected to the host:PASS")