def pds_io_sanity_with_snap_online_vol():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol_list = base_function(pvlLibHandle,logger)
# assigning vol
ports = get_two_port_diff_ctlr(ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
for i in range(5):
snap_list = []
#start fio on all the volumes
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
for vol in vol_list:
#Taking snaphost
snap_name = str(vol.name) + str("_%s"%SNAP_PREFIX)
logger.info("Taking snapshot :%s"%snap_name)
snapshot = SnapShotOps(snap_name,volobj=vol,create=True)
assert(snapshot != None)
snap_list.append(snapshot)
#Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
# delete snapshot
for snap in snap_list:
status = snap.delete_snapshot()
assert (status == True)
time.sleep(300)
#disconnect the vol
for vol in vol_list:
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
def pds_raid6_io_sanity():
host = HostOps(HOST[0])
fio = FioUtils()
# give function name as log file name
log = PvltLogger("raid6_io_sanity",'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
## velidation for prerequisits
drv_cnt = get_active_drive_count(ZONE,pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9 :
logger.error('Not sufficient drives active drive count=%s'%drv_cnt)
sys.exit()
if VOL_CNT > 1 :
logger.error('Update the configuration As this test is currently written for 1 vol only')
sys.exit()
md_type = MEDIA_GROUP_TYPE
## creating mediaGroup
logger.info("Creating a MediaGroup")
media_group = MediaGroupOps(pvlLibHandle, name, ZONE, md_type,create=True)
assert(media_group != None )
## creaing volumes
logger.info("Creating voluems")
#vol_list = []
volname = VOL_PREFIX
vol=VolumeOps(md=media_group,name=volname,size=VOL_SIZE,vol_res=VOL_RES,flavor=FLAVOR,create=True)
assert(vol != None )
# Assigning and connecting the volume
ports = get_two_port_diff_ctlr(ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status)
status = host.connect_volume(vol)
time.sleep(10)
assert(status == 0)
#ioitems = ['write','randwrite','readwrite','randrw','read']
#blk_list = ['4k','8k','16k','32k','64k','128k','256k','512k','1m','2m']
ioitems = ['write']
blk_list = ['4k']
################################ 1. FIO on online volume
#start fio on all the volumes with BS:4k/IODEPTH:128
for blk_size in blk_list:
for io_type in ioitems:
logger.info("Starting IO Online volumes : IO_TYPE=%s :BS=%s"%(io_type,blk_size))
kwargs = {'offset':"0",'bs':blk_size,'size':'5%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.generic_fio(io_type,[vol], host, kwargs)
assert (status == 0)
logger.info("Completed IO for Online volume with:IO_TYPE=%s||BS=%s"%(io_type,blk_size))
def pds_create_mediagroup():
global logger
global pvlLibHandle
log = PvltLogger("rebuild_operation", 'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
#assert(drv_cnt > 8 )
md_type = 'RAID-0 (18+0)' if drv_cnt == 18 else 'RAID-6 (6+2+1)'
md = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=False)
media_group = md
assert (md != None)
## deleting mediaGroup
#print("now deleting the mg in 5 sec")
#time.sleep(5)
#media_group.delete_mediagroup()
## creaing volumes
vol_list = []
for i in range(2):
volname = str(VOL_PREFIX) + str("_") + str(i)
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
vol_list.append(vol)
#print("following volumes have been created : %s"%vol_list)
#'''
## assign volume
ports = get_two_port_diff_ctlr(ZONE)
for v in vol_list:
assign = v.assign_volume(ports)
#connecting voluems to the host
for v in vol_list:
host = HostOps(HOST[0])
status = host.connect_volume(v)
#assert(status==0)
# stert fio on all the volumes
'''
def pds_rollback_assign_vol():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
#Creating a snapshot
logger.info("Creating a snapshot : %s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# run vol rollback
try:
status = vol.vol_rollback(snapshot,backup="true")
#assert(vol.vol_rollback(snapshot,backup="true") == False)
if(status == False):
raise Exception
except Exception:
logger.info("Rollback operations can`t be perfomed on Assigned vol, Please-Unassign the vol and try again ")
def pds_rollback_unassign_vol():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
#Creating a snapshot
logger.info("Creating a snapshot : %s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
# Restore the vol using its snapshot
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
# delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
logger.info("1 Vol restore for unassigned volume: PASS")
def pds_vol_rollback_sanity():
host = HostOps(HOST[0])
fio = FioUtils()
# give function name as log file name
log = PvltLogger("Volume_restore", 'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
## velidation for prerequisits
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
'''
if drv_cnt < 9 :
logger.error('Not sufficient drives active drive count=%s'%drv_cnt)
sys.exit()
if VOL_CNT > 1 :
logger.error('Update the configuration As this test is currently written for 1 vol only')
sys.exit()
'''
md_type = MEDIA_GROUP_TYPE
## creating mediaGroup
logger.info("Creating a MediaGroup")
media_group = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=True)
assert (media_group != None)
## creaing volumes
logger.info("Creating voluems")
volname = VOL_PREFIX
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
assert (vol != None)
time.sleep(10)
#Creating a snapshot
snap_name = "s1"
logger.info("Creating a snapshot : %s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol, create=True)
assert (snapshot != None)
time.sleep(10)
print vol.vol_id
print snapshot.snap_id
################################ 1 Vol restore for unassigned volume
# Restore the vol using its snapshot
status = vol.vol_rollback(snapshot, backup="true")
assert (status == True)
time.sleep(30)
print("1 Vol restore for unassigned volume: PASS")
# delete forensic snaphsot
#status = snapshot.delete_ferensic_snap(ZONE,media_group.name,snapshot.name)
status = snapshot.delete_all_forensic_snap()
#assert(status == 0)
'''
def pds_vol_restore_sanity():
host = HostOps(HOST[0])
fio = FioUtils()
# give function name as log file name
log = PvltLogger("Volume_restore", 'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
## creating mediaGroup
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
#assert(drv_cnt > 8 )
# take it from config
#md_type = 'RAID-0 (18+0)' if drv_cnt == 18 else 'RAID-6 (7+2)'
md_type = MEDIA_GROUP_TYPE
logger.info("Creating a MediaGroup")
media_group = MediaGroupOps(pvlLibHandle,
name,
ZONE,
md_type,
create=False)
assert (media_group != None)
## creaing volumes
logger.info("Creating voluems")
vol_list = []
for i in range(1):
volname = '%s_%s' % (VOL_PREFIX, i)
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
vol_list.append(vol)
#Creating a snapshot
snap_name = "s1"
logger.info("Creating a snapshot : %s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol_list[0], create=True)
assert (snapshot != None)
################################ 1 Vol restore for unassigned volume
# Restore the vol using its snapshot
status = snapshot.vol_restore()
#snap_id = snapshot.snap_id
#print snap_id
#logger.info("Running vol_restore using following snaphost : %s"%snap_name)
#status = pvlLibHandle.system.generic_api('POST', '/storage/snapshots/%s/restore'%snap_id)
#assert(status != 200)
#time.sleep(60)
print("1 Vol restore for unassigned volume: PASS")
'''
def startIO():
p1 = multiprocessing.current_process()
logger.info("Starting : %s : %s"%(p1.name,p1.pid))
print "Starting :",p1.name,p1.pid
host = HostOps(HOST[0]) # getting host object
fio = FioUtils() # getting fio object
kwargs = {'size':'100%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
#status,response=fio.rand_read_write(vol_list, host, kwargs)
status,response=fio.seq_write(vol_list, host, kwargs)
assert (status==0)
def pds_rollback_unassign_vol():
################################ 1 Vol restore for unassigned volume
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
# defining variables
name = MEDIA_GROUP_NAME
md_type = MEDIA_GROUP_TYP
volname = VOL_PREFIX
# validation for prerequisits
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
sys.exit()
if VOL_CNT > 1:
logger.error(
'Update the configuration As this test is currently written for 1 vol only'
)
sys.exit()
# creating mediaGroup
logger.info("Creating a MediaGroup")
media_group = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=True)
assert (media_group != None)
# creaing volumes
logger.info("Creating voluems")
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
assert (vol != None)
time.sleep(10)
#Creating a snapshot
snap_name = "s1"
logger.info("Creating a snapshot : %s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol, create=True)
assert (snapshot != None)
time.sleep(10)
# Restore the vol using its snapshot
status = vol.vol_rollback(snapshot, backup="true")
assert (status == True)
time.sleep(60)
# delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert (status == True)
print("1 Vol restore for unassigned volume: PASS")
def base_function(pvlLibHandle, logger):
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
# defining variables
name = MEDIA_GROUP_NAME
md_type = MEDIA_GROUP_TYPE
volname = VOL_PREFIX
# validation for prerequisits
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
sys.exit()
if VOL_CNT > 1:
logger.error(
'Update the configuration As this test is currently written for 1 vol only'
)
sys.exit()
# creating mediaGroup
logger.info("Creating a MediaGroup")
media_group = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=True)
assert (media_group != None)
# creaing volumes
logger.info("Creating voluems")
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
assert (vol != None)
time.sleep(10)
return media_group, vol
def IO_sanity_func(volType="online", funcname="pds_online_vol_IO_sanity"):
log = PvltLogger(funcname, 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
def io_patterns():
logger.info("Starting fresh write on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '30%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1,
"bs": "64k"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# starting overwrite with diff pattern
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '50%',
"verify_pattern": "0xFFFF",
"verify_interval": 4096,
"do_verify": 1,
"bs": "64k"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# starting multijob FIO load but without verification
logger.info(
"Starting multithreaded write without verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '70%',
"numjobs": "8",
"bsrange": "4k-2M"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# Starting mutithreaded write with verification
logger.info(
"Starting mutithreaded-write with verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '100%',
"verify_pattern": "0xFFFF",
"verify_interval": 4096,
"do_verify": 1,
"numjobs": "1",
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5",
"multithread_verification": "1"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
if volType == "online":
io_patterns()
assert (media_group.get_media_group_state() == 2)
for vol in vol_list:
state = vol.get_vol_state()
assert ('Online' in state)
if volType == "degraded":
# Power off one drive
media = media_group.md_disk[0]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
assert (media_group.get_media_group_state() == 6)
for vol in vol_list:
state = vol.get_vol_state()
assert ('Online, degraded' in state)
# starting io
io_patterns()
# poweron the drive again
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start async-rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
time.sleep(10)
assert (media_group.get_media_group_state() == 2)
if volType == "critical":
# poweroff 2 drive
media = [media_group.md_disk[0]] + [media_group.md_disk[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
assert (media_group.get_media_group_state() == 7)
for vol in vol_list:
state = vol.get_vol_state()
assert ('Online, critical' in state)
# starting io
io_patterns()
# Poweron one drive
media = [media_group.md_disk[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
media = [media_group.md_disk[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start async-rebuild
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
assert (media_group.get_media_group_state() == 2)
def pds_one_drive_rebuild():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
#calling base function to populate the objects
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
#start FIO thread here
temp_file_path = "%s/tmp/run_fio.%s" % (env['TESTDIR'], os.getpid())
cmd = "touch %s " % temp_file_path
run(cmd)
proc = multiprocessing.Process(target=fio.infinite_fio,
args=(vol_list, host, temp_file_path))
proc.start()
logger.info('Fio process id code %s' % proc.pid)
sleep_p(60)
if proc.is_alive():
sleep_p(60)
else:
logger.error('Fio process failed %s' % proc.pid)
assert (proc.exitcode == 0)
try:
# Power off one drive
mediaList = media_group.get_media_group_disk()
for i in range(1):
media = [mediaList[i]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
# poweron the drive again
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
if proc.is_alive():
# start async-rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
except Exception as E:
logger.error('Stack trace for failure \n%s\n' % traceback.format_exc())
raise E
finally:
cmd = "rm -f %s " % temp_file_path
run(cmd)
if proc.is_alive():
proc.join()
code = proc.exitcode
if code != 0:
logger.error('Fio failed on volumes ')
assert (code == 0)
def pds_create_mediagroup():
global logger
global pvlLibHandle
log = PvltLogger("rebuild_operation", 'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
#assert(drv_cnt > 8 )
md_type = 'RAID-0 (18+0)' if drv_cnt == 18 else 'RAID-6 (7+2)'
md = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=False)
media_group = md
assert (md != None)
## deleting mediaGroup
#print("now deleting the mg in 5 sec")
#time.sleep(5)
#media_group.delete_mediagroup()
## creaing volumes
vol_list = []
for i in range(4):
volname = str(VOL_PREFIX) + str("_") + str(i)
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
vol_list.append(vol)
#print("following volumes have been created : %s"%vol_list)
#'''
## assign volume
ports = get_two_port_diff_ctlr(ZONE)
for v in vol_list:
assign = v.assign_volume(ports)
#connecting voluems to the host
for v in vol_list:
host = HostOps(HOST[0])
status = host.connect_volume(v)
#assert(status==0)
# stert fio on all the volumes
#'''
def startIO():
p1 = multiprocessing.current_process()
logger.info("Starting : %s : %s" % (p1.name, p1.pid))
print "Starting :", p1.name, p1.pid
host = HostOps(HOST[0]) # getting host object
fio = FioUtils() # getting fio object
kwargs = {
'size': '100%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
### getting used disks in the mediaGroup
def rebuld_loop(no_of_iterations):
p2 = multiprocessing.current_process()
logger.info("Starting : %s : %s" % (p2.name, p2.pid))
print "Starting", p2.name, p2.pid
for i in range(no_of_iterations):
mediaList = media_group.get_media_group_disk()
media = [mediaList[0], mediaList[1]]
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
status = power_on_drive(media, pvlLibHandle)
time.sleep(120)
status = media_group.rebuild_media_group()
logger.info(status)
time.sleep(60)
#assert (status==0)
p1 = multiprocessing.Process(name="StartFio", target=startIO)
p1.deamon = False
p2 = multiprocessing.Process(name="2_drive_rebuld",
target=rebuld_loop,
args=(10, ))
p2.deamon = True
p1.start()
p2.start()
p1.join()
p2.join()
def pds_critical_vol_IO_sanity():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
#calling base function to populate the objects
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
# poweroff 2 drive
mediaList = media_group.get_media_group_disk()
media = [mediaList[0]] + [mediaList[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
for i in range(1):
snap_list = []
#star fresh write on all the volumes
logger.info("Starting fresh write on the vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with same pattern
logger.info("Starting over-write with same pattern -vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
'''
# start overwrite with diff pattern (rw=rw,splitbs)
logger.info("Starting over-write with diff pattern -rw=rw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDffff","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.seq_read_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with diff pattern (rw=randrw,splitbs)
logger.info("Starting over-write with diff pattern -rw=randrw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# randwrite on online vol
logger.info("Starting over-write with diff pattern -rw=randwrite")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
'''
# Poweron one drive and start the rebuild
media = [mediaList[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start rebuild with IO
logger.info("Start rebuild 2>1")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
time.sleep(60)
# starting IO load
#logger.info("Starting over-write with diff pattern -rw=randrw")
#kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
# "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
#status,response=fio.rand_read_write(vol_list, host, kwargs)
#assert (status == 0)
# Poweron second drive and start the rebuild
media = [mediaList[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start async-rebuild
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
# starting IO load
logger.info("Starting over-write with diff pattern -rw=randrw")
kwargs = {
'offset': "0",
'size': '50%',
"verify_pattern": "0xABCDabcd",
"verify_interval": 4096,
"do_verify": 1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
def pds_rollback_with_clone():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
# assigning vol
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connect vol
logger.info("connecting vol to the host")
status = host.connect_volume(vol)
assert(status == 0)
# write on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
# Creating a snapshot
logger.info("Creating a snapshot : %s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
# create clone
clone_name = str(SNAP_PREFIX)+str("_c1")
logger.info("Creating clone name : %s"%clone_name)
cloneObj = CloneOps(clone_name,snapshot,create=True)
# assign clone
logger.info("Assigning clone %s to contollers"%cloneObj.name)
assign = cloneObj.assign_clone(ports,hostnqn=[])
# connect clone to the host
logger.info("Connecting clone %s to the host"%cloneObj.name)
status = host.connect_volume(cloneObj)
assert(status==0)
# start IO on the clone
logger.info("Staring write IO to the clone %s"%clone_name)
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([cloneObj], host, kwargs)
assert (status==0)
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# unassign the vol
status = vol.unassign_volume()
assert(status == True)
# restore the volumes using s2
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
# connect the vol & verify the data
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
logger.info("connecting the vol")
status = host.connect_volume(vol)
assert(status == 0)
logger.info("Reading from the raw device")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_read([vol], host, kwargs)
assert (status==0)
# delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
# disconnect the vol to run teardown
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# disconnet the clone
logger.info("disconnecting clone : %s"%cloneObj.name)
status = host.disconnect_volume(cloneObj)
time.sleep(10)
def pds_rollback_with_IO():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
# assign the vol
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
#start fio on all the volumes
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status == 0)
#Taking snaphost
logger.info("Taking snapshot :%s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
#Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {'offset':"45g",'size':'5g',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.rand_write([vol], host, kwargs)
assert (status == 0)
#disconnect the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# unassign the vol
status = vol.unassign_volume()
assert(status == True)
#restore using latest snapshot
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
#cheking data integrity by rading the volumes data
#assign and then connecting the vol
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
logger.info("connecting the vol")
status = host.connect_volume(vol)
assert(status == 0)
logger.info("Reading from the raw device")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_read([vol], host, kwargs)
assert (status==0)
#delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
# disconnect the vol to run teardown
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_ctrl_fofb():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
ctrl_id = list(get_all_ids_in_zone(ZONE))
ctrl_obj_0 = ControllerOps(pvlLibHandle,ctrl_id[0],"rdma")
ctrl_obj_1 = ControllerOps(pvlLibHandle,ctrl_id[1],"rdma")
#calling base function to populate the objects
media_group,vol_list = base_function(pvlLibHandle,logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
def startIO():
p1 = multiprocessing.current_process()
logger.info("Starting : %s : %s"%(p1.name,p1.pid))
print "Starting :",p1.name,p1.pid
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'100%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
#start fio on all the volumes
p1 = multiprocessing.Process(name="StartFio",target=startIO)
p1.start()
#logger.info("Starting write on the vol-seq_write")
#kwargs = {'offset':"0",'size':'1%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
#status,response=fio.seq_write(vol_list, host, kwargs)
#assert (status == 0)
# start FOFB with fresh write
logger.info("Start controler poweroff/on")
for i in range(10):
# controller poeroff
status = ctrl_obj_0.power_off()
#assert(status == True)
time.sleep(60)
# controller poweron
status = ctrl_obj_0.power_on()
#assert(status == True)
time.sleep(150)
# poweroff/on another controller
status = ctrl_obj_1.power_off()
#assert(status == True)
time.sleep(60)
# poweroff/on another controller
status = ctrl_obj_1.power_on()
#assert(status == True)
time.sleep(60)
p1.join()
#disconnect the vol
for vol in vol_list:
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_online_vol_IO_sanity():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
#calling base function to populate the objects
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
for i in range(1):
snap_list = []
#star fresh write on all the volumes
logger.info("Starting fresh write on the vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with same pattern
logger.info("Starting over-write with same pattern -vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
'''
# start overwrite with diff pattern (rw=rw,splitbs)
logger.info("Starting over-write with diff pattern -rw=rw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDffff","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.seq_read_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with diff pattern (rw=randrw,splitbs)
logger.info("Starting over-write with diff pattern -rw=randrw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# randwrite on online vol
logger.info("Starting over-write with diff pattern -rw=randwrite")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
for vol in vol_list:
#Taking snaphost
snap_name = str(vol.name) + str("_%s"%SNAP_PREFIX)
logger.info("Taking snapshot :%s"%snap_name)
snapshot = SnapShotOps(snap_name,volobj=vol,create=True)
assert(snapshot != None)
snap_list.append(snapshot)
#Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {'offset':"0",'size':'1%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
# delete snapshot
for snap in snap_list:
status = snap.delete_snapshot()
assert (status == True)
time.sleep(600)
'''
#disconnect the vol
for vol in vol_list:
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_online_vol_IO_sanity():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
ctrl_id = list(get_all_ids_in_zone(ZONE))
ctrl_obj_0 = ControllerOps(pvlLibHandle,ctrl_id[0],"rdma")
ctrl_obj_1 = ControllerOps(pvlLibHandle,ctrl_id[1],"rdma")
#calling base function to populate the objects
media_group,vol_list = base_function(pvlLibHandle,logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
def startIO():
p1 = multiprocessing.current_process()
logger.info("Starting : %s : %s"%(p1.name,p1.pid))
print "Starting :",p1.name,p1.pid
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1,
"bs":"64k"}
status,response=fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# # start FIO thread here
#
# temp_file_path = "%s/tmp/run_fio.%s"%(env['TESTDIR'] ,os.getpid())
# cmd = "touch %s "%temp_file_path
# run(cmd)
# proc=multiprocessing.Process(target=fio.infinite_fio,args=(vol_list, host, temp_file_path))
# proc.start()
# logger.info('Fio process id code %s'%proc.pid)
# sleep_p(60)
# if proc.is_alive():
# sleep_p(60)
# else:
# logger.error('Fio process failed %s'%proc.pid)
# assert(proc.exitcode==0)
#
# cmd = "rm -f %s "%temp_file_path
# run(cmd)
# if proc.is_alive():
# proc.join()
# print"fio run finished"
# ## start graceful FOFB while the IOs are running
# #start fio on all the volumes
# p1 = multiprocessing.Process(name="StartFio",target=startIO)
# logger.info("starting IOs just in parallel with FOFB operations")
# p1.start()
# time.sleep(5)
def rebuilds_func():
# starting one driver rebuild
# Power off one drive
mediaList = media_group.get_media_group_disk()
for i in range(1):
media = [mediaList[i]]
logger.info("Powering off drive : %s"%media)
status = power_off_drive(media,pvlLibHandle)
time.sleep(30)
# poweron the same drive
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
#status=media_group.asynchronous_rebuild_media_group()
assert(media_group.synchronous_rebuild_media_group())
time.sleep(60)
## starting 2 drive rebuild
for i in range(10):
media = [mediaList[0]] + [mediaList[1]]
logger.info("Powering off drive : %s"%media)
status = power_off_drive(media,pvlLibHandle)
time.sleep(60)
# poweron the drive again
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start async-rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
#status=media_group.asynchronous_rebuild_media_group()
assert(media_group.synchronous_rebuild_media_group())
time.sleep(60)
## 2>1 && 1>0 rebuild
#mediaList = media_group.get_media_group_disk()
media = [mediaList[2]] + [mediaList[3]]
logger.info("Powering off drive : %s"%media)
status = power_off_drive(media,pvlLibHandle)
time.sleep(60)
# Poweron one drive and start the rebuild
media = [mediaList[2]]
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start rebuild with IO
logger.info("Start rebuild 2>1")
#status=media_group.synchronous_rebuild_media_group()
status=media_group.asynchronous_rebuild_media_group()
time.sleep(60)
if proc.is_alive():
# Poweron second drive and start the rebuild
media = [mediaList[3]]
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start async-rebuild
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
#status=media_group.asynchronous_rebuild_media_group()
assert(media_group.synchronous_rebuild_media_group())
time.sleep(60)
def graceful_fofb():
logger.info("Starting graceful FOFB power-off/on")
for i in range(1):
# controller poeroff
status = ctrl_obj_0.power_off()
#assert(status == True)
time.sleep(60)
# controller poweron
status = ctrl_obj_0.power_on()
#assert(status == True)
time.sleep(150)
# poweroff/on another controller
status = ctrl_obj_1.power_off()
#assert(status == True)
time.sleep(60)
# poweroff/on another controller
status = ctrl_obj_1.power_on()
#assert(status == True)
time.sleep(60)
def resize_vol(vol_list):
for vol in vol_list:
logger.info(" vol size before resize operation is : %s"%vol.vol_size)
status = host.get_vol_size(vol)
print status
# vol growing by 100G
logger.info("follwing vol is getting grown by 100GB : %s"%vol.name)
status = vol.volume_grow()
assert(status == True)
## rescan the vol on the host
status = host.get_added_vol_capacity(vol)
#printing new size of the vol
logger.info(" vol size after resize operation is : %s"%vol.vol_size)
status = host.get_vol_size(vol)
print status
#start FIO thread here
temp_file_path = "%s/tmp/run_fio.%s"%(env['TESTDIR'] ,os.getpid())
cmd = "touch %s "%temp_file_path
run(cmd)
proc=multiprocessing.Process(target=fio.infinite_fio,args=(vol_list, host, temp_file_path))
proc.start()
logger.info('Fio process id code %s'%proc.pid)
sleep_p(60)
if proc.is_alive():
sleep_p(60)
else:
logger.error('Fio process failed %s'%proc.pid)
assert(proc.exitcode==0)
try:
#1# now starting graceful FOFB operations
logger.info("Starting graceful FOFB power-off/on")
if proc.is_alive():
graceful_fofb()
## take snapshot and clones here
#2# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild
if proc.is_alive():
rebuilds_func()
#3# now starting graceful FOFB operations
logger.info("Starting graceful FOFB power-off/on")
if proc.is_alive():
graceful_fofb()
#00# take the clone and overwrite on it <<< i will implement this section after vol resize loop
#4# grow vol and do the stuffs
if proc.is_alive():
resize_vol(vol_list)
#5# Now again do the FOFB operations
logger.info("Starting graceful FOFB power-off/on")
if proc.is_alive():
graceful_fofb()
#6# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild
if proc.is_alive():
rebuilds_func()
#7# Start FOFB
if proc.is_alive():
graceful_fofb()
#8# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild
if proc.is_alive():
rebuilds_func()
except Exception as E:
logger.error('Stack trace for failure \n%s\n'%traceback.format_exc())
raise E
finally:
cmd = "rm -f %s "%temp_file_path
run(cmd)
if proc.is_alive():
proc.join()
code = proc.exitcode
if code != 0 :
logger.error('Fio failed on volumes ')
assert(code == 0 )
def pds_rollback_with_rebuild_20():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
# assign the vol
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
# powring off two drives
mediaList = media_group.get_media_group_disk()
media = [mediaList[0]] + [mediaList[1]]
logger.info("Powering OFF drive : %s"%media)
status = power_off_drive([media_group.md_disk[1],media_group.md_disk[0]],pvlLibHandle)
# write on the voluem
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
#Taking snaphost
logger.info("Taking snapshot :%s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
# poweron the drives
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start rebuild
logger.info("Start rebuild 2>0")
status=media_group.synchronous_rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# unassign the vol
status = vol.unassign_volume()
assert(status == True)
# restore the volumes using s2
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
# connect the vol & verify the data
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
logger.info("connecting the vol")
status = host.connect_volume(vol)
assert(status==0)
logger.info("Reading from the raw device")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_read([vol], host, kwargs)
assert (status==0)
# delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
# disconnect the vol to run teardown
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def IO_sanity_func(volType="online", funcname="pds_online_vol_IO_sanity"):
log = PvltLogger(funcname, 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
def io_patterns():
io_pattern = [
'0x5aef', '0xa1e', '0xb1', '0xfda789', '0xfefee', '0xaaaa',
'0xffff', '0xaee4', '0xbebe1234', '0xfff1aaa1'
]
bs = [
"4k", "8k", "16k", "32k", "64k", "128k", "256k", "512k", "1M", "2M"
]
#io_type = ["seq_write:seq_read","rand_write:rand_read",]
## IO operations along with dif pattern
for pattern, block_size in zip(io_pattern, bs):
logger.info("Starting IO on all the volumes")
kwargs = {
'offset': "20k",
'size': '70%',
"verify_pattern": pattern,
"verify_interval": 4096,
"do_verify": 1,
"bs": block_size
}
logger.info("Starting Seq_write")
# start write
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting Seq_read")
# start read with the same pattern
status, response = fio.seq_read(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting rand_write")
# start rand_write & read
status, response = fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting rand_read")
status, response = fio.rand_read(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting Seq_read_write")
# seq_read_write
status, response = fio.seq_read_write(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting rand_read_write")
# rand_read_write
status, response = fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# starting multijob FIO load but without verification
logger.info(
"Starting multithreaded write without verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '80%',
"numjobs": "8",
"bsrange": "4k-2M"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# Starting mutithreaded write with verification
logger.info(
"Starting mutithreaded-write with verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '100%',
"verify_pattern": "0xFFFF",
"verify_interval": 4096,
"do_verify": 1,
"numjobs": "1",
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5",
"multithread_verification": "1"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
if volType == "online":
io_patterns()
#assert(media_group.get_media_group_state() == 2)
#for vol in vol_list:
#state = vol.get_vol_state()
#assert('Online' in state)
if volType == "degraded":
# Power off one drive
media = media_group.md_disk[0]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
#assert(media_group.get_media_group_state() == 6)
#for vol in vol_list:
#state = vol.get_vol_state()
#assert('Online, degraded' in state)
# starting io
io_patterns()
# poweron the drive again
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
if volType == "critical":
# poweroff 2 drive
media = [media_group.md_disk[0]] + [media_group.md_disk[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
#assert(media_group.get_media_group_state() == 7)
#for vol in vol_list:
#state = vol.get_vol_state()
#assert('Online, critical' in state)
# starting io
io_patterns()
# Poweron one drive
media = [media_group.md_disk[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
media = [media_group.md_disk[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
def pds_vol_restore_sanity():
host = HostOps(HOST[0])
fio = FioUtils()
# give function name as log file name
log = PvltLogger("Volume_restore", 'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
## velidation for prerequisits
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
sys.exit()
if VOL_CNT > 1:
logger.error(
'Update the configuration As this test is currently written for 1 vol only'
)
sys.exit()
md_type = MEDIA_GROUP_TYPE
## creating mediaGroup
logger.info("Creating a MediaGroup")
media_group = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=True)
assert (media_group != None)
## creaing volumes
logger.info("Creating voluems")
#vol_list = []
volname = VOL_PREFIX
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=True)
assert (vol != None)
#Creating a snapshot
snap_name = "s1"
logger.info("Creating a snapshot : %s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol, create=True)
assert (snapshot != None)
################################ 1 Vol restore for unassigned volume
# Restore the vol using its snapshot
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
#print("1 Vol restore for unassigned volume: PASS")
################################ 2 Vol restore after assign volume
ports = get_two_port_diff_ctlr(ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
#print("2 Vol restore after assign volume:PASS")
################################## delete s1 and its child snapshot
# Need to write forensic snapshots deletion code
################################## 3. Vol restore after 5% IO on the vol
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
#start fio on all the volumes
logger.info("Starting write on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# Taking snaphost
snap_name = "s2"
logger.info("Taking snapshot :%s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol, create=True)
assert (snapshot != None)
# Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {
'offset': "45g",
'size': '5g',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.rand_read_write([vol], host, kwargs)
assert (status == 0)
# disconnect the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore using latest snapshot
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# cheking data integrity by rading the volumes data
# connecting the vol
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
assert (status == 0)
print("3. Vol restore after 5% IO on the vol:PASS")
################################ 4. Vol restore for degraded volumes
# Power off one drive
mediaList = media_group.get_media_group_disk()
media = [mediaList[0]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore the volumes using s2
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
print("4. Vol restore for degraded volumes:PASS")
################################ 5. Vol restore for critical voluems
# poweroff one more drive
media = [mediaList[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore the volumes using s2
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
print("5. Vol restore for critical voluems:PASS")
################################ 6. Vol restore with rebuild (2>1)
# Poweron one drive and start the rebuild
media = [mediaList[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(120)
logger.info("Start rebuild 2>1")
status = media_group.rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore the volumes using s2
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
print("6. Vol restore with rebuild (2>1):PASS")
################################ 7. Vol restore with rebuild (1>0)
media = [mediaList[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(120)
logger.info("Start rebuild 1>0")
status = media_group.rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore the volumes using s2
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
print("7. Vol restore with rebuild (1>0):PASS")
################################ 8. Vol restore with rebuild (2>0)
media = [media_group.md_disk[1], media_group.md_disk[0]]
logger.info("Powering OFF drive : %s" % media)
status = power_off_drive([media_group.md_disk[1], media_group.md_disk[0]],
pvlLibHandle)
time.sleep(60)
logger.info("Powering ON drive : %s" % media)
status = power_on_drive([media_group.md_disk[1], media_group.md_disk[0]],
pvlLibHandle)
time.sleep(60)
# start rebuild
logger.info("Start rebuild 2>0")
status = media_group.rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore the volumes using s2
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
print("8. Vol restore with rebuild (2>0):PASS")
################################ 9. Vol restore with clone is connected to the host
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write([vol], host, kwargs)
assert (status == 0)
# create clone
clone_list = []
clone_name = str(snap_name) + str("_c1")
logger.info("Creating clone name : %s" % clone_name)
cloneObj = CloneOps(clone_name, snapshot, create=True)
clone_list.append(cloneObj)
# assign clone
logger.info("Assigning clone %s to contollers" % clone_name)
assign = cloneObj.assign_clone(ports, hostnqn=[])
# connect clone to the host
logger.info("Connecting clone %s to the host" % clone_name)
status = host.connect_volume(cloneObj)
time.sleep(10)
assert (status == 0)
# start IO on the clone
logger.info("Staring write IO to the clone %s" % clone_name)
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(clone_list, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
assert (status == 0)
# restore the volumes using s2
#status = snapshot.vol_restore()
#assert(status == True)
#time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = host.connect_volume(vol)
time.sleep(10)
assert (status == 0)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read([vol], host, kwargs)
print("9. Vol restore with clone is connected to the host:PASS")
def pds_vol_restore_sanity():
host = HostOps(HOST[0])
fio = FioUtils()
# give function name as log file name
log = PvltLogger("Volume_restore", 'INFO')
logger = log.logger
name = MEDIA_GROUP_NAME
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
## creating mediaGroup
drv_cnt = get_active_drive_count(ZONE, pvlLibHandle)
logger.info(ZONE)
logger.info(drv_cnt)
if drv_cnt < 9:
logger.error('Not sufficient drives active drive count=%s' % drv_cnt)
#assert(drv_cnt > 8 )
# take it from config
md_type = 'RAID-0 (18+0)' if drv_cnt == 18 else 'RAID-6 (7+2)'
logger.info("Creating a MediaGroup")
md = MediaGroupOps(pvlLibHandle, name, ZONE, md_type, create=False)
media_group = md
assert (md != None)
## creaing volumes
logger.info("Creating voluems")
vol_list = []
for i in range(1):
# dont use + fro contact use %
volname = str(VOL_PREFIX) + str("_") + str(i)
volname = '%s_%s' % (VOL_PREFIX, i)
vol = VolumeOps(md=media_group,
name=volname,
size=VOL_SIZE,
vol_res=VOL_RES,
flavor=FLAVOR,
create=False)
vol_list.append(vol)
#Creating a snapshot
snap_name = "s1"
logger.info("Creating a snapshot : %s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol_list[0], create=False)
assert (snapshot != None)
################################ 1 Vol restore for unassigned volume
# Restore the vol using its snapshot
snap_id = snapshot.snap_id
print snap_id
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
print("1 Vol restore for unassigned volume: PASS")
################################ 2 Vol restore after assign volume
ports = get_two_port_diff_ctlr(ZONE)
logger.info("Assigning vols ")
assign = [v.assign_volume(ports, hostnqn=[]) for v in vol_list]
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
print("2 Vol restore after assign volume:PASS")
################################## delete s1 and its child snapshot
#vol,snap_list,clone_list = list_existing_obj(md.name)
#for snap in snap_list:
#snap_to_delete = SnapShotOps(snap,volobj=vol_list[0])
#if snap != snap_name:
#status = snap_to_delete.delete_snapshot
#assert(status != None)
#result = pvlLibHandle.storage.get_snapshot
#print result
#print json.dumps(result,indent=4)
################################## 3. Vol restore after 5% IO on the vol
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
#stert fio on all the volumes
logger.info("Starting write on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
# Taking snaphost
snap_name = "s2"
logger.info("Taking snapshot :%s" % snap_name)
snapshot = SnapShotOps(snap_name, volobj=vol_list[0], create=True)
assert (snapshot != None)
# Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {
'offset': "45g",
'size': '5g',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# disconnect the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
#assert(status==0)
# restore using latest snapshot
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
snap_id = snapshot.snap_id
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# cheking data integrity by rading the volumes data
# connecting the vol
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
assert (status == 0)
print("3. Vol restore after 5% IO on the vol:PASS")
################################ 4. Vol restore for degraded volumes
# Power off one drive
mediaList = media_group.get_media_group_disk()
media = [mediaList[0]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
# restore the volumes using s2
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
print("4. Vol restore for degraded volumes:PASS")
################################ 5. Vol restore for critical voluems
# poweroff one more drive
media = [mediaList[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
# restore the volumes using s2
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
print("5. Vol restore for critical voluems:PASS")
################################ 6. Vol restore with rebuild (2>1)
# Poweron one drive and start the rebuild
media = [mediaList[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(120)
logger.info("Start rebuild 2>1")
status = media_group.rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != None)
time.sleep(5)
# restore the volumes using s2
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
print("6. Vol restore with rebuild (2>1):PASS")
################################ 7. Vol restore with rebuild (1>0)
media = [mediaList[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(120)
logger.info("Start rebuild 1>0")
status = media_group.rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
# restore the volumes using s2
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
print("7. Vol restore with rebuild (1>0):PASS")
################################ 8. Vol restore with rebuild (2>0)
media = [mediaList[0], mediaList[1]]
logger.info("Powering OFF drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(120)
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(120)
# start rebuild
logger.info("Start rebuild 2>0")
status = media_group.rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
# restore the volumes using s2
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
print("8. Vol restore with rebuild (2>0):PASS")
################################ 9. Vol restore with clone is connected to the host
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# create clone
clone_name = str(snap_name) + str("_c1")
cloneObj = CloneOps(clone_name, snapshot, create=True)
# assign clone
assign = cloneobj.assign_clone(ports, hostnqn=[])
# connect clone to the host
status = host.connect_volume(cloneObj)
assert (status != 0)
time.sleep(5)
# start IO on the clone
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_write(cloneObj, host, kwargs)
assert (status == 0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = [host.disconnect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
# restore the volumes using s2
logger.info("Running vol_restore using following snaphost : %s" %
snap_name)
status = pvlLibHandle.system.generic_api(
'POST', '/storage/snapshots/%s/restore' % snap_id)
assert (status != 200)
time.sleep(60)
# connect the vol & verify the data
logger.info("connecting the vol")
status = [host.connect_volume(v) for v in vol_list]
assert (status != 0)
time.sleep(5)
logger.info("Reading from the raw device")
kwargs = {
'offset': "0",
'size': '5%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1
}
status, response = fio.seq_read(vol_list, host, kwargs)
print("9. Vol restore with clone is connected to the host:PASS")