def pds_rollback_assign_vol():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
#Creating a snapshot
logger.info("Creating a snapshot : %s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# run vol rollback
try:
status = vol.vol_rollback(snapshot,backup="true")
#assert(vol.vol_rollback(snapshot,backup="true") == False)
if(status == False):
raise Exception
except Exception:
logger.info("Rollback operations can`t be perfomed on Assigned vol, Please-Unassign the vol and try again ")
def pds_rollback_with_clone():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
# assigning vol
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connect vol
logger.info("connecting vol to the host")
status = host.connect_volume(vol)
assert(status == 0)
# write on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
# Creating a snapshot
logger.info("Creating a snapshot : %s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
# create clone
clone_name = str(SNAP_PREFIX)+str("_c1")
logger.info("Creating clone name : %s"%clone_name)
cloneObj = CloneOps(clone_name,snapshot,create=True)
# assign clone
logger.info("Assigning clone %s to contollers"%cloneObj.name)
assign = cloneObj.assign_clone(ports,hostnqn=[])
# connect clone to the host
logger.info("Connecting clone %s to the host"%cloneObj.name)
status = host.connect_volume(cloneObj)
assert(status==0)
# start IO on the clone
logger.info("Staring write IO to the clone %s"%clone_name)
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([cloneObj], host, kwargs)
assert (status==0)
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# unassign the vol
status = vol.unassign_volume()
assert(status == True)
# restore the volumes using s2
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
# connect the vol & verify the data
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
logger.info("connecting the vol")
status = host.connect_volume(vol)
assert(status == 0)
logger.info("Reading from the raw device")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_read([vol], host, kwargs)
assert (status==0)
# delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
# disconnect the vol to run teardown
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# disconnet the clone
logger.info("disconnecting clone : %s"%cloneObj.name)
status = host.disconnect_volume(cloneObj)
time.sleep(10)
def pds_rollback_with_rebuild_20():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
# assign the vol
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
# powring off two drives
mediaList = media_group.get_media_group_disk()
media = [mediaList[0]] + [mediaList[1]]
logger.info("Powering OFF drive : %s"%media)
status = power_off_drive([media_group.md_disk[1],media_group.md_disk[0]],pvlLibHandle)
# write on the voluem
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
#Taking snaphost
logger.info("Taking snapshot :%s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
# poweron the drives
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start rebuild
logger.info("Start rebuild 2>0")
status=media_group.synchronous_rebuild_media_group()
# overwrite on the volume
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status==0)
# disconenct the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# unassign the vol
status = vol.unassign_volume()
assert(status == True)
# restore the volumes using s2
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
# connect the vol & verify the data
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
logger.info("connecting the vol")
status = host.connect_volume(vol)
assert(status==0)
logger.info("Reading from the raw device")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_read([vol], host, kwargs)
assert (status==0)
# delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
# disconnect the vol to run teardown
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_rollback_with_IO():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
#calling base function to populate the objects
media_group,vol = base_function(pvlLibHandle,logger)
# assign the vol
ports= get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
#start fio on all the volumes
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write([vol], host, kwargs)
assert (status == 0)
#Taking snaphost
logger.info("Taking snapshot :%s"%SNAP_PREFIX)
snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True)
assert(snapshot != None)
#Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {'offset':"45g",'size':'5g',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.rand_write([vol], host, kwargs)
assert (status == 0)
#disconnect the vol
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
# unassign the vol
status = vol.unassign_volume()
assert(status == True)
#restore using latest snapshot
status = vol.vol_rollback(snapshot,backup="true")
assert(status == True)
#cheking data integrity by rading the volumes data
#assign and then connecting the vol
logger.info("Assigning vols ")
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
logger.info("connecting the vol")
status = host.connect_volume(vol)
assert(status == 0)
logger.info("Reading from the raw device")
kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_read([vol], host, kwargs)
assert (status==0)
#delete forensic snapshot
status = snapshot.delete_all_forensic_snap()
assert(status == True)
# disconnect the vol to run teardown
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_one_drive_rebuild():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
#calling base function to populate the objects
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
#start FIO thread here
temp_file_path = "%s/tmp/run_fio.%s" % (env['TESTDIR'], os.getpid())
cmd = "touch %s " % temp_file_path
run(cmd)
proc = multiprocessing.Process(target=fio.infinite_fio,
args=(vol_list, host, temp_file_path))
proc.start()
logger.info('Fio process id code %s' % proc.pid)
sleep_p(60)
if proc.is_alive():
sleep_p(60)
else:
logger.error('Fio process failed %s' % proc.pid)
assert (proc.exitcode == 0)
try:
# Power off one drive
mediaList = media_group.get_media_group_disk()
for i in range(1):
media = [mediaList[i]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(30)
# poweron the drive again
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
if proc.is_alive():
# start async-rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
except Exception as E:
logger.error('Stack trace for failure \n%s\n' % traceback.format_exc())
raise E
finally:
cmd = "rm -f %s " % temp_file_path
run(cmd)
if proc.is_alive():
proc.join()
code = proc.exitcode
if code != 0:
logger.error('Fio failed on volumes ')
assert (code == 0)
def IO_sanity_func(volType="online", funcname="pds_online_vol_IO_sanity"):
log = PvltLogger(funcname, 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
def io_patterns():
logger.info("Starting fresh write on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '30%',
"verify_pattern": "0xABCD",
"verify_interval": 4096,
"do_verify": 1,
"bs": "64k"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# starting overwrite with diff pattern
logger.info("Starting overwrite on the vol-seq_write")
kwargs = {
'offset': "0",
'size': '50%',
"verify_pattern": "0xFFFF",
"verify_interval": 4096,
"do_verify": 1,
"bs": "64k"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# starting multijob FIO load but without verification
logger.info(
"Starting multithreaded write without verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '70%',
"numjobs": "8",
"bsrange": "4k-2M"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# Starting mutithreaded write with verification
logger.info(
"Starting mutithreaded-write with verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '100%',
"verify_pattern": "0xFFFF",
"verify_interval": 4096,
"do_verify": 1,
"numjobs": "1",
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5",
"multithread_verification": "1"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
if volType == "online":
io_patterns()
assert (media_group.get_media_group_state() == 2)
for vol in vol_list:
state = vol.get_vol_state()
assert ('Online' in state)
if volType == "degraded":
# Power off one drive
media = media_group.md_disk[0]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
assert (media_group.get_media_group_state() == 6)
for vol in vol_list:
state = vol.get_vol_state()
assert ('Online, degraded' in state)
# starting io
io_patterns()
# poweron the drive again
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start async-rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
time.sleep(10)
assert (media_group.get_media_group_state() == 2)
if volType == "critical":
# poweroff 2 drive
media = [media_group.md_disk[0]] + [media_group.md_disk[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
assert (media_group.get_media_group_state() == 7)
for vol in vol_list:
state = vol.get_vol_state()
assert ('Online, critical' in state)
# starting io
io_patterns()
# Poweron one drive
media = [media_group.md_disk[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
media = [media_group.md_disk[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start async-rebuild
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
assert (media_group.get_media_group_state() == 2)
def IO_sanity_func(volType="online", funcname="pds_online_vol_IO_sanity"):
log = PvltLogger(funcname, 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
def io_patterns():
io_pattern = [
'0x5aef', '0xa1e', '0xb1', '0xfda789', '0xfefee', '0xaaaa',
'0xffff', '0xaee4', '0xbebe1234', '0xfff1aaa1'
]
bs = [
"4k", "8k", "16k", "32k", "64k", "128k", "256k", "512k", "1M", "2M"
]
#io_type = ["seq_write:seq_read","rand_write:rand_read",]
## IO operations along with dif pattern
for pattern, block_size in zip(io_pattern, bs):
logger.info("Starting IO on all the volumes")
kwargs = {
'offset': "20k",
'size': '70%',
"verify_pattern": pattern,
"verify_interval": 4096,
"do_verify": 1,
"bs": block_size
}
logger.info("Starting Seq_write")
# start write
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting Seq_read")
# start read with the same pattern
status, response = fio.seq_read(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting rand_write")
# start rand_write & read
status, response = fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting rand_read")
status, response = fio.rand_read(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting Seq_read_write")
# seq_read_write
status, response = fio.seq_read_write(vol_list, host, kwargs)
assert (status == 0)
logger.info("Starting rand_read_write")
# rand_read_write
status, response = fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# starting multijob FIO load but without verification
logger.info(
"Starting multithreaded write without verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '80%',
"numjobs": "8",
"bsrange": "4k-2M"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# Starting mutithreaded write with verification
logger.info(
"Starting mutithreaded-write with verification on the vol-seq_write"
)
kwargs = {
'offset': "0",
'size': '100%',
"verify_pattern": "0xFFFF",
"verify_interval": 4096,
"do_verify": 1,
"numjobs": "1",
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5",
"multithread_verification": "1"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
if volType == "online":
io_patterns()
#assert(media_group.get_media_group_state() == 2)
#for vol in vol_list:
#state = vol.get_vol_state()
#assert('Online' in state)
if volType == "degraded":
# Power off one drive
media = media_group.md_disk[0]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
#assert(media_group.get_media_group_state() == 6)
#for vol in vol_list:
#state = vol.get_vol_state()
#assert('Online, degraded' in state)
# starting io
io_patterns()
# poweron the drive again
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
if volType == "critical":
# poweroff 2 drive
media = [media_group.md_disk[0]] + [media_group.md_disk[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
time.sleep(5)
#assert(media_group.get_media_group_state() == 7)
#for vol in vol_list:
#state = vol.get_vol_state()
#assert('Online, critical' in state)
# starting io
io_patterns()
# Poweron one drive
media = [media_group.md_disk[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
media = [media_group.md_disk[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
def pds_ctrl_fofb():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
ctrl_id = list(get_all_ids_in_zone(ZONE))
ctrl_obj_0 = ControllerOps(pvlLibHandle,ctrl_id[0],"rdma")
ctrl_obj_1 = ControllerOps(pvlLibHandle,ctrl_id[1],"rdma")
#calling base function to populate the objects
media_group,vol_list = base_function(pvlLibHandle,logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
def startIO():
p1 = multiprocessing.current_process()
logger.info("Starting : %s : %s"%(p1.name,p1.pid))
print "Starting :",p1.name,p1.pid
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'100%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
status,response=fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
#start fio on all the volumes
p1 = multiprocessing.Process(name="StartFio",target=startIO)
p1.start()
#logger.info("Starting write on the vol-seq_write")
#kwargs = {'offset':"0",'size':'1%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1}
#status,response=fio.seq_write(vol_list, host, kwargs)
#assert (status == 0)
# start FOFB with fresh write
logger.info("Start controler poweroff/on")
for i in range(10):
# controller poeroff
status = ctrl_obj_0.power_off()
#assert(status == True)
time.sleep(60)
# controller poweron
status = ctrl_obj_0.power_on()
#assert(status == True)
time.sleep(150)
# poweroff/on another controller
status = ctrl_obj_1.power_off()
#assert(status == True)
time.sleep(60)
# poweroff/on another controller
status = ctrl_obj_1.power_on()
#assert(status == True)
time.sleep(60)
p1.join()
#disconnect the vol
for vol in vol_list:
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_online_vol_IO_sanity():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3],'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS)
ctrl_id = list(get_all_ids_in_zone(ZONE))
ctrl_obj_0 = ControllerOps(pvlLibHandle,ctrl_id[0],"rdma")
ctrl_obj_1 = ControllerOps(pvlLibHandle,ctrl_id[1],"rdma")
#calling base function to populate the objects
media_group,vol_list = base_function(pvlLibHandle,logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports,hostnqn=[])
assert(status == True)
# connet to the host
status = host.connect_volume(vol)
assert(status == 0)
def startIO():
p1 = multiprocessing.current_process()
logger.info("Starting : %s : %s"%(p1.name,p1.pid))
print "Starting :",p1.name,p1.pid
logger.info("Starting write on the vol-seq_write")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1,
"bs":"64k"}
status,response=fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# # start FIO thread here
#
# temp_file_path = "%s/tmp/run_fio.%s"%(env['TESTDIR'] ,os.getpid())
# cmd = "touch %s "%temp_file_path
# run(cmd)
# proc=multiprocessing.Process(target=fio.infinite_fio,args=(vol_list, host, temp_file_path))
# proc.start()
# logger.info('Fio process id code %s'%proc.pid)
# sleep_p(60)
# if proc.is_alive():
# sleep_p(60)
# else:
# logger.error('Fio process failed %s'%proc.pid)
# assert(proc.exitcode==0)
#
# cmd = "rm -f %s "%temp_file_path
# run(cmd)
# if proc.is_alive():
# proc.join()
# print"fio run finished"
# ## start graceful FOFB while the IOs are running
# #start fio on all the volumes
# p1 = multiprocessing.Process(name="StartFio",target=startIO)
# logger.info("starting IOs just in parallel with FOFB operations")
# p1.start()
# time.sleep(5)
def rebuilds_func():
# starting one driver rebuild
# Power off one drive
mediaList = media_group.get_media_group_disk()
for i in range(1):
media = [mediaList[i]]
logger.info("Powering off drive : %s"%media)
status = power_off_drive(media,pvlLibHandle)
time.sleep(30)
# poweron the same drive
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
#status=media_group.asynchronous_rebuild_media_group()
assert(media_group.synchronous_rebuild_media_group())
time.sleep(60)
## starting 2 drive rebuild
for i in range(10):
media = [mediaList[0]] + [mediaList[1]]
logger.info("Powering off drive : %s"%media)
status = power_off_drive(media,pvlLibHandle)
time.sleep(60)
# poweron the drive again
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start async-rebuild with io
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
#status=media_group.asynchronous_rebuild_media_group()
assert(media_group.synchronous_rebuild_media_group())
time.sleep(60)
## 2>1 && 1>0 rebuild
#mediaList = media_group.get_media_group_disk()
media = [mediaList[2]] + [mediaList[3]]
logger.info("Powering off drive : %s"%media)
status = power_off_drive(media,pvlLibHandle)
time.sleep(60)
# Poweron one drive and start the rebuild
media = [mediaList[2]]
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start rebuild with IO
logger.info("Start rebuild 2>1")
#status=media_group.synchronous_rebuild_media_group()
status=media_group.asynchronous_rebuild_media_group()
time.sleep(60)
if proc.is_alive():
# Poweron second drive and start the rebuild
media = [mediaList[3]]
logger.info("Powering ON drive : %s"%media)
status = power_on_drive(media,pvlLibHandle)
time.sleep(60)
# start async-rebuild
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
#status=media_group.asynchronous_rebuild_media_group()
assert(media_group.synchronous_rebuild_media_group())
time.sleep(60)
def graceful_fofb():
logger.info("Starting graceful FOFB power-off/on")
for i in range(1):
# controller poeroff
status = ctrl_obj_0.power_off()
#assert(status == True)
time.sleep(60)
# controller poweron
status = ctrl_obj_0.power_on()
#assert(status == True)
time.sleep(150)
# poweroff/on another controller
status = ctrl_obj_1.power_off()
#assert(status == True)
time.sleep(60)
# poweroff/on another controller
status = ctrl_obj_1.power_on()
#assert(status == True)
time.sleep(60)
def resize_vol(vol_list):
for vol in vol_list:
logger.info(" vol size before resize operation is : %s"%vol.vol_size)
status = host.get_vol_size(vol)
print status
# vol growing by 100G
logger.info("follwing vol is getting grown by 100GB : %s"%vol.name)
status = vol.volume_grow()
assert(status == True)
## rescan the vol on the host
status = host.get_added_vol_capacity(vol)
#printing new size of the vol
logger.info(" vol size after resize operation is : %s"%vol.vol_size)
status = host.get_vol_size(vol)
print status
#start FIO thread here
temp_file_path = "%s/tmp/run_fio.%s"%(env['TESTDIR'] ,os.getpid())
cmd = "touch %s "%temp_file_path
run(cmd)
proc=multiprocessing.Process(target=fio.infinite_fio,args=(vol_list, host, temp_file_path))
proc.start()
logger.info('Fio process id code %s'%proc.pid)
sleep_p(60)
if proc.is_alive():
sleep_p(60)
else:
logger.error('Fio process failed %s'%proc.pid)
assert(proc.exitcode==0)
try:
#1# now starting graceful FOFB operations
logger.info("Starting graceful FOFB power-off/on")
if proc.is_alive():
graceful_fofb()
## take snapshot and clones here
#2# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild
if proc.is_alive():
rebuilds_func()
#3# now starting graceful FOFB operations
logger.info("Starting graceful FOFB power-off/on")
if proc.is_alive():
graceful_fofb()
#00# take the clone and overwrite on it <<< i will implement this section after vol resize loop
#4# grow vol and do the stuffs
if proc.is_alive():
resize_vol(vol_list)
#5# Now again do the FOFB operations
logger.info("Starting graceful FOFB power-off/on")
if proc.is_alive():
graceful_fofb()
#6# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild
if proc.is_alive():
rebuilds_func()
#7# Start FOFB
if proc.is_alive():
graceful_fofb()
#8# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild
if proc.is_alive():
rebuilds_func()
except Exception as E:
logger.error('Stack trace for failure \n%s\n'%traceback.format_exc())
raise E
finally:
cmd = "rm -f %s "%temp_file_path
run(cmd)
if proc.is_alive():
proc.join()
code = proc.exitcode
if code != 0 :
logger.error('Fio failed on volumes ')
assert(code == 0 )
def pds_online_vol_IO_sanity():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
#calling base function to populate the objects
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
for i in range(1):
snap_list = []
#star fresh write on all the volumes
logger.info("Starting fresh write on the vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with same pattern
logger.info("Starting over-write with same pattern -vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
'''
# start overwrite with diff pattern (rw=rw,splitbs)
logger.info("Starting over-write with diff pattern -rw=rw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDffff","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.seq_read_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with diff pattern (rw=randrw,splitbs)
logger.info("Starting over-write with diff pattern -rw=randrw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# randwrite on online vol
logger.info("Starting over-write with diff pattern -rw=randwrite")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
for vol in vol_list:
#Taking snaphost
snap_name = str(vol.name) + str("_%s"%SNAP_PREFIX)
logger.info("Taking snapshot :%s"%snap_name)
snapshot = SnapShotOps(snap_name,volobj=vol,create=True)
assert(snapshot != None)
snap_list.append(snapshot)
#Write on remaning space of the volumes
logger.info("Starting write on other space of the vol ")
kwargs = {'offset':"0",'size':'1%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
# delete snapshot
for snap in snap_list:
status = snap.delete_snapshot()
assert (status == True)
time.sleep(600)
'''
#disconnect the vol
for vol in vol_list:
logger.info("disconnecting the vol from the host")
status = host.disconnect_volume(vol)
time.sleep(10)
def pds_critical_vol_IO_sanity():
#objects initialization & Authentication
log = PvltLogger(inspect.stack()[0][3], 'INFO')
logger = log.logger
host = HostOps(HOST[0])
fio = FioUtils()
pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger)
result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS)
#calling base function to populate the objects
media_group, vol_list = base_function(pvlLibHandle, logger)
# assigning vol
ports = get_volume_network_map(pvlLibHandle, ZONE)
logger.info("Assigning vols ")
for vol in vol_list:
status = vol.assign_volume(ports, hostnqn=[])
assert (status == True)
# connet to the host
status = host.connect_volume(vol)
assert (status == 0)
# poweroff 2 drive
mediaList = media_group.get_media_group_disk()
media = [mediaList[0]] + [mediaList[1]]
logger.info("Powering off drive : %s" % media)
status = power_off_drive(media, pvlLibHandle)
for i in range(1):
snap_list = []
#star fresh write on all the volumes
logger.info("Starting fresh write on the vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with same pattern
logger.info("Starting over-write with same pattern -vol-seq_write")
kwargs = {
'offset':
"0",
'size':
'50%',
"verify_pattern":
"0xABCD",
"verify_interval":
4096,
"do_verify":
1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.seq_write(vol_list, host, kwargs)
assert (status == 0)
'''
# start overwrite with diff pattern (rw=rw,splitbs)
logger.info("Starting over-write with diff pattern -rw=rw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDffff","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.seq_read_write(vol_list, host, kwargs)
assert (status == 0)
# start overwrite with diff pattern (rw=randrw,splitbs)
logger.info("Starting over-write with diff pattern -rw=randrw")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
# randwrite on online vol
logger.info("Starting over-write with diff pattern -rw=randwrite")
kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
"bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
status,response=fio.rand_write(vol_list, host, kwargs)
assert (status == 0)
'''
# Poweron one drive and start the rebuild
media = [mediaList[0]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start rebuild with IO
logger.info("Start rebuild 2>1")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
time.sleep(60)
# starting IO load
#logger.info("Starting over-write with diff pattern -rw=randrw")
#kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1,
# "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"}
#status,response=fio.rand_read_write(vol_list, host, kwargs)
#assert (status == 0)
# Poweron second drive and start the rebuild
media = [mediaList[1]]
logger.info("Powering ON drive : %s" % media)
status = power_on_drive(media, pvlLibHandle)
time.sleep(60)
# start async-rebuild
logger.info("Start rebuild 1>0")
#status=media_group.synchronous_rebuild_media_group()
status = media_group.asynchronous_rebuild_media_group()
# starting IO load
logger.info("Starting over-write with diff pattern -rw=randrw")
kwargs = {
'offset': "0",
'size': '50%',
"verify_pattern": "0xABCDabcd",
"verify_interval": 4096,
"do_verify": 1,
"bssplit":
"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"
}
status, response = fio.rand_read_write(vol_list, host, kwargs)
assert (status == 0)
