def pds_rollback_assign_vol(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3],'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS) #calling base function to populate the objects media_group,vol = base_function(pvlLibHandle,logger) #Creating a snapshot logger.info("Creating a snapshot : %s"%SNAP_PREFIX) snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True) assert(snapshot != None) ports= get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) # run vol rollback try: status = vol.vol_rollback(snapshot,backup="true") #assert(vol.vol_rollback(snapshot,backup="true") == False) if(status == False): raise Exception except Exception: logger.info("Rollback operations can`t be perfomed on Assigned vol, Please-Unassign the vol and try again ")
def pds_rollback_with_clone(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3],'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS) #calling base function to populate the objects media_group,vol = base_function(pvlLibHandle,logger) # assigning vol ports= get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) # connect vol logger.info("connecting vol to the host") status = host.connect_volume(vol) assert(status == 0) # write on the volume logger.info("Starting overwrite on the vol-seq_write") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_write([vol], host, kwargs) assert (status==0) # Creating a snapshot logger.info("Creating a snapshot : %s"%SNAP_PREFIX) snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True) assert(snapshot != None) # create clone clone_name = str(SNAP_PREFIX)+str("_c1") logger.info("Creating clone name : %s"%clone_name) cloneObj = CloneOps(clone_name,snapshot,create=True) # assign clone logger.info("Assigning clone %s to contollers"%cloneObj.name) assign = cloneObj.assign_clone(ports,hostnqn=[]) # connect clone to the host logger.info("Connecting clone %s to the host"%cloneObj.name) status = host.connect_volume(cloneObj) assert(status==0) # start IO on the clone logger.info("Staring write IO to the clone %s"%clone_name) kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_write([cloneObj], host, kwargs) assert (status==0) # overwrite on the volume logger.info("Starting overwrite on the vol-seq_write") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1} status,response=fio.seq_write([vol], host, kwargs) assert (status==0) # disconenct the vol logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10) # unassign the vol status = vol.unassign_volume() assert(status == True) # restore the volumes using s2 status = vol.vol_rollback(snapshot,backup="true") assert(status == True) # connect the vol & verify the data logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) logger.info("connecting the vol") status = host.connect_volume(vol) assert(status == 0) logger.info("Reading from the raw device") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_read([vol], host, kwargs) assert (status==0) # delete forensic snapshot status = snapshot.delete_all_forensic_snap() assert(status == True) # disconnect the vol to run teardown logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10) # disconnet the clone logger.info("disconnecting clone : %s"%cloneObj.name) status = host.disconnect_volume(cloneObj) time.sleep(10)
def pds_rollback_with_rebuild_20(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3],'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS) #calling base function to populate the objects media_group,vol = base_function(pvlLibHandle,logger) # assign the vol ports= get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) # connet to the host status = host.connect_volume(vol) assert(status == 0) # powring off two drives mediaList = media_group.get_media_group_disk() media = [mediaList[0]] + [mediaList[1]] logger.info("Powering OFF drive : %s"%media) status = power_off_drive([media_group.md_disk[1],media_group.md_disk[0]],pvlLibHandle) # write on the voluem logger.info("Starting overwrite on the vol-seq_write") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_write([vol], host, kwargs) assert (status==0) #Taking snaphost logger.info("Taking snapshot :%s"%SNAP_PREFIX) snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True) assert(snapshot != None) # poweron the drives logger.info("Powering ON drive : %s"%media) status = power_on_drive(media,pvlLibHandle) time.sleep(60) # start rebuild logger.info("Start rebuild 2>0") status=media_group.synchronous_rebuild_media_group() # overwrite on the volume logger.info("Starting overwrite on the vol-seq_write") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1} status,response=fio.seq_write([vol], host, kwargs) assert (status==0) # disconenct the vol logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10) # unassign the vol status = vol.unassign_volume() assert(status == True) # restore the volumes using s2 status = vol.vol_rollback(snapshot,backup="true") assert(status == True) # connect the vol & verify the data logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) logger.info("connecting the vol") status = host.connect_volume(vol) assert(status==0) logger.info("Reading from the raw device") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_read([vol], host, kwargs) assert (status==0) # delete forensic snapshot status = snapshot.delete_all_forensic_snap() assert(status == True) # disconnect the vol to run teardown logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10)
def pds_rollback_with_IO(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3],'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS) #calling base function to populate the objects media_group,vol = base_function(pvlLibHandle,logger) # assign the vol ports= get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) # connet to the host status = host.connect_volume(vol) assert(status == 0) #start fio on all the volumes logger.info("Starting write on the vol-seq_write") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_write([vol], host, kwargs) assert (status == 0) #Taking snaphost logger.info("Taking snapshot :%s"%SNAP_PREFIX) snapshot = SnapShotOps(SNAP_PREFIX,volobj=vol,create=True) assert(snapshot != None) #Write on remaning space of the volumes logger.info("Starting write on other space of the vol ") kwargs = {'offset':"45g",'size':'5g',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1} status,response=fio.rand_write([vol], host, kwargs) assert (status == 0) #disconnect the vol logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10) # unassign the vol status = vol.unassign_volume() assert(status == True) #restore using latest snapshot status = vol.vol_rollback(snapshot,backup="true") assert(status == True) #cheking data integrity by rading the volumes data #assign and then connecting the vol logger.info("Assigning vols ") status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) logger.info("connecting the vol") status = host.connect_volume(vol) assert(status == 0) logger.info("Reading from the raw device") kwargs = {'offset':"0",'size':'40%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_read([vol], host, kwargs) assert (status==0) #delete forensic snapshot status = snapshot.delete_all_forensic_snap() assert(status == True) # disconnect the vol to run teardown logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10)
def pds_one_drive_rebuild(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3], 'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS) #calling base function to populate the objects media_group, vol_list = base_function(pvlLibHandle, logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports, hostnqn=[]) assert (status == True) # connet to the host status = host.connect_volume(vol) assert (status == 0) #start FIO thread here temp_file_path = "%s/tmp/run_fio.%s" % (env['TESTDIR'], os.getpid()) cmd = "touch %s " % temp_file_path run(cmd) proc = multiprocessing.Process(target=fio.infinite_fio, args=(vol_list, host, temp_file_path)) proc.start() logger.info('Fio process id code %s' % proc.pid) sleep_p(60) if proc.is_alive(): sleep_p(60) else: logger.error('Fio process failed %s' % proc.pid) assert (proc.exitcode == 0) try: # Power off one drive mediaList = media_group.get_media_group_disk() for i in range(1): media = [mediaList[i]] logger.info("Powering off drive : %s" % media) status = power_off_drive(media, pvlLibHandle) time.sleep(30) # poweron the drive again logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) if proc.is_alive(): # start async-rebuild with io logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() status = media_group.asynchronous_rebuild_media_group() except Exception as E: logger.error('Stack trace for failure \n%s\n' % traceback.format_exc()) raise E finally: cmd = "rm -f %s " % temp_file_path run(cmd) if proc.is_alive(): proc.join() code = proc.exitcode if code != 0: logger.error('Fio failed on volumes ') assert (code == 0)
def IO_sanity_func(volType="online", funcname="pds_online_vol_IO_sanity"): log = PvltLogger(funcname, 'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS) media_group, vol_list = base_function(pvlLibHandle, logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports, hostnqn=[]) assert (status == True) # connet to the host status = host.connect_volume(vol) assert (status == 0) def io_patterns(): logger.info("Starting fresh write on the vol-seq_write") kwargs = { 'offset': "0", 'size': '30%', "verify_pattern": "0xABCD", "verify_interval": 4096, "do_verify": 1, "bs": "64k" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) # starting overwrite with diff pattern logger.info("Starting overwrite on the vol-seq_write") kwargs = { 'offset': "0", 'size': '50%', "verify_pattern": "0xFFFF", "verify_interval": 4096, "do_verify": 1, "bs": "64k" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) # starting multijob FIO load but without verification logger.info( "Starting multithreaded write without verification on the vol-seq_write" ) kwargs = { 'offset': "0", 'size': '70%', "numjobs": "8", "bsrange": "4k-2M" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) # Starting mutithreaded write with verification logger.info( "Starting mutithreaded-write with verification on the vol-seq_write" ) kwargs = { 'offset': "0", 'size': '100%', "verify_pattern": "0xFFFF", "verify_interval": 4096, "do_verify": 1, "numjobs": "1", "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5", "multithread_verification": "1" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) if volType == "online": io_patterns() assert (media_group.get_media_group_state() == 2) for vol in vol_list: state = vol.get_vol_state() assert ('Online' in state) if volType == "degraded": # Power off one drive media = media_group.md_disk[0] logger.info("Powering off drive : %s" % media) status = power_off_drive(media, pvlLibHandle) time.sleep(5) assert (media_group.get_media_group_state() == 6) for vol in vol_list: state = vol.get_vol_state() assert ('Online, degraded' in state) # starting io io_patterns() # poweron the drive again logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) # start async-rebuild with io logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() status = media_group.asynchronous_rebuild_media_group() time.sleep(10) assert (media_group.get_media_group_state() == 2) if volType == "critical": # poweroff 2 drive media = [media_group.md_disk[0]] + [media_group.md_disk[1]] logger.info("Powering off drive : %s" % media) status = power_off_drive(media, pvlLibHandle) time.sleep(5) assert (media_group.get_media_group_state() == 7) for vol in vol_list: state = vol.get_vol_state() assert ('Online, critical' in state) # starting io io_patterns() # Poweron one drive media = [media_group.md_disk[0]] logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) media = [media_group.md_disk[1]] logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) # start async-rebuild logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() status = media_group.asynchronous_rebuild_media_group() assert (media_group.get_media_group_state() == 2)
def IO_sanity_func(volType="online", funcname="pds_online_vol_IO_sanity"): log = PvltLogger(funcname, 'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS) media_group, vol_list = base_function(pvlLibHandle, logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports, hostnqn=[]) assert (status == True) # connet to the host status = host.connect_volume(vol) assert (status == 0) def io_patterns(): io_pattern = [ '0x5aef', '0xa1e', '0xb1', '0xfda789', '0xfefee', '0xaaaa', '0xffff', '0xaee4', '0xbebe1234', '0xfff1aaa1' ] bs = [ "4k", "8k", "16k", "32k", "64k", "128k", "256k", "512k", "1M", "2M" ] #io_type = ["seq_write:seq_read","rand_write:rand_read",] ## IO operations along with dif pattern for pattern, block_size in zip(io_pattern, bs): logger.info("Starting IO on all the volumes") kwargs = { 'offset': "20k", 'size': '70%', "verify_pattern": pattern, "verify_interval": 4096, "do_verify": 1, "bs": block_size } logger.info("Starting Seq_write") # start write status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) logger.info("Starting Seq_read") # start read with the same pattern status, response = fio.seq_read(vol_list, host, kwargs) assert (status == 0) logger.info("Starting rand_write") # start rand_write & read status, response = fio.rand_write(vol_list, host, kwargs) assert (status == 0) logger.info("Starting rand_read") status, response = fio.rand_read(vol_list, host, kwargs) assert (status == 0) logger.info("Starting Seq_read_write") # seq_read_write status, response = fio.seq_read_write(vol_list, host, kwargs) assert (status == 0) logger.info("Starting rand_read_write") # rand_read_write status, response = fio.rand_read_write(vol_list, host, kwargs) assert (status == 0) # starting multijob FIO load but without verification logger.info( "Starting multithreaded write without verification on the vol-seq_write" ) kwargs = { 'offset': "0", 'size': '80%', "numjobs": "8", "bsrange": "4k-2M" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) # Starting mutithreaded write with verification logger.info( "Starting mutithreaded-write with verification on the vol-seq_write" ) kwargs = { 'offset': "0", 'size': '100%', "verify_pattern": "0xFFFF", "verify_interval": 4096, "do_verify": 1, "numjobs": "1", "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5", "multithread_verification": "1" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) if volType == "online": io_patterns() #assert(media_group.get_media_group_state() == 2) #for vol in vol_list: #state = vol.get_vol_state() #assert('Online' in state) if volType == "degraded": # Power off one drive media = media_group.md_disk[0] logger.info("Powering off drive : %s" % media) status = power_off_drive(media, pvlLibHandle) time.sleep(5) #assert(media_group.get_media_group_state() == 6) #for vol in vol_list: #state = vol.get_vol_state() #assert('Online, degraded' in state) # starting io io_patterns() # poweron the drive again logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) if volType == "critical": # poweroff 2 drive media = [media_group.md_disk[0]] + [media_group.md_disk[1]] logger.info("Powering off drive : %s" % media) status = power_off_drive(media, pvlLibHandle) time.sleep(5) #assert(media_group.get_media_group_state() == 7) #for vol in vol_list: #state = vol.get_vol_state() #assert('Online, critical' in state) # starting io io_patterns() # Poweron one drive media = [media_group.md_disk[0]] logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) media = [media_group.md_disk[1]] logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60)
def pds_ctrl_fofb(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3],'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS) ctrl_id = list(get_all_ids_in_zone(ZONE)) ctrl_obj_0 = ControllerOps(pvlLibHandle,ctrl_id[0],"rdma") ctrl_obj_1 = ControllerOps(pvlLibHandle,ctrl_id[1],"rdma") #calling base function to populate the objects media_group,vol_list = base_function(pvlLibHandle,logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) # connet to the host status = host.connect_volume(vol) assert(status == 0) def startIO(): p1 = multiprocessing.current_process() logger.info("Starting : %s : %s"%(p1.name,p1.pid)) print "Starting :",p1.name,p1.pid logger.info("Starting write on the vol-seq_write") kwargs = {'offset':"0",'size':'100%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} status,response=fio.seq_write(vol_list, host, kwargs) assert (status == 0) #start fio on all the volumes p1 = multiprocessing.Process(name="StartFio",target=startIO) p1.start() #logger.info("Starting write on the vol-seq_write") #kwargs = {'offset':"0",'size':'1%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1} #status,response=fio.seq_write(vol_list, host, kwargs) #assert (status == 0) # start FOFB with fresh write logger.info("Start controler poweroff/on") for i in range(10): # controller poeroff status = ctrl_obj_0.power_off() #assert(status == True) time.sleep(60) # controller poweron status = ctrl_obj_0.power_on() #assert(status == True) time.sleep(150) # poweroff/on another controller status = ctrl_obj_1.power_off() #assert(status == True) time.sleep(60) # poweroff/on another controller status = ctrl_obj_1.power_on() #assert(status == True) time.sleep(60) p1.join() #disconnect the vol for vol in vol_list: logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10)
def pds_online_vol_IO_sanity(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3],'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0",CHASSIS_IP,logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER,password=CHASSIS_PASS) ctrl_id = list(get_all_ids_in_zone(ZONE)) ctrl_obj_0 = ControllerOps(pvlLibHandle,ctrl_id[0],"rdma") ctrl_obj_1 = ControllerOps(pvlLibHandle,ctrl_id[1],"rdma") #calling base function to populate the objects media_group,vol_list = base_function(pvlLibHandle,logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports,hostnqn=[]) assert(status == True) # connet to the host status = host.connect_volume(vol) assert(status == 0) def startIO(): p1 = multiprocessing.current_process() logger.info("Starting : %s : %s"%(p1.name,p1.pid)) print "Starting :",p1.name,p1.pid logger.info("Starting write on the vol-seq_write") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCD","verify_interval":4096,"do_verify":1, "bs":"64k"} status,response=fio.seq_write(vol_list, host, kwargs) assert (status == 0) # # start FIO thread here # # temp_file_path = "%s/tmp/run_fio.%s"%(env['TESTDIR'] ,os.getpid()) # cmd = "touch %s "%temp_file_path # run(cmd) # proc=multiprocessing.Process(target=fio.infinite_fio,args=(vol_list, host, temp_file_path)) # proc.start() # logger.info('Fio process id code %s'%proc.pid) # sleep_p(60) # if proc.is_alive(): # sleep_p(60) # else: # logger.error('Fio process failed %s'%proc.pid) # assert(proc.exitcode==0) # # cmd = "rm -f %s "%temp_file_path # run(cmd) # if proc.is_alive(): # proc.join() # print"fio run finished" # ## start graceful FOFB while the IOs are running # #start fio on all the volumes # p1 = multiprocessing.Process(name="StartFio",target=startIO) # logger.info("starting IOs just in parallel with FOFB operations") # p1.start() # time.sleep(5) def rebuilds_func(): # starting one driver rebuild # Power off one drive mediaList = media_group.get_media_group_disk() for i in range(1): media = [mediaList[i]] logger.info("Powering off drive : %s"%media) status = power_off_drive(media,pvlLibHandle) time.sleep(30) # poweron the same drive logger.info("Powering ON drive : %s"%media) status = power_on_drive(media,pvlLibHandle) time.sleep(60) # start rebuild with io logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() #status=media_group.asynchronous_rebuild_media_group() assert(media_group.synchronous_rebuild_media_group()) time.sleep(60) ## starting 2 drive rebuild for i in range(10): media = [mediaList[0]] + [mediaList[1]] logger.info("Powering off drive : %s"%media) status = power_off_drive(media,pvlLibHandle) time.sleep(60) # poweron the drive again logger.info("Powering ON drive : %s"%media) status = power_on_drive(media,pvlLibHandle) time.sleep(60) # start async-rebuild with io logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() #status=media_group.asynchronous_rebuild_media_group() assert(media_group.synchronous_rebuild_media_group()) time.sleep(60) ## 2>1 && 1>0 rebuild #mediaList = media_group.get_media_group_disk() media = [mediaList[2]] + [mediaList[3]] logger.info("Powering off drive : %s"%media) status = power_off_drive(media,pvlLibHandle) time.sleep(60) # Poweron one drive and start the rebuild media = [mediaList[2]] logger.info("Powering ON drive : %s"%media) status = power_on_drive(media,pvlLibHandle) time.sleep(60) # start rebuild with IO logger.info("Start rebuild 2>1") #status=media_group.synchronous_rebuild_media_group() status=media_group.asynchronous_rebuild_media_group() time.sleep(60) if proc.is_alive(): # Poweron second drive and start the rebuild media = [mediaList[3]] logger.info("Powering ON drive : %s"%media) status = power_on_drive(media,pvlLibHandle) time.sleep(60) # start async-rebuild logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() #status=media_group.asynchronous_rebuild_media_group() assert(media_group.synchronous_rebuild_media_group()) time.sleep(60) def graceful_fofb(): logger.info("Starting graceful FOFB power-off/on") for i in range(1): # controller poeroff status = ctrl_obj_0.power_off() #assert(status == True) time.sleep(60) # controller poweron status = ctrl_obj_0.power_on() #assert(status == True) time.sleep(150) # poweroff/on another controller status = ctrl_obj_1.power_off() #assert(status == True) time.sleep(60) # poweroff/on another controller status = ctrl_obj_1.power_on() #assert(status == True) time.sleep(60) def resize_vol(vol_list): for vol in vol_list: logger.info(" vol size before resize operation is : %s"%vol.vol_size) status = host.get_vol_size(vol) print status # vol growing by 100G logger.info("follwing vol is getting grown by 100GB : %s"%vol.name) status = vol.volume_grow() assert(status == True) ## rescan the vol on the host status = host.get_added_vol_capacity(vol) #printing new size of the vol logger.info(" vol size after resize operation is : %s"%vol.vol_size) status = host.get_vol_size(vol) print status #start FIO thread here temp_file_path = "%s/tmp/run_fio.%s"%(env['TESTDIR'] ,os.getpid()) cmd = "touch %s "%temp_file_path run(cmd) proc=multiprocessing.Process(target=fio.infinite_fio,args=(vol_list, host, temp_file_path)) proc.start() logger.info('Fio process id code %s'%proc.pid) sleep_p(60) if proc.is_alive(): sleep_p(60) else: logger.error('Fio process failed %s'%proc.pid) assert(proc.exitcode==0) try: #1# now starting graceful FOFB operations logger.info("Starting graceful FOFB power-off/on") if proc.is_alive(): graceful_fofb() ## take snapshot and clones here #2# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild if proc.is_alive(): rebuilds_func() #3# now starting graceful FOFB operations logger.info("Starting graceful FOFB power-off/on") if proc.is_alive(): graceful_fofb() #00# take the clone and overwrite on it <<< i will implement this section after vol resize loop #4# grow vol and do the stuffs if proc.is_alive(): resize_vol(vol_list) #5# Now again do the FOFB operations logger.info("Starting graceful FOFB power-off/on") if proc.is_alive(): graceful_fofb() #6# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild if proc.is_alive(): rebuilds_func() #7# Start FOFB if proc.is_alive(): graceful_fofb() #8# Now perfrom 1>0 || 2>0 || 2>1,1>0 rebuild if proc.is_alive(): rebuilds_func() except Exception as E: logger.error('Stack trace for failure \n%s\n'%traceback.format_exc()) raise E finally: cmd = "rm -f %s "%temp_file_path run(cmd) if proc.is_alive(): proc.join() code = proc.exitcode if code != 0 : logger.error('Fio failed on volumes ') assert(code == 0 )
def pds_online_vol_IO_sanity(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3], 'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS) #calling base function to populate the objects media_group, vol_list = base_function(pvlLibHandle, logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports, hostnqn=[]) assert (status == True) # connet to the host status = host.connect_volume(vol) assert (status == 0) for i in range(1): snap_list = [] #star fresh write on all the volumes logger.info("Starting fresh write on the vol-seq_write") kwargs = { 'offset': "0", 'size': '50%', "verify_pattern": "0xABCD", "verify_interval": 4096, "do_verify": 1, "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) # start overwrite with same pattern logger.info("Starting over-write with same pattern -vol-seq_write") kwargs = { 'offset': "0", 'size': '50%', "verify_pattern": "0xABCD", "verify_interval": 4096, "do_verify": 1, "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) ''' # start overwrite with diff pattern (rw=rw,splitbs) logger.info("Starting over-write with diff pattern -rw=rw") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDffff","verify_interval":4096,"do_verify":1, "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} status,response=fio.seq_read_write(vol_list, host, kwargs) assert (status == 0) # start overwrite with diff pattern (rw=randrw,splitbs) logger.info("Starting over-write with diff pattern -rw=randrw") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1, "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} status,response=fio.rand_read_write(vol_list, host, kwargs) assert (status == 0) # randwrite on online vol logger.info("Starting over-write with diff pattern -rw=randwrite") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1, "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} status,response=fio.rand_write(vol_list, host, kwargs) assert (status == 0) for vol in vol_list: #Taking snaphost snap_name = str(vol.name) + str("_%s"%SNAP_PREFIX) logger.info("Taking snapshot :%s"%snap_name) snapshot = SnapShotOps(snap_name,volobj=vol,create=True) assert(snapshot != None) snap_list.append(snapshot) #Write on remaning space of the volumes logger.info("Starting write on other space of the vol ") kwargs = {'offset':"0",'size':'1%',"verify_pattern":"0xffff","verify_interval":4096,"do_verify":1} status,response=fio.rand_write(vol_list, host, kwargs) assert (status == 0) # delete snapshot for snap in snap_list: status = snap.delete_snapshot() assert (status == True) time.sleep(600) ''' #disconnect the vol for vol in vol_list: logger.info("disconnecting the vol from the host") status = host.disconnect_volume(vol) time.sleep(10)
def pds_critical_vol_IO_sanity(): #objects initialization & Authentication log = PvltLogger(inspect.stack()[0][3], 'INFO') logger = log.logger host = HostOps(HOST[0]) fio = FioUtils() pvlLibHandle = pvlclient.New("2.0", CHASSIS_IP, logger=logger) result = pvlLibHandle.auth.login(user=CHASSIS_USER, password=CHASSIS_PASS) #calling base function to populate the objects media_group, vol_list = base_function(pvlLibHandle, logger) # assigning vol ports = get_volume_network_map(pvlLibHandle, ZONE) logger.info("Assigning vols ") for vol in vol_list: status = vol.assign_volume(ports, hostnqn=[]) assert (status == True) # connet to the host status = host.connect_volume(vol) assert (status == 0) # poweroff 2 drive mediaList = media_group.get_media_group_disk() media = [mediaList[0]] + [mediaList[1]] logger.info("Powering off drive : %s" % media) status = power_off_drive(media, pvlLibHandle) for i in range(1): snap_list = [] #star fresh write on all the volumes logger.info("Starting fresh write on the vol-seq_write") kwargs = { 'offset': "0", 'size': '50%', "verify_pattern": "0xABCD", "verify_interval": 4096, "do_verify": 1, "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) # start overwrite with same pattern logger.info("Starting over-write with same pattern -vol-seq_write") kwargs = { 'offset': "0", 'size': '50%', "verify_pattern": "0xABCD", "verify_interval": 4096, "do_verify": 1, "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5" } status, response = fio.seq_write(vol_list, host, kwargs) assert (status == 0) ''' # start overwrite with diff pattern (rw=rw,splitbs) logger.info("Starting over-write with diff pattern -rw=rw") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDffff","verify_interval":4096,"do_verify":1, "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} status,response=fio.seq_read_write(vol_list, host, kwargs) assert (status == 0) # start overwrite with diff pattern (rw=randrw,splitbs) logger.info("Starting over-write with diff pattern -rw=randrw") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1, "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} status,response=fio.rand_read_write(vol_list, host, kwargs) assert (status == 0) # randwrite on online vol logger.info("Starting over-write with diff pattern -rw=randwrite") kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1, "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} status,response=fio.rand_write(vol_list, host, kwargs) assert (status == 0) ''' # Poweron one drive and start the rebuild media = [mediaList[0]] logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) # start rebuild with IO logger.info("Start rebuild 2>1") #status=media_group.synchronous_rebuild_media_group() status = media_group.asynchronous_rebuild_media_group() time.sleep(60) # starting IO load #logger.info("Starting over-write with diff pattern -rw=randrw") #kwargs = {'offset':"0",'size':'50%',"verify_pattern":"0xABCDabcd","verify_interval":4096,"do_verify":1, # "bssplit":"4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5"} #status,response=fio.rand_read_write(vol_list, host, kwargs) #assert (status == 0) # Poweron second drive and start the rebuild media = [mediaList[1]] logger.info("Powering ON drive : %s" % media) status = power_on_drive(media, pvlLibHandle) time.sleep(60) # start async-rebuild logger.info("Start rebuild 1>0") #status=media_group.synchronous_rebuild_media_group() status = media_group.asynchronous_rebuild_media_group() # starting IO load logger.info("Starting over-write with diff pattern -rw=randrw") kwargs = { 'offset': "0", 'size': '50%', "verify_pattern": "0xABCDabcd", "verify_interval": 4096, "do_verify": 1, "bssplit": "4k/15:16k/15:32k/15:64k/15:128k/15:256k/10:512k/5:1m/5:2m/5" } status, response = fio.rand_read_write(vol_list, host, kwargs) assert (status == 0)