def test3_post_service_interuption_check_volume_attachment_of_new_vols_from_old_snaps(
self):
vols = []
errmsg = ""
for snap in self.snapshots:
vols.extend(
self.tester.get_volumes(status='available', snapid=snap.id))
if not vols:
raise Exception(
"No vols were found as created from previous snapshots")
if not self.instances:
raise Exception('No instances to use for this test')
if not self.snapshots:
raise Exception('No snapshots to use for this test')
for vol in vols:
vol.update()
availvols = copy.copy(vols)
self.status('Attempting to attach the following volumes:',
testcolor=TestColor.get_canned_color('whiteonblue'))
self.tester.show_volumes(availvols)
#Iterate through the volumes, and attach them all to at least one instance in each zone.
for instance in self.instances:
if not availvols:
break
for vol in availvols:
if vol.zone == instance.placement:
availvols.remove(vol)
try:
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status(
"This is a temp work around for testing, this is to avoid bug euca-5297"
+ str(vse),
testcolor=TestColor.get_canned_color(
'failred'))
time.sleep(10)
self.tester.monitor_euvolumes_to_status(
[vol],
status='in-use',
attached_status='attached',
timeout=60)
for snap in self.snapshots:
snap.update()
if vol.snapshot_id == snap.id:
if vol.md5len != snap.eutest_volume_md5len:
self.debug(
'Need to adjust md5sum for length of snapshot...'
)
vol.md5len = snap.eutest_volume_md5len
instance.vol_write_random_data_get_md5(
vol, length=snap.eutest_volume_md5len)
if vol.md5 != snap.eutest_volume_md5:
errmsg += "Volume:" + str(vol.id) + " MD5 did not match snapshots " \
+ str(snap.id) + ": snapmd5:" + str(snap.eutest_volume_md5) \
+ " --vs volmd5:-- " + str(vol.md5)
except Exception, e:
errmsg += str(instance.id) +"Volume:" + str(vol.id) \
+ " error when attaching and comparing md5, err:" + str(e)
def pretest7_per_service_restart_attach_vols_to_bfebs_instances(self, volsperinstance=2):
"""
Definition: Attach all volumes created in this test for bfebs to all bfebs instances created in this test.
This tests should be completed per zone prior to restarting services.
"""
self.status(
"' pretest7_per_service_restart_attach_vols_to_bfebs_instances' starting...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
for instance in self.bfebs_instances:
for x in xrange(0, volsperinstance):
for vol in self.bfebs_volumes:
if vol.zone == instance.placement and vol.status == "available":
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status(
"This is a temp work around for testing, this is to avoid bug euca-5297" + str(vse),
testcolor=TestColor.get_canned_color("failred"),
)
time.sleep(10)
self.debug("Monitoring volume post VolumeStateException...")
vol.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status(
[vol], status="in-use", attached_status="attached", timeout=60
)
def pretest7_per_service_restart_attach_vols_to_bfebs_instances(
self, volsperinstance=2):
'''
Definition: Attach all volumes created in this test for bfebs to all bfebs instances created in this test.
This tests should be completed per zone prior to restarting services.
'''
self.status(
"\' pretest7_per_service_restart_attach_vols_to_bfebs_instances\' starting...",
testcolor=TestColor.get_canned_color('whiteonblue'))
for instance in self.bfebs_instances:
for x in xrange(0, volsperinstance):
for vol in self.bfebs_volumes:
if vol.zone == instance.placement and vol.status == 'available':
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status(
"This is a temp work around for testing, this is to avoid bug euca-5297"
+ str(vse),
testcolor=TestColor.get_canned_color(
'failred'))
time.sleep(10)
self.debug(
'Monitoring volume post VolumeStateException...'
)
vol.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status(
[vol],
status='in-use',
attached_status='attached',
timeout=60)
def test2_post_service_interuption_check_volume_from_snapshots(self):
self.status("Checking creation of volumes from pre-existing snapshots post service interruption...",
testcolor=TestColor.get_canned_color('whiteonblue'))
vols = []
if not self.snapshots:
raise Exception("self.snapshots not populated?")
for snap in self.snapshots:
for zone in self.zones:
vols.extend(self.tester.create_volumes(zone, snapshot=snap, monitor_to_state=None))
self.volumes.extend(self.tester.monitor_created_euvolumes_to_state(vols, state='available'))
self.status("Done creating volumes from pre-existing snapshots post service interruption.",
testcolor=TestColor.get_canned_color('whiteonblue'))
def qa_214_test3(self,
volcount=5,
testcount=5,
size=1,
timepergig=120,
deletetimeout=120):
"""
Definition: repeat : ( create volume Z -> attach volume Z -> detach volume Z -> delete volume Z)
:param volcount: integer how many concurrent volumes to create/delete
:param testcount: integer how many times to repeat this test
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
instance = self.instance or self.launch_test_instance()
for x in xrange(0, testcount):
self.status("\'qa_214_test3\' number:" + str(x) + "/" +
str(testcount))
volumes = self.tester.create_volumes(self.zone,
size=size,
count=volcount,
timepergig=timepergig)
for volume in volumes:
instance.attach_volume(volume)
time.sleep(1)
instance.detach_euvolume(volume)
self.tester.delete_volumes(volumes,
poll_interval=5,
timeout=deletetimeout)
self.status('qa_214_test3: Completed:' + str(x) + '/' +
str(testcount) + ' tests',
testcolor=TestColor.get_canned_color('whiteonblue'))
def pretest6_pre_service_restart_create_bfebs_volume_resources(self, volsperinstance=2, size=1, timepergig=300):
"""
Definition: Create volumes to be used in this test based upon volsperinstance and size args provided.
This tests should be completed per zone prior to restarting services.
:param volsperinstance: integer how many concurrent volumes to create/delete per instance
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
"""
volumes = []
for zone in self.zones:
instancecount = 0
for instance in self.bfebs_instances:
if instance.placement == zone:
instancecount += 1
volcount = instancecount * volsperinstance
volumes.extend(
self.tester.create_volumes(
zone, size=size, count=volcount, monitor_to_state=None, timepergig=timepergig
)
)
self.bfebs_volumes = self.tester.monitor_created_euvolumes_to_state(volumes, timepergig=timepergig)
self.status(
"pretest6_pre_service_restart_create_bfebs_volume_resources, done",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
def qa_214_test1(self,
volcount=5,
testcount=20,
size=1,
timepergig=120,
deletetimeout=120):
"""
Definition: repeats: (create volume X -> delete volume X) 'count' number of times
:param volcount: integer how many concurrent volumes to create/delete
:param testcount: integer how many times to repeat this test
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
for x in xrange(0, testcount):
self.status("\'qa_214_test1\' number:" + str(x) + "/" +
str(testcount))
self.volumes = self.tester.create_volumes(self.zone,
size=size,
count=volcount,
timepergig=timepergig)
self.tester.delete_volumes(self.volumes,
poll_interval=5,
timeout=deletetimeout)
self.status('qa_214_test1: Completed:' + str(x) + '/' +
str(testcount) + ' tests',
testcolor=TestColor.get_canned_color('whiteonblue'))
def pretest6_pre_service_restart_create_bfebs_volume_resources(
self, volsperinstance=2, size=1, timepergig=300):
"""
Definition: Create volumes to be used in this test based upon volsperinstance and size args provided.
This tests should be completed per zone prior to restarting services.
:param volsperinstance: integer how many concurrent volumes to create/delete per instance
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
"""
volumes = []
for zone in self.zones:
instancecount = 0
for instance in self.bfebs_instances:
if instance.placement == zone:
instancecount += 1
volcount = instancecount * volsperinstance
volumes.extend(
self.tester.create_volumes(zone,
size=size,
count=volcount,
monitor_to_state=None,
timepergig=timepergig))
self.bfebs_volumes = self.tester.monitor_created_euvolumes_to_state(
volumes, timepergig=timepergig)
self.status(
'pretest6_pre_service_restart_create_bfebs_volume_resources, done',
testcolor=TestColor.get_canned_color('whiteonblue'))
def test2_post_service_interuption_check_volume_from_snapshots(self):
self.status(
"Checking creation of volumes from pre-existing snapshots post service interruption...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
vols = []
if not self.snapshots:
raise Exception("self.snapshots not populated?")
for snap in self.snapshots:
for zone in self.zones:
vols.extend(self.tester.create_volumes(zone, snapshot=snap, monitor_to_state=None))
self.volumes.extend(self.tester.monitor_created_euvolumes_to_state(vols, state="available"))
self.status(
"Done creating volumes from pre-existing snapshots post service interruption.",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
def start_all_services_on_storage_controllers(self, sc_list):
self.status("Waiting for storage controller's services to start...",
testcolor=TestColor.get_canned_color('whiteonblue'))
all_services_on_sc = []
if not sc_list:
raise Exception(
"sc_list was empty in: start_all_services_on_storage_controllers"
)
#wait = 10 * 60
for sc in sc_list:
self.debug('Getting all services co-existing on sc: ' +
str(sc.hostname))
all_services_on_sc.extend(
self.tester.service_manager.get_all_services_by_filter(
hostname=sc.hostname))
for service in all_services_on_sc:
service.start()
self.debug(
'Issued start to all services, now wait for: all_services_operational'
)
self.tester.service_manager.all_services_operational()
try:
for service in all_services_on_sc:
self.tester.service_manager.wait_for_service(service)
except Exception, e:
dbgout = self.tester.get_traceback()
self.tester.service_manager.print_services_list(all_services_on_sc)
dbgout += "\nFailed waiting for all services on rebooted machines to recover\n"
raise Exception(dbgout + str(e))
def print_all_test_resources(self):
self.status(
"Printing test resources prior to service interruption...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
self.tester.print_euinstance_list(self.instances)
self.tester.print_euvolume_list(self.volumes)
self.tester.print_eusnapshot_list(self.snapshots)
def test2_post_service_interuption_check_volume_from_snapshots(self):
"""
Definition: Attempts to check volume creation from snapshots created before SC restart.
"""
self.status("Checking creation of volumes from pre-existing snapshots post service interruption...",
testcolor=TestColor.get_canned_color('whiteonblue'))
vols = []
if not self.snapshots:
raise Exception("self.snapshots not populated?")
for snap in self.snapshots:
for zone in self.zones:
vols.extend(self.tester.create_volumes(zone, snapshot=snap, monitor_to_state=None))
#Now monitor created volumes to state available, use a large timeout per gig here.
self.volumes.extend(self.tester.monitor_created_euvolumes_to_state(vols, state='available', timepergig=360))
self.status("Done creating volumes from pre-existing snapshots post service interruption.",
testcolor=TestColor.get_canned_color('whiteonblue'))
def test3_post_service_interuption_check_volume_attachment_of_new_vols_from_old_snaps(self):
vols = []
errmsg = ""
for snap in self.snapshots:
vols.extend(self.tester.get_volumes(status='available', snapid=snap.id))
if not vols:
raise Exception("No vols were found as created from previous snapshots")
if not self.instances:
raise Exception('No instances to use for this test')
if not self.snapshots:
raise Exception('No snapshots to use for this test')
for vol in vols:
vol.update()
availvols = copy.copy(vols)
self.status('Attempting to attach the following volumes:',testcolor=TestColor.get_canned_color('whiteonblue'))
self.tester.show_volumes(availvols)
#Iterate through the volumes, and attach them all to at least one instance in each zone.
for instance in self.instances:
if not availvols:
break
for vol in availvols:
if vol.zone == instance.placement:
availvols.remove(vol)
try:
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status("This is a temp work around for testing, this is to avoid bug euca-5297"+str(vse),
testcolor=TestColor.get_canned_color('failred'))
time.sleep(10)
self.tester.monitor_euvolumes_to_status([vol],status='in-use',attached_status='attached',timeout=60)
for snap in self.snapshots:
snap.update()
if vol.snapshot_id == snap.id:
if vol.md5len != snap.eutest_volume_md5len:
self.debug('Need to adjust md5sum for length of snapshot...')
vol.md5len = snap.eutest_volume_md5len
instance.vol_write_random_data_get_md5(vol,length=snap.eutest_volume_md5len)
if vol.md5 != snap.eutest_volume_md5:
errmsg += "Volume:" + str(vol.id) + " MD5 did not match snapshots " \
+ str(snap.id) + ": snapmd5:" + str(snap.eutest_volume_md5) \
+ " --vs volmd5:-- " + str(vol.md5)
except Exception, e:
errmsg += str(instance.id) +"Volume:" + str(vol.id) \
+ " error when attaching and comparing md5, err:" + str(e)
def test1_post_service_interruption_check_attached_volumes(self, check_vols=None, check_instances=None):
"""
Definition: Attempts to verify that volumes maintained their attached state through the SC reboot.
"""
check_vols = check_vols or self.volumes
check_instances = check_instances or self.instances
write_length = 10000
errmsg = ""
self.tester.show_volumes(check_vols)
for vol in check_vols:
vol.update()
if vol.status == "in-use" and (vol.attach_data and vol.attach_data.status == 'attached' ):
for instance in check_instances:
if instance.id == vol.attach_data.instance_id:
if not vol in instance.attached_vols:
errmsg += "Volume:" + str(vol.id) \
+ " is attached to " + str(instance.id) + ", but not in instance attached list?"
break
else:
errmsg += "Volume:" + str(vol.id) + ", status:" + str(vol.status) \
+ " was not attached post service interruption \n"
if errmsg:
raise Exception(errmsg)
self.status("Attached state passed. Now checking read/write with attached volumes...",
testcolor=TestColor.get_canned_color('whiteonblue'))
for instance in check_instances:
instance.update()
instance.reset_ssh_connection()
try:
bad_vols = instance.get_unsynced_volumes()
if bad_vols:
errmsg = str(instance.id) + "Unsynced vols found:"
for bad_vol in bad_vols:
errmsg = errmsg + str(bad_vol.id) + ":" + str(bad_vol.status) + ","
errmsg = errmsg + "\n"
except Exception, ve:
errmsg = errmsg + "\n" + str(ve)
if errmsg:
raise Exception(errmsg)
for vol in instance.attached_vols:
try:
md5before = vol.md5
md5now = instance.md5_attached_euvolume(vol,updatevol=False)
if md5before != md5now:
errmsg += str(instance.id) +"Volume:" + str(vol.id) \
+ "has different md5 sum after service interruption. Before:'" \
+ str(md5before) + "' - vs - '" + str(md5now) + "'"
vol.length = write_length
instance.vol_write_random_data_get_md5(vol,length=write_length, overwrite=True)
instance.sys('sync',code=0)
except Exception, e:
errmsg += str(instance.id) + "Volume:" + \
str(vol.id) + ", error while using vol post service interruption, err: "+str(e)
def test1_post_service_interruption_check_attached_volumes(self):
write_length = 10000
errmsg = ""
self.status("Checking volumes for attached state post service interruption...",
testcolor=TestColor.get_canned_color('whiteonblue'))
self.tester.print_euvolume_list(self.volumes)
for vol in self.volumes:
vol.update()
if vol.status == "in-use" and (vol.attach_data and vol.attach_data.status == 'attached' ):
for instance in self.instances:
if instance.id == vol.attach_data.instance_id:
if not vol in instance.attached_vols:
errmsg += "Volume:" + str(vol.id) \
+ " is attached to " + str(instance.id) + ", but not in instance attached list?"
break
else:
errmsg += "Volume:" + str(vol.id) + ", status:" + str(vol.status) \
+ " was not attached post service interruption \n"
if errmsg:
raise Exception(errmsg)
self.status("Attached state passed. Now checking read/write with attached volumes...",
testcolor=TestColor.get_canned_color('whiteonblue'))
for instance in self.instances:
instance.update()
instance.reset_ssh_connection()
for vol in instance.attached_vols:
try:
md5before = vol.md5
md5now = instance.md5_attached_euvolume(vol,updatevol=False)
if md5before != md5now:
errmsg += str(instance.id) +"Volume:" + str(vol.id) \
+ "has different md5 sum after service interruption. Before:'" \
+ str(md5before) + "' - vs - '" + str(md5now) + "'"
vol.length = write_length
instance.vol_write_random_data_get_md5(vol,length=write_length, overwrite=True)
instance.sys('sync',code=0)
except Exception, e:
errmsg += str(instance.id) + "Volume:" + \
str(vol.id) + ", error while using vol post service interruption, err: "+str(e)
def test3_post_service_interuption_check_volume_attachment_of_new_vols_from_old_snaps(self):
vols = []
errmsg = ""
for snap in self.snapshots:
vols.extend(self.tester.get_volumes(snapid=snap.id))
if not vols:
raise Exception("No vols were found as created from previous snapshots")
if not self.instances:
raise Exception("No instances to use for this test")
if not self.snapshots:
raise Exception("No snapshots to use for this test")
self.tester.print_euvolume_list(vols)
for instance in self.instances:
for vol in vols:
if vol.zone == instance.placement:
try:
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status(
"This is a temp work around for testing, this is to avoid bug euca-5297" + str(vse),
testcolor=TestColor.get_canned_color("failred"),
)
time.sleep(10)
self.tester.monitor_euvolumes_to_status(
[vol], status="in-use", attached_status="attached", timeout=60
)
for snap in self.snapshots:
snap.update()
if vol.snapshot_id == snap.id:
if vol.md5len != snap.eutest_volume_md5len:
self.debug("Need to adjust md5sum for length of snapshot...")
vol.md5len = snap.eutest_volume_md5len
instance.vol_write_random_data_get_md5(vol, length=snap.eutest_volume_md5len)
if vol.md5 != snap.eutest_volume_md5:
errmsg += (
"Volume:"
+ str(vol.id)
+ " MD5 did not match snapshots "
+ str(snap.id)
+ ": snapmd5:"
+ str(snap.eutest_volume_md5)
+ " --vs volmd5:-- "
+ str(vol.md5)
)
except Exception, e:
errmsg += (
str(instance.id)
+ "Volume:"
+ str(vol.id)
+ " error when attaching and comparing md5, err:"
+ str(e)
)
def pretest3_pre_service_restart_attach_all_volumes(self):
'''
Definition: Attach all volumes created in this test to all instances created in this test.
This tests should be completed per zone prior to restarting services.
'''
self.status("\'pre_service_restart_attach_all_volumes\' starting...",
testcolor=TestColor.get_canned_color('whiteonblue'))
for instance in self.instances:
for x in xrange(0,self.args.volsperinstance):
for vol in self.volumes:
if vol.zone == instance.placement and vol.status == 'available':
try:
instance.attach_volume(vol,timeout=90)
except ec2ops.VolumeStateException, vse:
self.status("This is a temp work around for testing, this is to avoid bug euca-5297"+str(vse),
testcolor=TestColor.get_canned_color('failred'))
time.sleep(10)
self.debug('Monitoring volume post VolumeStateException...')
vol.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status([vol],status='in-use',attached_status='attached',timeout=60)
def test2_post_service_interuption_check_volume_from_snapshots(self):
"""
Definition: Attempts to check volume creation from snapshots created before SC restart.
"""
self.status(
"Checking creation of volumes from pre-existing snapshots post service interruption...",
testcolor=TestColor.get_canned_color('whiteonblue'))
vols = []
if not self.snapshots:
raise Exception("self.snapshots not populated?")
for snap in self.snapshots:
for zone in self.zones:
vols.extend(
self.tester.create_volumes(zone,
snapshot=snap,
monitor_to_state=None))
#Now monitor created volumes to state available, use a large timeout per gig here.
self.volumes.extend(
self.tester.monitor_created_euvolumes_to_state(vols,
state='available',
timepergig=360))
self.status(
"Done creating volumes from pre-existing snapshots post service interruption.",
testcolor=TestColor.get_canned_color('whiteonblue'))
def test4_post_service_interuption_check_snapshot_creation(self):
self.status('Attempting to verify snapshot creation post reboot',\
testcolor=TestColor.get_canned_color('whiteonblue'))
testvols = []
testsnaps = []
for zone in self.zones:
for vol in self.volumes:
if vol.zone == zone:
testvols.append(vol)
break
for vol in testvols:
testsnaps.extend(self.tester.create_snapshots(vol, monitor_to_completed=False))
wait_on_progress = int(len(testsnaps)*25)
self.tester.monitor_eusnaps_to_completed(testsnaps,wait_on_progress=wait_on_progress)
self.snapshots.extend(testsnaps)
def test4_post_service_interuption_check_snapshot_creation(self):
self.status('Attempting to verify snapshot creation post reboot',\
testcolor=TestColor.get_canned_color('whiteonblue'))
testvols = []
testsnaps = []
for zone in self.zones:
for vol in self.volumes:
if vol.zone == zone:
testvols.append(vol)
break
for vol in testvols:
testsnaps.extend(
self.tester.create_snapshots(vol, monitor_to_completed=False))
wait_on_progress = int(len(testsnaps) * 25)
self.tester.monitor_eusnaps_to_completed(
testsnaps, wait_on_progress=wait_on_progress)
self.snapshots.extend(testsnaps)
def attach_all_avail_vols_to_instances_in_zones(self, zonelist=None, timeout=360, overwrite=False):
"""
Description:
Iterates though zones and attempts to attach volumes to an instance within each zone.
:parram zonelist: list of zones to include in test
:param timeout: timeout used for attach volume method
:param overwrite: boolean to indicate whether a non-zero filled volume should have new unique data prepended for md5sum.
This should be used when zero fill volume property is not in use upon volume first attach.
It should not be used after the 1st attach and volume has been converted to a euvolume within this test.
"""
zonelist = zonelist or self.zonelist
if not zonelist:
raise Exception("attach_all_avail_vols_to_instances_in_zones: Zonelist is empty")
for zone in zonelist:
if not zone.volumes:
raise Exception('attach_all_avail_vols_to_instances_in_zones: Zone.volumes is empty')
if not zone.instances:
raise Exception('attach_all_avail_vols_to_instances_in_zones: Instance list is empty')
i=0
for volume in zone.volumes:
volume.update()
if (volume.status == "available"):
if i > (len(zone.instances)-1):
i = 0
self.debug("Attempting to attach to "+str(i)+"/"+str(len(zone.instances))+" instances in zone:"+str(zone))
instance = zone.instances[i]
try:
instance.attach_euvolume(volume,timeout=timeout, overwrite=overwrite)
except ec2ops.VolumeStateException, vse:
self.status("This is a temp work around for testing, this is to avoid bug euca-5297"+str(vse),
testcolor=TestColor.get_canned_color('failred'))
time.sleep(10)
self.debug('Monitoring volume post VolumeStateException...')
volume.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status([volume],status='in-use',attached_status='attached',timeout=60)
except Exception, e:
self.debug("attach_all_vols_to_instances_in_zones failed to attach volume")
raise e
i += 1
def qa_214_test2(self, testcount=20, size=1, volcount=1):
'''
Definition: create volumes Y, then repeat ( attach volumes Y-> detach volumes Y)
:param testcount: integer times to run this test
:param size: integer size of volume(s) to create in GB
:param volcount: number of volumes to create for this test
'''
instance = self.instance or self.launch_test_instance()
volumes = self.tester.create_volumes(self.zone,
size=size,
count=volcount)
for x in xrange(0, testcount):
self.status("\'qa_214_test2\' number:" + str(x) + "/" +
str(testcount))
for volume in volumes:
instance.attach_volume(volume)
time.sleep(1)
instance.detach_euvolume(volume)
self.status('qa_214_test2: Completed:' + str(x) + '/' +
str(testcount) + ' tests',
testcolor=TestColor.get_canned_color('whiteonblue'))
def start_all_services_on_storage_controllers(self, sc_list):
self.status("Waiting for storage controller's services to start...",
testcolor=TestColor.get_canned_color('whiteonblue'))
all_services_on_sc = []
if not sc_list:
raise Exception("sc_list was empty in: start_all_services_on_storage_controllers")
#wait = 10 * 60
for sc in sc_list:
self.debug('Getting all services co-existing on sc: ' + str(sc.hostname))
all_services_on_sc.extend(self.tester.service_manager.get_all_services_by_filter(hostname=sc.hostname))
for service in all_services_on_sc:
service.start()
self.debug('Issued start to all services, now wait for: all_services_operational')
self.tester.service_manager.all_services_operational()
try:
for service in all_services_on_sc:
self.tester.service_manager.wait_for_service(service)
except Exception, e:
dbgout = self.tester.get_traceback()
self.tester.service_manager.print_services_list(all_services_on_sc)
dbgout += "\nFailed waiting for all services on rebooted machines to recover\n"
raise Exception(dbgout + str(e))
def test1_post_service_interruption_check_attached_volumes(self):
write_length = 10000
errmsg = ""
self.status(
"Checking volumes for attached state post service interruption...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
self.tester.print_euvolume_list(self.volumes)
for vol in self.volumes:
vol.update()
if vol.status == "in-use" and (vol.attach_data and vol.attach_data.status == "attached"):
for instance in self.instances:
if instance.id == vol.attach_data.instance_id:
if not vol in instance.attached_vols:
errmsg += (
"Volume:"
+ str(vol.id)
+ " is attached to "
+ str(instance.id)
+ ", but not in instance attached list?"
)
break
else:
errmsg += (
"Volume:"
+ str(vol.id)
+ ", status:"
+ str(vol.status)
+ " was not attached post service interruption \n"
)
if errmsg:
raise Exception(errmsg)
self.status(
"Attached state passed. Now checking read/write with attached volumes...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
for instance in self.instances:
instance.update()
instance.reset_ssh_connection()
for vol in instance.attached_vols:
try:
md5before = vol.md5
md5now = instance.md5_attached_euvolume(vol, updatevol=False)
if md5before != md5now:
errmsg += (
str(instance.id)
+ "Volume:"
+ str(vol.id)
+ "has different md5 sum after service interruption. Before:'"
+ str(md5before)
+ "' - vs - '"
+ str(md5now)
+ "'"
)
vol.length = write_length
instance.vol_write_random_data_get_md5(vol, length=write_length, overwrite=True)
instance.sys("sync", code=0)
except Exception, e:
errmsg += (
str(instance.id)
+ "Volume:"
+ str(vol.id)
+ ", error while using vol post service interruption, err: "
+ str(e)
)
if vol.zone == instance.placement and vol.status == "available":
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status(
"This is a temp work around for testing, this is to avoid bug euca-5297" + str(vse),
testcolor=TestColor.get_canned_color("failred"),
)
time.sleep(10)
self.debug("Monitoring volume post VolumeStateException...")
vol.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status(
[vol], status="in-use", attached_status="attached", timeout=60
)
self.status(
"'pre_service_restart_attach_all_volumes' done", testcolor=TestColor.get_canned_color("whiteonblue")
)
def pretest4_pre_service_restart_create_snap_per_zone(self, timepergig=300):
"""
Definition: Create a single snapshot. This tests should be completed per zone prior to restarting services.
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
snaps = []
snapvols = []
# Add one volume from each zone...
for zone in self.zones:
for vol in self.volumes:
if vol.zone == zone:
snapvols.append(vol)
"This is a temp work around for testing, this is to avoid bug euca-5297"
+ str(vse),
testcolor=TestColor.get_canned_color(
'failred'))
time.sleep(10)
self.debug(
'Monitoring volume post VolumeStateException...'
)
vol.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status(
[vol],
status='in-use',
attached_status='attached',
timeout=60)
self.status("\'pre_service_restart_attach_all_volumes\' done",
testcolor=TestColor.get_canned_color('whiteonblue'))
def pretest4_pre_service_restart_create_snap_per_zone(
self, timepergig=300):
"""
Definition: Create a single snapshot. This tests should be completed per zone prior to restarting services.
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
snaps = []
snapvols = []
#Add one volume from each zone...
for zone in self.zones:
for vol in self.volumes:
if vol.zone == zone:
snapvols.append(vol)
def print_all_test_resources(self):
self.status('Printing test resources prior to service interruption...',
testcolor=TestColor.get_canned_color('whiteonblue'))
self.tester.show_instances(self.instances)
self.tester.show_volumes(self.volumes)
self.tester.show_snapshots(self.snapshots)
def reboot_sc_machine_verify_post_reboot(self, timeout=600):
"""
Definition: Restart the eucalyptus storage controller service
"""
self.print_all_test_resources()
#List to delay restart of storage controllers co-located with a cloud controller
debug_str = ""
#first make sure everything is good before we star the test...
self.tester.service_manager.all_services_operational()
storage_controllers = []
self.zones = self.zones or self.tester.get_zones()
for zone in self.zones:
self.debug('Getting storage controllers for :' + str(zone))
storage_controllers.extend(
self.tester.service_manager.get_all_storage_controllers(
partition=zone))
self.tester.service_manager.print_services_list(storage_controllers)
if not storage_controllers:
raise Exception(
'Storage controller list was not populated for zones:' +
",".join(self.zones))
for sc in storage_controllers:
debug_str = ""
all_services_on_sc = self.tester.service_manager.get_all_services_by_filter(
hostname=sc.hostname)
for service in all_services_on_sc:
debug_str += "(" + str(
service.hostname) + ":" + service.type + "), "
self.status("Now rebooting machine hosting services:" +
str(debug_str),
testcolor=TestColor.get_canned_color('whiteonblue'))
sc.machine.reboot()
if not storage_controllers:
raise Exception(
'Storage controller list post reboot was not populated for zones:'
+ ",".join(self.zones))
start = time.time()
elapsed = 0
waiting = copy.copy(storage_controllers)
while elapsed < timeout and waiting:
elapsed = int(time.time() - start)
for sc in waiting:
if self.tester.ping(sc.hostname, poll_count=1):
waiting.remove(sc)
if waiting:
debug_str = ""
for sc in waiting:
debug_str += " " + str(sc.hostname) + ","
self.debug("Waiting on SC's to become reachable post reboot:" +
str(debug_str) + ", elapsed:" + str(elapsed))
time.sleep(10)
else:
self.status(
"All SC machines are reachable again, now wait for SSH...",
testcolor=TestColor.get_canned_color('whiteonblue'))
break
if waiting:
raise Exception("SC machines were not reachable after: " +
str(elapsed) + " seconds:" + str(debug_str))
if not storage_controllers:
raise Exception(
'Storage controller list was not populated after waiting for reachable'
)
start = time.time()
elapsed = 0
waiting_for_ssh = copy.copy(storage_controllers)
debug_str = ""
while elapsed < 90 and waiting_for_ssh:
elapsed = int(time.time() - start)
for sc in waiting_for_ssh:
try:
sc.machine.refresh_ssh()
sc.machine.cmd(" ")
waiting_for_ssh.remove(sc)
except Exception, e:
self.debug('Failed to refresh ssh to:' + str(sc.hostname) +
', err:' + str(e))
if waiting_for_ssh:
debug_str = ""
for sc in waiting:
debug_str += " " + str(sc.hostname) + ","
self.debug("Waiting on SSH connections to SCs:" +
str(debug_str) + ", elapsed:" + str(elapsed))
time.sleep(10)
else:
self.status(
"All SC machines have established SSH connections, now wait on services to come back...",
testcolor=TestColor.get_canned_color('whiteonblue'))
break
def test1_post_service_interruption_check_attached_volumes(
self, check_vols=None, check_instances=None):
"""
Definition: Attempts to verify that volumes maintained their attached state through the SC reboot.
"""
check_vols = check_vols or self.volumes
check_instances = check_instances or self.instances
write_length = 10000
errmsg = ""
self.tester.show_volumes(check_vols)
for vol in check_vols:
vol.update()
if vol.status == "in-use" and (
vol.attach_data and vol.attach_data.status == 'attached'):
for instance in check_instances:
if instance.id == vol.attach_data.instance_id:
if not vol in instance.attached_vols:
errmsg += "Volume:" + str(vol.id) \
+ " is attached to " + str(instance.id) + ", but not in instance attached list?"
break
else:
errmsg += "Volume:" + str(vol.id) + ", status:" + str(vol.status) \
+ " was not attached post service interruption \n"
if errmsg:
raise Exception(errmsg)
self.status(
"Attached state passed. Now checking read/write with attached volumes...",
testcolor=TestColor.get_canned_color('whiteonblue'))
for instance in check_instances:
instance.update()
instance.reset_ssh_connection()
try:
bad_vols = instance.get_unsynced_volumes()
if bad_vols:
errmsg = str(instance.id) + "Unsynced vols found:"
for bad_vol in bad_vols:
errmsg = errmsg + str(bad_vol.id) + ":" + str(
bad_vol.status) + ","
errmsg = errmsg + "\n"
except Exception, ve:
errmsg = errmsg + "\n" + str(ve)
if errmsg:
raise Exception(errmsg)
for vol in instance.attached_vols:
try:
md5before = vol.md5
md5now = instance.md5_attached_euvolume(vol,
updatevol=False)
if md5before != md5now:
errmsg += str(instance.id) +"Volume:" + str(vol.id) \
+ "has different md5 sum after service interruption. Before:'" \
+ str(md5before) + "' - vs - '" + str(md5now) + "'"
vol.length = write_length
instance.vol_write_random_data_get_md5(vol,
length=write_length,
overwrite=True)
instance.sys('sync', code=0)
except Exception, e:
errmsg += str(instance.id) + "Volume:" + \
str(vol.id) + ", error while using vol post service interruption, err: "+str(e)
def print_all_test_resources(self):
self.status('Printing test resources prior to service interruption...',
testcolor=TestColor.get_canned_color('whiteonblue'))
self.tester.show_instances(self.instances)
self.tester.show_volumes(self.volumes)
self.tester.show_snapshots(self.snapshots)
def qa_214_test2(self, testcount=20, size=1,volcount=1):
'''
Definition: create volumes Y, then repeat ( attach volumes Y-> detach volumes Y)
:param testcount: integer times to run this test
:param size: integer size of volume(s) to create in GB
:param volcount: number of volumes to create for this test
'''
instance = self.instance or self.launch_test_instance()
volumes = self.tester.create_volumes(self.zone, size=size, count=volcount)
for x in xrange(0,testcount):
self.status("\'qa_214_test2\' number:"+str(x)+"/"+str(testcount))
for volume in volumes:
instance.attach_volume(volume)
time.sleep(1)
instance.detach_euvolume(volume)
self.status('qa_214_test2: Completed:'+str(x)+'/'+str(testcount)+' tests',testcolor=TestColor.get_canned_color('whiteonblue'))
for instance in self.instances:
for x in xrange(0,self.args.volsperinstance):
for vol in self.volumes:
if vol.zone == instance.placement and vol.status == 'available':
try:
instance.attach_volume(vol,timeout=90)
except ec2ops.VolumeStateException, vse:
self.status("This is a temp work around for testing, this is to avoid bug euca-5297"+str(vse),
testcolor=TestColor.get_canned_color('failred'))
time.sleep(10)
self.debug('Monitoring volume post VolumeStateException...')
vol.eutest_attached_status = None
self.tester.monitor_euvolumes_to_status([vol],status='in-use',attached_status='attached',timeout=60)
self.status("\'pre_service_restart_attach_all_volumes\' done",
testcolor=TestColor.get_canned_color('whiteonblue'))
def pretest4_pre_service_restart_create_snap_per_zone(self,timepergig=300):
"""
Definition: Create a single snapshot. This tests should be completed per zone prior to restarting services.
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
snaps = []
snapvols = []
#Add one volume from each zone...
for zone in self.zones:
for vol in self.volumes:
if vol.zone == zone:
snapvols.append(vol)
def qa_214_test1(self,
volcount=5,
testcount=20,
size=1,
timepergig=120,
deletetimeout=120):
"""
Definition: repeats: (create volume X -> delete volume X) 'count' number of times
:param volcount: integer how many concurrent volumes to create/delete
:param testcount: integer how many times to repeat this test
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
for x in xrange(0,testcount):
self.status("\'qa_214_test1\' number:"+str(x)+"/"+str(testcount))
self.volumes = self.tester.create_volumes(self.zone, size=size, count=volcount, timepergig=timepergig)
self.tester.delete_volumes(self.volumes, poll_interval=5, timeout=deletetimeout)
self.status('qa_214_test1: Completed:'+str(x)+'/'+str(testcount)+' tests',testcolor=TestColor.get_canned_color('whiteonblue'))
def qa_214_test4(self,
volcount=5,
snapcount=2,
testcount=5,
size=1,
timepergig=120,
deletetimeout=120):
"""
Definition: repeat: (create volumes V -> Create snapshots S from volumes V,
create volume SV from snapshot S, delete volume V,
delete snapshot S, delete volume SV)
:param volcount: integer how many concurrent volumes to create/delete
:param testcount: integer how many times to repeat this test
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
wait_on_progress = 15 * snapcount
for x in xrange(0,testcount):
volumes=[]
snaps=[]
self.status("\'qa_214_test4\' number:"+str(x)+"/"+str(testcount)+", volcount:"+str(volcount)+", snapcount:"+str(snapcount)+", size:"+str(size))
self.status('Creating '+str(volcount)+' new volumes...')
volumes = self.tester.create_volumes(self.zone, size=size, count=volcount, timepergig=timepergig)
for vol in volumes:
v_index = volumes.index(vol)+1
self.status('Creating '+str(snapcount)+' snapshots from our new volume ('+str(v_index)+'/'+str(len(volumes))+'):'+str(vol.id) )
snaps.extend(self.tester.create_snapshots(vol, count=snapcount, wait_on_progress=wait_on_progress, monitor_to_state=False))
for snap in snaps:
s_index = snaps.index(snap)+1
self.status('Creating '+str(volcount)+' volumes from our new snapshot('+str(s_index)+'/'+str(len(snaps))+'):'+str(snap.id) )
volumes.extend(self.tester.create_volumes(self.zone, count=volcount, snapshot=snap, timepergig=timepergig))
self.status('Test#'+str(x)+': Main block of test complete, deleting '+str(len(volumes))+' volumes, and '+str(len(snaps))+' snapshots')
self.tester.delete_volumes(volumes, poll_interval=5, timeout=deletetimeout)
self.tester.delete_snapshot(snaps, basetimeout=deletetimeout)
self.status('qa_214_test4: Completed:'+str(x)+'/'+str(testcount)+' tests',testcolor=TestColor.get_canned_color('whiteonblue'))
def qa_214_test3(self,
volcount=5,
testcount=5,
size=1,
timepergig=120,
deletetimeout=120):
"""
Definition: repeat : ( create volume Z -> attach volume Z -> detach volume Z -> delete volume Z)
:param volcount: integer how many concurrent volumes to create/delete
:param testcount: integer how many times to repeat this test
:param size: integer size of volume(s) in GB
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
instance = self.instance or self.launch_test_instance()
for x in xrange(0,testcount):
self.status("\'qa_214_test3\' number:"+str(x)+"/"+str(testcount))
volumes = self.tester.create_volumes(self.zone, size=size, count=volcount, timepergig=timepergig)
for volume in volumes:
instance.attach_volume(volume)
time.sleep(1)
instance.detach_euvolume(volume)
self.tester.delete_volumes(volumes, poll_interval=5, timeout=deletetimeout)
self.status('qa_214_test3: Completed:'+str(x)+'/'+str(testcount)+' tests',testcolor=TestColor.get_canned_color('whiteonblue'))
def reboot_sc_machine_verify_post_reboot(self, timeout=600):
"""
Definition: Restart the eucalyptus storage controller service
"""
self.print_all_test_resources()
# List to delay restart of storage controllers co-located with a cloud controller
debug_str = ""
# first make sure everything is good before we star the test...
self.tester.service_manager.all_services_operational()
storage_controllers = []
self.zones = self.zones or self.tester.get_zones()
for zone in self.zones:
self.debug("Getting storage controllers for :" + str(zone))
storage_controllers.extend(self.tester.service_manager.get_all_storage_controllers(partition=zone))
self.tester.service_manager.print_services_list(storage_controllers)
if not storage_controllers:
raise Exception("Storage controller list was not populated for zones:" + ",".join(self.zones))
for sc in storage_controllers:
debug_str = ""
all_services_on_sc = self.tester.service_manager.get_all_services_by_filter(hostname=sc.hostname)
for service in all_services_on_sc:
debug_str += "(" + str(service.hostname) + ":" + service.type + "), "
self.status(
"Now rebooting machine hosting services:" + str(debug_str),
testcolor=TestColor.get_canned_color("whiteonblue"),
)
sc.machine.reboot()
if not storage_controllers:
raise Exception("Storage controller list post reboot was not populated for zones:" + ",".join(self.zones))
start = time.time()
elapsed = 0
waiting = copy.copy(storage_controllers)
while elapsed < timeout and waiting:
elapsed = int(time.time() - start)
for sc in waiting:
if self.tester.ping(sc.hostname, poll_count=1):
waiting.remove(sc)
if waiting:
debug_str = ""
for sc in waiting:
debug_str += " " + str(sc.hostname) + ","
self.debug(
"Waiting on SC's to become reachable post reboot:" + str(debug_str) + ", elapsed:" + str(elapsed)
)
time.sleep(10)
else:
self.status(
"All SC machines are reachable again, now wait for SSH...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
break
if waiting:
raise ("SC machines were not reachable after: " + str(elapsed) + " seconds:" + str(debug_str))
if not storage_controllers:
raise Exception("Storage controller list was not populated after waiting for reachable")
start = time.time()
elapsed = 0
waiting_for_ssh = copy.copy(storage_controllers)
debug_str = ""
while elapsed < 90 and waiting_for_ssh:
elapsed = int(time.time() - start)
for sc in waiting_for_ssh:
try:
sc.machine.refresh_ssh()
sc.machine.cmd(" ")
waiting_for_ssh.remove(sc)
except Exception, e:
self.debug("Failed to refresh ssh to:" + str(sc.hostname) + ", err:" + str(e))
if waiting_for_ssh:
debug_str = ""
for sc in waiting:
debug_str += " " + str(sc.hostname) + ","
self.debug("Waiting on SSH connections to SCs:" + str(debug_str) + ", elapsed:" + str(elapsed))
time.sleep(10)
else:
self.status(
"All SC machines have established SSH connections, now wait on services to come back...",
testcolor=TestColor.get_canned_color("whiteonblue"),
)
break
for x in xrange(0, self.args.volsperinstance):
for vol in self.volumes:
if vol.zone == instance.placement and vol.status == "available":
try:
instance.attach_volume(vol, timeout=90)
except ec2ops.VolumeStateException, vse:
self.status(
"This is a temp work around for testing, this is to avoid bug euca-5297" + str(vse),
testcolor=TestColor.get_canned_color("failred"),
)
time.sleep(10)
self.tester.monitor_euvolumes_to_status(
[vol], status="in-use", attached_status="attached", timeout=60
)
self.status(
"'pre_service_restart_attach_all_volumes' done", testcolor=TestColor.get_canned_color("whiteonblue")
)
def pretest4_pre_service_restart_create_snap_per_zone(self, timepergig=120):
"""
Definition: Create a single snapshot. This tests should be completed per zone prior to restarting services.
:param timepergig: integer time allowed per GB in seconds during creation
:param deletetimeout: integer timeout in seconds waiting for volume to transition to 'deleted' state
"""
snaps = []
snapvols = []
# Add one volume from each zone...
for zone in self.zones:
for vol in self.volumes:
if vol.zone == zone:
snapvols.append(vol)