def test_exception_handling(self):
"""
Test if the scheduled job can handle exceptions
"""
def raise_an_exception(*args, **kwargs):
raise RuntimeError('Emulated snapshot delete error')
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1, vdisk_2 = structure['vdisks'].values()
storagedriver_1 = structure['storagedrivers'][1]
vdisks = [vdisk_1, vdisk_2]
for vdisk in vdisks:
[
dynamic for dynamic in vdisk._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
for i in xrange(0, 2):
metadata = {
'label':
str(i),
'is_consistent':
False,
'is_sticky':
False,
'timestamp':
str((int(time.time() -
datetime.timedelta(2).total_seconds() - i)))
}
snapshot_id = VDiskController.create_snapshot(
vdisk.guid, metadata)
if vdisk == vdisk_1:
StorageRouterClient.delete_snapshot_callbacks[
vdisk.volume_id] = {
snapshot_id: raise_an_exception
}
with self.assertRaises(RuntimeError):
GenericController.delete_snapshots_storagedriver(
storagedriver_guid=storagedriver_1.guid)
self.assertEqual(1, len(vdisk_2.snapshot_ids),
'One snapshot should be removed for vdisk 2')
self.assertEqual(2, len(vdisk_1.snapshot_ids),
'No snapshots should be removed for vdisk 1')
def test_scrubbing_exception_handling(self):
"""
Test if the scheduled job can handle scrub related exceptions
"""
def raise_an_exception(*args, **kwargs):
raise RuntimeError(SCRUB_VDISK_EXCEPTION_MESSAGE)
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1 = structure['vdisks'][1]
storagedriver_1 = structure['storagedrivers'][1]
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
for i in xrange(0, 2):
metadata = {
'label':
str(i),
'is_consistent':
False,
'is_sticky':
False,
'timestamp':
str((int(time.time() - datetime.timedelta(2).total_seconds() -
i)))
}
snapshot_id = VDiskController.create_snapshot(
vdisk_1.guid, metadata)
StorageRouterClient.delete_snapshot_callbacks[
vdisk_1.volume_id] = {
snapshot_id: raise_an_exception
}
GenericController.delete_snapshots_storagedriver(
storagedriver_guid=storagedriver_1.guid)
self.assertEqual(2, len(vdisk_1.snapshot_ids),
'No snapshots should be removed for vdisk 1')
def test_snapshot_sticky(self):
"""
is_sticky: True --> Sticky snapshots of any kind should never be deleted (Only possible to delete manually)
"""
minute = 60
hour = minute * 60
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
base = datetime.datetime.now().date()
vdisk_1 = structure['vdisks'][1]
storagedriver_1 = structure['storagedrivers'][1]
label = 'c'
# Extra time to add to the hourly timestamps
additional_time = minute * 30
# Hours to create a snapshot on
sticky_hours = [2]
consistent_hours = [2]
inconsistent_hours = []
# Snapshot details
is_sticky = len(sticky_hours) > 0
is_consistent = len(consistent_hours) > 0
is_automatic = False
for day in xrange(35):
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * day)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
day,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
self._print_message('- Deleting snapshots')
GenericController.delete_snapshots_storagedriver(
storagedriver_guid=storagedriver_1.guid,
timestamp=base_timestamp + (minute * 30))
self._validate(vdisk=vdisk_1,
current_day=day,
base_date=base,
sticky_hours=sticky_hours,
consistent_hours=consistent_hours,
inconsistent_hours=inconsistent_hours)
self._print_message('- Creating snapshots')
for x in consistent_hours + inconsistent_hours:
timestamp = base_timestamp + (hour * x) + additional_time
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_{0}_{1}:00'.format(
label, x),
'is_sticky':
is_sticky,
'timestamp':
str(timestamp),
'is_automatic':
is_automatic,
'is_consistent':
is_consistent
})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1 = structure['vdisks'][1]
storagedriver_1 = structure['storagedrivers'][1]
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
base = datetime.datetime.now().date()
minute = 60
hour = minute * 60
consistent_hours = [6, 12, 18]
inconsistent_hours = xrange(2, 23)
for day in xrange(0, 35):
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * day)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
day,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
GenericController.delete_snapshots_storagedriver(
storagedriver_guid=storagedriver_1.guid,
timestamp=base_timestamp + (minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk=vdisk_1,
current_day=day,
base_date=base,
sticky_hours=[],
consistent_hours=consistent_hours,
inconsistent_hours=inconsistent_hours)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in inconsistent_hours:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp)
})
if h in consistent_hours:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts)
})
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1 = structure['vdisks'][1]
storagedriver_1 = structure['storagedrivers'][1]
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
base = datetime.datetime.now().date()
base_timestamp = self._make_timestamp(base, datetime.timedelta(1))
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp)
})
structure = DalHelper.build_dal_structure(
structure={'vdisks': [(2, 1, 1, 1)]}, previous_structure=structure)
clone_vdisk = structure['vdisks'][2]
base_snapshot_guid = vdisk_1.snapshot_ids[0] # Oldest
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.save()
for day in range(10):
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * day)
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp)
})
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * 2)
GenericController.delete_snapshots_storagedriver(
storagedriver_guid=storagedriver_1.guid,
timestamp=base_timestamp + (minute * 30))
self.assertIn(
base_snapshot_guid, vdisk_1.snapshot_ids,
'Snapshot was deleted while there are still clones of it')
