def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1 = structure['vdisks'][1]
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
base = datetime.datetime.now().date()
minute = 60
hour = minute * 60
consistent_hours = [6, 12, 18]
inconsistent_hours = xrange(2, 23)
for day in xrange(0, 35):
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * day)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
day,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
GenericController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk=vdisk_1,
current_day=day,
base_date=base,
sticky_hours=[],
consistent_hours=consistent_hours,
inconsistent_hours=inconsistent_hours)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in inconsistent_hours:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp)
})
if h in consistent_hours:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts)
})
def test_different_snapshot_flags(self):
"""
Tests the GenericController.delete_snapshots() call, but with different snapshot flags
Scenario 1: is_automatic: True, is_consistent: True --> Automatically created consistent snapshots should be deleted
Scenario 2: is_automatic: True, is_consistent: False --> Automatically created non-consistent snapshots should be deleted
Scenario 3: is_automatic: False, is_consistent: True --> Manually created consistent snapshots should be deleted
Scenario 4: is_automatic: False, is_consistent: False --> Manually created non-consistent snapshots should be deleted
Scenario 5: is_sticky: True --> Sticky snapshots of any kind should never be deleted (Only possible to delete manually)
"""
minute = 60
hour = minute * 60
for scenario in range(5):
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
base = datetime.datetime.now().date()
vdisk_1 = structure['vdisks'][1]
is_sticky = False
sticky_hours = []
if scenario % 2 == 0:
label = 'c'
additional_time = minute * 30
consistent_hours = [2]
inconsistent_hours = []
else:
label = 'i'
additional_time = 0
consistent_hours = []
inconsistent_hours = [2]
if scenario == 4:
is_sticky = True
sticky_hours = consistent_hours
for day in xrange(35):
base_timestamp = self._make_timestamp(
base,
datetime.timedelta(1) * day)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
day,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
self._print_message('- Deleting snapshots')
GenericController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
self._validate(vdisk=vdisk_1,
current_day=day,
base_date=base,
sticky_hours=sticky_hours,
consistent_hours=consistent_hours,
inconsistent_hours=inconsistent_hours)
self._print_message('- Creating snapshots')
for x in consistent_hours + inconsistent_hours:
timestamp = base_timestamp + (hour * x) + additional_time
VDiskController.create_snapshot(
vdisk_guid=vdisk_1.guid,
metadata={
'label': 'ss_{0}_{1}:00'.format(label, x),
'is_sticky': is_sticky,
'timestamp': str(timestamp),
'is_automatic': scenario in [0, 1],
'is_consistent': len(consistent_hours) > 0
})
self.persistent._clean()
self.volatile._clean()
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_timestamp = self._make_timestamp(base, day * 2)
GenericController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
self.assertIn(base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots], 'Snapshot was deleted while there are still clones of it')
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1 = structure['vdisks'][1]
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
base = datetime.datetime.now().date()
base_timestamp = self._make_timestamp(base, datetime.timedelta(1))
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp)
})
structure = DalHelper.build_dal_structure(
structure={'vdisks': [(2, 1, 1, 1)]}, previous_structure=structure)
clone_vdisk = structure['vdisks'][2]
base_snapshot_guid = vdisk_1.snapshot_ids[0] # Oldest
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.save()
for day in range(10):
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * day)
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp)
})
base_timestamp = self._make_timestamp(base,
datetime.timedelta(1) * 2)
GenericController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
self.assertIn(
base_snapshot_guid, vdisk_1.snapshot_ids,
'Snapshot was deleted while there are still clones of it')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
GenericController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
