def ovs_3700_validate_test():
"""
Validate something test
"""
def _get_scrubber_log_size():
scrubber_log_name = '/var/log/upstart/ovs-scrubber.log'
if os.path.exists(scrubber_log_name):
return os.stat(scrubber_log_name).st_size
return 0
loop = 'loop0'
vpool = GeneralVPool.get_vpool_by_name(TestVDisk.vpool_name)
vdisk = GeneralVDisk.create_volume(size=2, vpool=vpool, name='ovs-3700-disk', loop_device=loop, wait=True)
GeneralVDisk.create_snapshot(vdisk=vdisk, snapshot_name='snap0')
GeneralVDisk.generate_hash_file(full_name='/mnt/{0}/{1}_{2}.txt'.format(loop, vdisk.name, '1'), size=512)
GeneralVDisk.create_snapshot(vdisk=vdisk, snapshot_name='snap1')
GeneralVDisk.generate_hash_file(full_name='/mnt/{0}/{1}_{2}.txt'.format(loop, vdisk.name, '2'), size=512)
GeneralVDisk.delete_snapshot(disk=vdisk, snapshot_name='snap1')
GeneralVDisk.generate_hash_file(full_name='/mnt/{0}/{1}_{2}.txt'.format(loop, vdisk.name, '3'), size=512)
GeneralVDisk.create_snapshot(vdisk=vdisk, snapshot_name='snap2')
pre_scrubber_logsize = _get_scrubber_log_size()
ScheduledTaskController.gather_scrub_work()
post_scrubber_logsize = _get_scrubber_log_size()
GeneralVDisk.delete_volume(vdisk=vdisk, vpool=vpool, loop_device=loop)
assert post_scrubber_logsize > pre_scrubber_logsize, "Scrubber actions were not logged!"
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
failure_domain = FailureDomain()
failure_domain.name = 'Test'
failure_domain.save()
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.path = '/dev/non-existent'
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = 'disk_partition_id'
disk_partition.disk = disk
disk_partition.path = '/dev/disk/non-existent'
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[
dynamic for dynamic in disk._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
print ''
print 'Day cycle: {0}: {1}'.format(
d,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d'))
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(
str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(
str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp),
'machineguid':
None
})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts),
'machineguid':
None
})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistents
every now an then. The delelete policy is exectued every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and an additional disk
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
debug = True
amount_of_days = 50
base = datetime.now().date()
day = timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = DeleteSnapshots._make_timestamp(base, day * d)
print ''
print 'Day cycle: {}: {}'.format(
d, datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')
)
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.deletescrubsnapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistents
every now an then. The delelete policy is exectued every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and an additional disk
vpool = VPool()
vpool.name = 'vpool'
vpool.backend_type = BackendType()
vpool.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = PMachine()
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = PMachine()
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
debug = True
amount_of_days = 50
now = int(mktime(datetime.now().date().timetuple())) # Last night
minute = 60
hour = minute * 60
day = hour * 24
for d in xrange(0, amount_of_days):
base_timestamp = now + (day * d)
print ''
print 'Day cycle: {}: {}'.format(
d,
datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')
)
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.deletescrubsnapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_3]: # [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, now, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{}:00'.format(str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{}:30'.format(str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_i_{}:00'.format(str(h)),
'is_consistent': False,
'timestamp': timestamp,
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_c_{}:30'.format(str(h)),
'is_consistent': True,
'timestamp': ts,
'machineguid': None})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and an additional disk
failure_domain = FailureDomain()
failure_domain.name = "Test"
failure_domain.save()
backend_type = BackendType()
backend_type.name = "BackendType"
backend_type.code = "BT"
backend_type.save()
vpool = VPool()
vpool.name = "vpool"
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = "PMachine"
pmachine.username = "******"
pmachine.ip = "127.0.0.1"
pmachine.hvtype = "VMWARE"
pmachine.save()
storage_router = StorageRouter()
storage_router.name = "storage_router"
storage_router.ip = "127.0.0.1"
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = "physical_disk_1"
disk.path = "/dev/non-existent"
disk.size = 500 * 1024 ** 3
disk.state = "OK"
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = "disk_partition_id"
disk_partition.disk = disk
disk_partition.path = "/dev/disk/non-existent"
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = "OK"
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = "/var/tmp"
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = "vmachine_1"
vmachine_1.devicename = "dummy"
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = "vdisk_1_1"
vdisk_1_1.volume_id = "vdisk_1_1"
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = "dummy"
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = "vdisk_1_2"
vdisk_1_2.volume_id = "vdisk_1_2"
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = "dummy"
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = "vmachine_2"
vmachine_2.devicename = "dummy"
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = "vdisk_2_1"
vdisk_2_1.volume_id = "vdisk_2_1"
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = "dummy"
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = "vdisk_3"
vdisk_3.volume_id = "vdisk_3"
vdisk_3.vpool = vpool
vdisk_3.devicename = "dummy"
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == "snapshots"][0].timeout = 0
# Run the testing scenario
debug = True
amount_of_days = 50
base = datetime.now().date()
day = timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = DeleteSnapshots._make_timestamp(base, day * d)
print ""
print "Day cycle: {0}: {1}".format(d, datetime.fromtimestamp(base_timestamp).strftime("%Y-%m-%d"))
# At the start of the day, delete snapshot policy runs at 00:30
print "- Deleting snapshots"
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print "- Validating snapshots"
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print "- Creating snapshots"
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(
machineguid=vm.guid,
label="ss_i_{0}:00".format(str(h)),
is_consistent=False,
timestamp=timestamp,
)
if h in [6, 12, 18]:
ts = timestamp + (minute * 30)
VMachineController.snapshot(
machineguid=vm.guid, label="ss_c_{0}:30".format(str(h)), is_consistent=True, timestamp=ts
)
VDiskController.create_snapshot(
diskguid=vdisk_3.guid,
metadata={
"label": "ss_i_{0}:00".format(str(h)),
"is_consistent": False,
"timestamp": str(timestamp),
"machineguid": None,
},
)
if h in [6, 12, 18]:
ts = timestamp + (minute * 30)
VDiskController.create_snapshot(
diskguid=vdisk_3.guid,
metadata={
"label": "ss_c_{0}:30".format(str(h)),
"is_consistent": True,
"timestamp": str(ts),
"machineguid": None,
},
)
def test_scrubbing(self):
"""
Validates the scrubbing workflow
* Scenario 1: Validate disabled scrub task and single vDisk scrub logic
* Scenario 2: 1 vPool, 10 vDisks, 1 scrub role
Scrubbing fails for 5 vDisks, check if scrubbing completed for all other vDisks
Run scrubbing a 2nd time and verify scrubbing now works for failed vDisks
* Scenario 3: 1 vPool, 10 vDisks, 5 scrub roles
Check if vDisks are divided among all threads
* Scenario 4: 3 vPools, 9 vDisks, 5 scrub roles
Validate 6 threads will be spawned and used out of a potential of 15 (5 scrub roles * 3 vPools)
We limit max amount of threads spawned per vPool to 2 in case 3 to 5 vPools are present
"""
_ = self
for i in xrange(1, 6):
Configuration.set('/ovs/framework/hosts/{0}/ports'.format(i),
{'storagedriver': [10000, 10100]})
##############
# Scenario 1 #
##############
structure = Helper.build_service_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk = structure['vdisks'][1]
vpool = structure['vpools'][1]
storagerouter = structure['storagerouters'][1]
System._machine_id = {storagerouter.ip: '1'}
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid),
json.dumps({}, indent=4),
raw=True)
LockedClient.scrub_controller = {
'possible_threads': None,
'volumes': {},
'waiter': Waiter(1)
}
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {
'success': False,
'scrub_work': [0]
}
with self.assertRaises(Exception) as raise_info:
VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid)
self.assertIn(vdisk.name, raise_info.exception.message)
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {
'success': True,
'scrub_work': [0]
}
VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid)
with vdisk.storagedriver_client.make_locked_client(
vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg=
'Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}'
.format(vdisk.name, 0))
##############
# Scenario 2 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 1, 1, 1),
(5, 1, 1, 1), (6, 1, 1, 1), (7, 1, 1, 1), (8, 1, 1, 1),
(9, 1, 1, 1), (10, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpool = structure['vpools'][1]
vdisks = structure['vdisks']
storagerouter = structure['storagerouters'][1]
System._machine_id = {storagerouter.ip: '1'}
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid),
json.dumps({}, indent=4),
raw=True)
LockedClient.scrub_controller = {
'possible_threads':
['scrub_{0}_{1}'.format(vpool.guid, storagerouter.guid)],
'volumes': {},
'waiter':
Waiter(1)
}
failed_vdisks = []
successful_vdisks = []
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
success = vdisk_id % 2 == 0
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {
'success': success,
'scrub_work': range(vdisk_id)
}
if success is True:
successful_vdisks.append(vdisk)
else:
failed_vdisks.append(vdisk)
# Execute scrubbing a 1st time
with self.assertRaises(Exception) as raise_info:
ScheduledTaskController.execute_scrub()
for vdisk in failed_vdisks:
self.assertIn(vdisk.name, raise_info.exception.message)
# Validate expected successful vDisks
for vdisk in successful_vdisks:
with vdisk.storagedriver_client.make_locked_client(
vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg='Scrubbed vDisk {0} does still have scrubbing work left'
.format(vdisk.name))
# Validate expected failed vDisks
for vdisk in failed_vdisks:
with vdisk.storagedriver_client.make_locked_client(
vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=int(vdisk.name),
msg=
'Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}'
.format(vdisk.name, int(vdisk.name)))
# Execute scrubbing again
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller['volumes'][
vdisk.volume_id]['success'] = True
ScheduledTaskController.execute_scrub()
for vdisk in vdisks.values():
with vdisk.storagedriver_client.make_locked_client(
vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg=
'Scrubbed vDisk {0} does still have scrubbing work left after scrubbing a 2nd time'
.format(vdisk.name))
##############
# Scenario 3 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 1, 1, 1),
(5, 1, 1, 1), (6, 1, 1, 1), (7, 1, 1, 1), (8, 1, 1, 1),
(9, 1, 1, 1), (10, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1, 2, 3, 4, 5],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpool = structure['vpools'][1]
vdisks = structure['vdisks']
storagerouters = structure['storagerouters']
System._machine_id = dict(
(sr.ip, sr.machine_id) for sr in storagerouters.values())
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid),
json.dumps({}, indent=4),
raw=True)
thread_names = [
'scrub_{0}_{1}'.format(vpool.guid, storagerouter.guid)
for storagerouter in storagerouters.values()
]
LockedClient.scrub_controller = {
'possible_threads': thread_names,
'volumes': {},
'waiter': Waiter(len(thread_names))
}
LockedClient.thread_names = thread_names[:]
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {
'success': True,
'scrub_work': range(vdisk_id)
}
ScheduledTaskController.execute_scrub()
self.assertEqual(first=len(LockedClient.thread_names),
second=0,
msg='Not all threads have been used in the process')
##############
# Scenario 4 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure({
'vpools': [1, 2, 3],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 2, 2, 2),
(5, 2, 2, 2), (6, 2, 2, 2), (7, 3, 3, 3), (8, 3, 3, 3),
(9, 3, 3, 3)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1), (2, 2),
(3, 3)], # (<id>, <storagedriver_id>)
'storagerouters': [1, 2, 3, 4, 5],
'storagedrivers': [(1, 1, 1), (2, 2, 1), (3, 3, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpools = structure['vpools']
vdisks = structure['vdisks']
storagerouters = structure['storagerouters']
thread_names = []
for vpool in vpools.values():
Configuration.set(
'/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid),
json.dumps({}, indent=4),
raw=True)
for storagerouter in storagerouters.values():
thread_names.append('scrub_{0}_{1}'.format(
vpool.guid, storagerouter.guid))
LockedClient.scrub_controller = {
'possible_threads': thread_names,
'volumes': {},
'waiter': Waiter(len(thread_names) - 9)
}
LockedClient.thread_names = thread_names[:]
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {
'success': True,
'scrub_work': range(vdisk_id)
}
ScheduledTaskController.execute_scrub()
self.assertEqual(
first=len(LockedClient.thread_names),
second=
9, # 5 srs * 3 vps = 15 threads, but only 2 will be spawned per vPool --> 15 - 6 = 9 left
msg='Not all threads have been used in the process')
# 3 vPools will cause the scrubber to only launch 2 threads per vPool --> 1 possible thread should be unused per vPool
for vpool in vpools.values():
threads_left = [
thread_name for thread_name in LockedClient.thread_names
if vpool.guid in thread_name
]
self.assertEqual(
first=len(threads_left),
second=3,
msg='Unexpected amount of threads left for vPool {0}'.format(
vpool.name))
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
failure_domain = FailureDomain()
failure_domain.name = 'Test'
failure_domain.save()
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.path = '/dev/non-existent'
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = 'disk_partition_id'
disk_partition.disk = disk
disk_partition.path = '/dev/disk/non-existent'
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
print ''
print 'Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d'))
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_timestamp = self._make_timestamp(base, day * 2)
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
self.assertIn(base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots], 'Snapshot was deleted while there are still clones of it')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def test_scrubbing(self):
"""
Validates the scrubbing workflow
* Scenario 1: Validate disabled scrub task and single vDisk scrub logic
* Scenario 2: 1 vPool, 10 vDisks, 1 scrub role
Scrubbing fails for 5 vDisks, check if scrubbing completed for all other vDisks
Run scrubbing a 2nd time and verify scrubbing now works for failed vDisks
* Scenario 3: 1 vPool, 10 vDisks, 5 scrub roles
Check if vDisks are divided among all threads
* Scenario 4: 3 vPools, 9 vDisks, 5 scrub roles
Validate 6 threads will be spawned and used out of a potential of 15 (5 scrub roles * 3 vPools)
We limit max amount of threads spawned per vPool to 2 in case 3 to 5 vPools are present
"""
_ = self
for i in xrange(1, 6):
Configuration.set('/ovs/framework/hosts/{0}/ports'.format(i), {'storagedriver': [10000, 10100]})
##############
# Scenario 1 #
##############
structure = Helper.build_service_structure(
{'vpools': [1],
'vdisks': [(1, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk = structure['vdisks'][1]
vpool = structure['vpools'][1]
storagerouter = structure['storagerouters'][1]
System._machine_id = {storagerouter.ip: '1'}
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid), json.dumps({}, indent=4), raw=True)
LockedClient.scrub_controller = {'possible_threads': None,
'volumes': {},
'waiter': Waiter(1)}
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': False,
'scrub_work': [0]}
with self.assertRaises(Exception) as raise_info:
VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid)
self.assertIn(vdisk.name, raise_info.exception.message)
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': True,
'scrub_work': [0]}
VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid)
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg='Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}'.format(vdisk.name, 0))
##############
# Scenario 2 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure(
{'vpools': [1],
'vdisks': [(1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 1, 1, 1), (5, 1, 1, 1),
(6, 1, 1, 1), (7, 1, 1, 1), (8, 1, 1, 1), (9, 1, 1, 1), (10, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpool = structure['vpools'][1]
vdisks = structure['vdisks']
storagerouter = structure['storagerouters'][1]
System._machine_id = {storagerouter.ip: '1'}
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid), json.dumps({}, indent=4), raw=True)
LockedClient.scrub_controller = {'possible_threads': ['scrub_{0}_{1}'.format(vpool.guid, storagerouter.guid)],
'volumes': {},
'waiter': Waiter(1)}
failed_vdisks = []
successful_vdisks = []
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
success = vdisk_id % 2 == 0
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': success,
'scrub_work': range(vdisk_id)}
if success is True:
successful_vdisks.append(vdisk)
else:
failed_vdisks.append(vdisk)
# Execute scrubbing a 1st time
with self.assertRaises(Exception) as raise_info:
ScheduledTaskController.execute_scrub()
for vdisk in failed_vdisks:
self.assertIn(vdisk.name, raise_info.exception.message)
# Validate expected successful vDisks
for vdisk in successful_vdisks:
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg='Scrubbed vDisk {0} does still have scrubbing work left'.format(vdisk.name))
# Validate expected failed vDisks
for vdisk in failed_vdisks:
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(first=len(locked_client.get_scrubbing_workunits()),
second=int(vdisk.name),
msg='Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}'.format(vdisk.name, int(vdisk.name)))
# Execute scrubbing again
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller['volumes'][vdisk.volume_id]['success'] = True
ScheduledTaskController.execute_scrub()
for vdisk in vdisks.values():
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg='Scrubbed vDisk {0} does still have scrubbing work left after scrubbing a 2nd time'.format(vdisk.name))
##############
# Scenario 3 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure(
{'vpools': [1],
'vdisks': [(1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 1, 1, 1), (5, 1, 1, 1),
(6, 1, 1, 1), (7, 1, 1, 1), (8, 1, 1, 1), (9, 1, 1, 1), (10, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouters': [1, 2, 3, 4, 5],
'storagedrivers': [(1, 1, 1)]} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpool = structure['vpools'][1]
vdisks = structure['vdisks']
storagerouters = structure['storagerouters']
System._machine_id = dict((sr.ip, sr.machine_id) for sr in storagerouters.values())
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid), json.dumps({}, indent=4), raw=True)
thread_names = ['scrub_{0}_{1}'.format(vpool.guid, storagerouter.guid) for storagerouter in storagerouters.values()]
LockedClient.scrub_controller = {'possible_threads': thread_names,
'volumes': {},
'waiter': Waiter(len(thread_names))}
LockedClient.thread_names = thread_names[:]
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': True,
'scrub_work': range(vdisk_id)}
ScheduledTaskController.execute_scrub()
self.assertEqual(first=len(LockedClient.thread_names),
second=0,
msg='Not all threads have been used in the process')
##############
# Scenario 4 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure(
{'vpools': [1, 2, 3],
'vdisks': [(1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 2, 2, 2), (5, 2, 2, 2),
(6, 2, 2, 2), (7, 3, 3, 3), (8, 3, 3, 3), (9, 3, 3, 3)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1), (2, 2), (3, 3)], # (<id>, <storagedriver_id>)
'storagerouters': [1, 2, 3, 4, 5],
'storagedrivers': [(1, 1, 1), (2, 2, 1), (3, 3, 1)]} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpools = structure['vpools']
vdisks = structure['vdisks']
storagerouters = structure['storagerouters']
thread_names = []
for vpool in vpools.values():
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid), json.dumps({}, indent=4), raw=True)
for storagerouter in storagerouters.values():
thread_names.append('scrub_{0}_{1}'.format(vpool.guid, storagerouter.guid))
LockedClient.scrub_controller = {'possible_threads': thread_names,
'volumes': {},
'waiter': Waiter(len(thread_names) - 9)}
LockedClient.thread_names = thread_names[:]
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': True,
'scrub_work': range(vdisk_id)}
ScheduledTaskController.execute_scrub()
self.assertEqual(first=len(LockedClient.thread_names),
second=9, # 5 srs * 3 vps = 15 threads, but only 2 will be spawned per vPool --> 15 - 6 = 9 left
msg='Not all threads have been used in the process')
# 3 vPools will cause the scrubber to only launch 2 threads per vPool --> 1 possible thread should be unused per vPool
for vpool in vpools.values():
threads_left = [thread_name for thread_name in LockedClient.thread_names if vpool.guid in thread_name]
self.assertEqual(first=len(threads_left),
second=3,
msg='Unexpected amount of threads left for vPool {0}'.format(vpool.name))
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[
dynamic for dynamic in vdisk_1_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp),
'machineguid':
None
})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp),
'machineguid':
None
})
base_timestamp = self._make_timestamp(base, day * 2)
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
self.assertIn(
base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots],
'Snapshot was deleted while there are still clones of it')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
d,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp),
'machineguid':
None
})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts),
'machineguid':
None
})
