def test_delete_snapshot_scrubbing_lock(self):
"""
Tests the skip-if-scrubbed logic
"""
snapshot_while_scrub_results = []
def delete_snapshot_while_scrubbing(*args, **kwargs):
_ = args, kwargs
try:
snapshot_while_scrub_results.append(VDiskController.delete_snapshot(vdisk_1.guid, vdisk_1.snapshot_ids[0]))
except RuntimeError as ex:
snapshot_while_scrub_results.append(ex)
structure = DalHelper.build_dal_structure(
{'vpools': [1],
'vdisks': [(1, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisks = structure['vdisks']
vdisk_1 = vdisks[1]
# Create automatic snapshot for both vDisks
success, fail = GenericController.snapshot_all_vdisks()
self.assertEqual(first=len(fail), second=0, msg='Expected 0 failed snapshots')
self.assertEqual(first=len(success), second=1, msg='Expected 1 successful snapshots')
self.assertEqual(first=len(vdisk_1.snapshot_ids), second=1, msg='Expected 1 snapshot ID for vDisk {0}'.format(vdisk_1.name))
self.assertEqual(first=len(vdisk_1.snapshots), second=1, msg='Expected 1 snapshot for vDisk {0}'.format(vdisk_1.name))
proxy_names, thread_names, vdisk_namespaces = self.generate_scrub_related_info(structure)
LockedClient.scrub_controller = {'possible_threads': thread_names,
'volumes': {},
'waiter': Waiter(len(thread_names[0:1]))} # only 1 disks -> 1 thread
# Scrub all volumes
for vdisk_id, vdisk in vdisks.iteritems():
LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': True,
'scrub_work': range(vdisk_id)}
hooks = {'post_vdisk_scrub_registration': delete_snapshot_while_scrubbing} # Make the scrubber wait
ScrubShared._test_hooks.update(hooks)
GenericController.execute_scrub()
# Ensure delete snapshot fails for vdisk_1 because it is being scrubbed
result_while_scrub = snapshot_while_scrub_results[0]
self.assertIsInstance(result_while_scrub, Exception, 'Expected an exception to have occurred')
self.assertEqual(str(result_while_scrub), 'VDisk is being scrubbed. Unable to remove snapshots at this time', 'Excpetion should be about disk being scrubbed')
self.assertEqual(first=len(vdisk_1.snapshot_ids), second=1, msg='Expected 1 snapshot ID for vDisk {0}'.format(vdisk_1.name))
self.assertEqual(first=len(vdisk_1.snapshots), second=1, msg='Expected 1 snapshot for vDisk {0}'.format(vdisk_1.name))
def execute_scrubbing():
"""
Execute scrubbing on the cluster
:return:
"""
return GenericController.execute_scrub()
def test_scrubbing(self):
"""
Validates the scrubbing workflow
* Scenario 1: Validate disabled scrub task and single vDisk scrub logic
* Scenario 2: 1 vPool, 10 vDisks, 1 scrub role
Scrubbing fails for 5 vDisks, check if scrubbing completed for all other vDisks
Run scrubbing a 2nd time and verify scrubbing now works for failed vDisks
* Scenario 3: 1 vPool, 10 vDisks, 5 scrub roles
Check if vDisks are divided among all threads
* Scenario 4: 3 vPools, 9 vDisks, 5 scrub roles
Validate 6 threads will be spawned and used out of a potential of 15 (5 scrub roles * 3 vPools)
We limit max amount of threads spawned per vPool to 2 in case 3 to 5 vPools are present
"""
_ = self
for i in xrange(1, 6):
Configuration.set("/ovs/framework/hosts/{0}/ports".format(i), {"storagedriver": [10000, 10100]})
##############
# Scenario 1 #
##############
structure = Helper.build_service_structure(
{
"vpools": [1],
"vdisks": [(1, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
"mds_services": [(1, 1)], # (<id>, <storagedriver_id>)
"storagerouters": [1],
"storagedrivers": [(1, 1, 1)],
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk = structure["vdisks"][1]
vpool = structure["vpools"][1]
storagerouter = structure["storagerouters"][1]
System._machine_id = {storagerouter.ip: "1"}
Configuration.set(
"/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True
)
LockedClient.scrub_controller = {"possible_threads": None, "volumes": {}, "waiter": Waiter(1)}
LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": False, "scrub_work": [0]}
with self.assertRaises(Exception) as raise_info:
VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid)
self.assertIn(vdisk.name, raise_info.exception.message)
LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": True, "scrub_work": [0]}
VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid)
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg="Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}".format(
vdisk.name, 0
),
)
##############
# Scenario 2 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure(
{
"vpools": [1],
"vdisks": [
(1, 1, 1, 1),
(2, 1, 1, 1),
(3, 1, 1, 1),
(4, 1, 1, 1),
(5, 1, 1, 1),
(6, 1, 1, 1),
(7, 1, 1, 1),
(8, 1, 1, 1),
(9, 1, 1, 1),
(10, 1, 1, 1),
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
"mds_services": [(1, 1)], # (<id>, <storagedriver_id>)
"storagerouters": [1],
"storagedrivers": [(1, 1, 1)],
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpool = structure["vpools"][1]
vdisks = structure["vdisks"]
storagerouter = structure["storagerouters"][1]
System._machine_id = {storagerouter.ip: "1"}
Configuration.set(
"/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True
)
LockedClient.scrub_controller = {
"possible_threads": ["scrub_{0}_{1}".format(vpool.guid, storagerouter.guid)],
"volumes": {},
"waiter": Waiter(1),
}
failed_vdisks = []
successful_vdisks = []
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
success = vdisk_id % 2 == 0
LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {
"success": success,
"scrub_work": range(vdisk_id),
}
if success is True:
successful_vdisks.append(vdisk)
else:
failed_vdisks.append(vdisk)
# Execute scrubbing a 1st time
with self.assertRaises(Exception) as raise_info:
GenericController.execute_scrub()
for vdisk in failed_vdisks:
self.assertIn(vdisk.name, raise_info.exception.message)
# Validate expected successful vDisks
for vdisk in successful_vdisks:
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg="Scrubbed vDisk {0} does still have scrubbing work left".format(vdisk.name),
)
# Validate expected failed vDisks
for vdisk in failed_vdisks:
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=int(vdisk.name),
msg="Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}".format(
vdisk.name, int(vdisk.name)
),
)
# Execute scrubbing again
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller["volumes"][vdisk.volume_id]["success"] = True
GenericController.execute_scrub()
for vdisk in vdisks.values():
with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client:
self.assertEqual(
first=len(locked_client.get_scrubbing_workunits()),
second=0,
msg="Scrubbed vDisk {0} does still have scrubbing work left after scrubbing a 2nd time".format(
vdisk.name
),
)
##############
# Scenario 3 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure(
{
"vpools": [1],
"vdisks": [
(1, 1, 1, 1),
(2, 1, 1, 1),
(3, 1, 1, 1),
(4, 1, 1, 1),
(5, 1, 1, 1),
(6, 1, 1, 1),
(7, 1, 1, 1),
(8, 1, 1, 1),
(9, 1, 1, 1),
(10, 1, 1, 1),
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
"mds_services": [(1, 1)], # (<id>, <storagedriver_id>)
"storagerouters": [1, 2, 3, 4, 5],
"storagedrivers": [(1, 1, 1)],
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpool = structure["vpools"][1]
vdisks = structure["vdisks"]
storagerouters = structure["storagerouters"]
System._machine_id = dict((sr.ip, sr.machine_id) for sr in storagerouters.values())
Configuration.set(
"/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True
)
thread_names = [
"scrub_{0}_{1}".format(vpool.guid, storagerouter.guid) for storagerouter in storagerouters.values()
]
LockedClient.scrub_controller = {
"possible_threads": thread_names,
"volumes": {},
"waiter": Waiter(len(thread_names)),
}
LockedClient.thread_names = thread_names[:]
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": True, "scrub_work": range(vdisk_id)}
GenericController.execute_scrub()
self.assertEqual(
first=len(LockedClient.thread_names), second=0, msg="Not all threads have been used in the process"
)
##############
# Scenario 4 #
##############
self.volatile.clean()
self.persistent.clean()
structure = Helper.build_service_structure(
{
"vpools": [1, 2, 3],
"vdisks": [
(1, 1, 1, 1),
(2, 1, 1, 1),
(3, 1, 1, 1),
(4, 2, 2, 2),
(5, 2, 2, 2),
(6, 2, 2, 2),
(7, 3, 3, 3),
(8, 3, 3, 3),
(9, 3, 3, 3),
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
"mds_services": [(1, 1), (2, 2), (3, 3)], # (<id>, <storagedriver_id>)
"storagerouters": [1, 2, 3, 4, 5],
"storagedrivers": [(1, 1, 1), (2, 2, 1), (3, 3, 1)],
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vpools = structure["vpools"]
vdisks = structure["vdisks"]
storagerouters = structure["storagerouters"]
thread_names = []
for vpool in vpools.values():
Configuration.set(
"/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True
)
for storagerouter in storagerouters.values():
thread_names.append("scrub_{0}_{1}".format(vpool.guid, storagerouter.guid))
LockedClient.scrub_controller = {
"possible_threads": thread_names,
"volumes": {},
"waiter": Waiter(len(thread_names) - 9),
}
LockedClient.thread_names = thread_names[:]
for vdisk_id in sorted(vdisks):
vdisk = vdisks[vdisk_id]
LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": True, "scrub_work": range(vdisk_id)}
GenericController.execute_scrub()
self.assertEqual(
first=len(LockedClient.thread_names),
second=9, # 5 srs * 3 vps = 15 threads, but only 2 will be spawned per vPool --> 15 - 6 = 9 left
msg="Not all threads have been used in the process",
)
# 3 vPools will cause the scrubber to only launch 2 threads per vPool --> 1 possible thread should be unused per vPool
for vpool in vpools.values():
threads_left = [thread_name for thread_name in LockedClient.thread_names if vpool.guid in thread_name]
self.assertEqual(
first=len(threads_left),
second=3,
msg="Unexpected amount of threads left for vPool {0}".format(vpool.name),
)
def check_scrubbing_test():
"""
Check scrubbing of vdisks test
"""
initial_counter = 100
step = 5
vdisk = None
vpool_name = General.get_config().get('vpool', 'name')
vpool = GeneralVPool.get_vpool_by_name(vpool_name=vpool_name)
assert vpool, "No vpool found where one was expected"
template_folder = GeneralVMachine.template_target_folder
image_name = GeneralVMachine.template_image
disk_name = "scrubdisk"
GeneralVMachine.logger.info("Starting RAW disk creation")
out, err, _ = General.execute_command('qemu-img convert -O raw {0}{1} /mnt/{2}/{3}.raw'.format(template_folder, image_name, vpool_name, disk_name))
if err:
GeneralVMachine.logger.error("Error while creating raw disk: {0}".format(err))
def snapshot_vdisk(vdisk):
metadata = {'label': 'snap-' + vdisk.name,
'is_consistent': True,
'timestamp': time.time(),
'is_automatic': False,
'is_sticky': False}
VDiskController.create_snapshot(vdisk.guid, metadata)
counter = initial_counter
while counter and vdisk is None:
time.sleep(step)
vdisk = VDiskList.get_by_devicename_and_vpool('/' + disk_name + '.raw', vpool)
counter -= step
assert counter > 0, "Vdisk with name {0} didn't appear in the model after 60 seconds".format(disk_name)
# snapshot disks for the first time
snapshot_vdisk(vdisk)
counter = initial_counter
while counter > 0:
time.sleep(step)
out, err, _ = General.execute_command('dd if=/dev/zero of=/mnt/{0}/{1}.raw bs=10K count=1000 conv=notrunc'.format(vpool_name, disk_name))
counter -= step
snapshot_vdisk(vdisk)
# saving disk 'stored' info / the only attribute that is lowered after scrubbing
vdisk.invalidate_dynamics(['statistics'])
disk_backend_data = vdisk.statistics['stored']
# deleting middle snapshots
for snapshot in vdisk.snapshots[1:-1]:
VDiskController.delete_snapshot(vdisk.guid, snapshot['guid'])
# starting scrubber
try:
GenericController.execute_scrub()
# waiting for model to catch up
counter = initial_counter
while counter > 0:
time.sleep(step)
vdisk.invalidate_dynamics(['statistics'])
# checking result of scrub work
if vdisk.statistics['stored'] < disk_backend_data:
GeneralVMachine.logger.info("It took {0} seconds for the value to change from {1} to {2}\n".format((initial_counter - counter) * step,
disk_backend_data,
vdisk.statistics['stored']))
break
counter -= step
finally:
# removing vdisk
GeneralVMachine.logger.info("Removing vpool vdisks from {0} vpool".format(vpool_name))
out, err, _ = General.execute_command("rm -rf /mnt/{0}/*.raw".format(vpool_name))
if err:
GeneralVMachine.logger.error("Error while removing vdisk: {0}".format(err))
assert counter > 0, "Scrubbing didn't run as expected, backend size of vdisk remained at {0}:\n".format(disk_backend_data)
