def test_delete_snapshot_scrubbing_lock(self): """ Tests the skip-if-scrubbed logic """ snapshot_while_scrub_results = [] def delete_snapshot_while_scrubbing(*args, **kwargs): _ = args, kwargs try: snapshot_while_scrub_results.append(VDiskController.delete_snapshot(vdisk_1.guid, vdisk_1.snapshot_ids[0])) except RuntimeError as ex: snapshot_while_scrub_results.append(ex) structure = DalHelper.build_dal_structure( {'vpools': [1], 'vdisks': [(1, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>) 'mds_services': [(1, 1)], 'storagerouters': [1], 'storagedrivers': [(1, 1, 1)]} # (<id>, <vpool_id>, <storagerouter_id>) ) vdisks = structure['vdisks'] vdisk_1 = vdisks[1] # Create automatic snapshot for both vDisks success, fail = GenericController.snapshot_all_vdisks() self.assertEqual(first=len(fail), second=0, msg='Expected 0 failed snapshots') self.assertEqual(first=len(success), second=1, msg='Expected 1 successful snapshots') self.assertEqual(first=len(vdisk_1.snapshot_ids), second=1, msg='Expected 1 snapshot ID for vDisk {0}'.format(vdisk_1.name)) self.assertEqual(first=len(vdisk_1.snapshots), second=1, msg='Expected 1 snapshot for vDisk {0}'.format(vdisk_1.name)) proxy_names, thread_names, vdisk_namespaces = self.generate_scrub_related_info(structure) LockedClient.scrub_controller = {'possible_threads': thread_names, 'volumes': {}, 'waiter': Waiter(len(thread_names[0:1]))} # only 1 disks -> 1 thread # Scrub all volumes for vdisk_id, vdisk in vdisks.iteritems(): LockedClient.scrub_controller['volumes'][vdisk.volume_id] = {'success': True, 'scrub_work': range(vdisk_id)} hooks = {'post_vdisk_scrub_registration': delete_snapshot_while_scrubbing} # Make the scrubber wait ScrubShared._test_hooks.update(hooks) GenericController.execute_scrub() # Ensure delete snapshot fails for vdisk_1 because it is being scrubbed result_while_scrub = snapshot_while_scrub_results[0] self.assertIsInstance(result_while_scrub, Exception, 'Expected an exception to have occurred') self.assertEqual(str(result_while_scrub), 'VDisk is being scrubbed. Unable to remove snapshots at this time', 'Excpetion should be about disk being scrubbed') self.assertEqual(first=len(vdisk_1.snapshot_ids), second=1, msg='Expected 1 snapshot ID for vDisk {0}'.format(vdisk_1.name)) self.assertEqual(first=len(vdisk_1.snapshots), second=1, msg='Expected 1 snapshot for vDisk {0}'.format(vdisk_1.name))
def execute_scrubbing(): """ Execute scrubbing on the cluster :return: """ return GenericController.execute_scrub()
def test_scrubbing(self): """ Validates the scrubbing workflow * Scenario 1: Validate disabled scrub task and single vDisk scrub logic * Scenario 2: 1 vPool, 10 vDisks, 1 scrub role Scrubbing fails for 5 vDisks, check if scrubbing completed for all other vDisks Run scrubbing a 2nd time and verify scrubbing now works for failed vDisks * Scenario 3: 1 vPool, 10 vDisks, 5 scrub roles Check if vDisks are divided among all threads * Scenario 4: 3 vPools, 9 vDisks, 5 scrub roles Validate 6 threads will be spawned and used out of a potential of 15 (5 scrub roles * 3 vPools) We limit max amount of threads spawned per vPool to 2 in case 3 to 5 vPools are present """ _ = self for i in xrange(1, 6): Configuration.set("/ovs/framework/hosts/{0}/ports".format(i), {"storagedriver": [10000, 10100]}) ############## # Scenario 1 # ############## structure = Helper.build_service_structure( { "vpools": [1], "vdisks": [(1, 1, 1, 1)], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>) "mds_services": [(1, 1)], # (<id>, <storagedriver_id>) "storagerouters": [1], "storagedrivers": [(1, 1, 1)], } # (<id>, <vpool_id>, <storagerouter_id>) ) vdisk = structure["vdisks"][1] vpool = structure["vpools"][1] storagerouter = structure["storagerouters"][1] System._machine_id = {storagerouter.ip: "1"} Configuration.set( "/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True ) LockedClient.scrub_controller = {"possible_threads": None, "volumes": {}, "waiter": Waiter(1)} LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": False, "scrub_work": [0]} with self.assertRaises(Exception) as raise_info: VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid) self.assertIn(vdisk.name, raise_info.exception.message) LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": True, "scrub_work": [0]} VDiskController.scrub_single_vdisk(vdisk.guid, storagerouter.guid) with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client: self.assertEqual( first=len(locked_client.get_scrubbing_workunits()), second=0, msg="Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}".format( vdisk.name, 0 ), ) ############## # Scenario 2 # ############## self.volatile.clean() self.persistent.clean() structure = Helper.build_service_structure( { "vpools": [1], "vdisks": [ (1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 1, 1, 1), (5, 1, 1, 1), (6, 1, 1, 1), (7, 1, 1, 1), (8, 1, 1, 1), (9, 1, 1, 1), (10, 1, 1, 1), ], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>) "mds_services": [(1, 1)], # (<id>, <storagedriver_id>) "storagerouters": [1], "storagedrivers": [(1, 1, 1)], } # (<id>, <vpool_id>, <storagerouter_id>) ) vpool = structure["vpools"][1] vdisks = structure["vdisks"] storagerouter = structure["storagerouters"][1] System._machine_id = {storagerouter.ip: "1"} Configuration.set( "/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True ) LockedClient.scrub_controller = { "possible_threads": ["scrub_{0}_{1}".format(vpool.guid, storagerouter.guid)], "volumes": {}, "waiter": Waiter(1), } failed_vdisks = [] successful_vdisks = [] for vdisk_id in sorted(vdisks): vdisk = vdisks[vdisk_id] success = vdisk_id % 2 == 0 LockedClient.scrub_controller["volumes"][vdisk.volume_id] = { "success": success, "scrub_work": range(vdisk_id), } if success is True: successful_vdisks.append(vdisk) else: failed_vdisks.append(vdisk) # Execute scrubbing a 1st time with self.assertRaises(Exception) as raise_info: GenericController.execute_scrub() for vdisk in failed_vdisks: self.assertIn(vdisk.name, raise_info.exception.message) # Validate expected successful vDisks for vdisk in successful_vdisks: with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client: self.assertEqual( first=len(locked_client.get_scrubbing_workunits()), second=0, msg="Scrubbed vDisk {0} does still have scrubbing work left".format(vdisk.name), ) # Validate expected failed vDisks for vdisk in failed_vdisks: with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client: self.assertEqual( first=len(locked_client.get_scrubbing_workunits()), second=int(vdisk.name), msg="Scrubbed vDisk {0} does not have the expected amount of scrubbing items: {1}".format( vdisk.name, int(vdisk.name) ), ) # Execute scrubbing again for vdisk_id in sorted(vdisks): vdisk = vdisks[vdisk_id] LockedClient.scrub_controller["volumes"][vdisk.volume_id]["success"] = True GenericController.execute_scrub() for vdisk in vdisks.values(): with vdisk.storagedriver_client.make_locked_client(vdisk.volume_id) as locked_client: self.assertEqual( first=len(locked_client.get_scrubbing_workunits()), second=0, msg="Scrubbed vDisk {0} does still have scrubbing work left after scrubbing a 2nd time".format( vdisk.name ), ) ############## # Scenario 3 # ############## self.volatile.clean() self.persistent.clean() structure = Helper.build_service_structure( { "vpools": [1], "vdisks": [ (1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 1, 1, 1), (5, 1, 1, 1), (6, 1, 1, 1), (7, 1, 1, 1), (8, 1, 1, 1), (9, 1, 1, 1), (10, 1, 1, 1), ], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>) "mds_services": [(1, 1)], # (<id>, <storagedriver_id>) "storagerouters": [1, 2, 3, 4, 5], "storagedrivers": [(1, 1, 1)], } # (<id>, <vpool_id>, <storagerouter_id>) ) vpool = structure["vpools"][1] vdisks = structure["vdisks"] storagerouters = structure["storagerouters"] System._machine_id = dict((sr.ip, sr.machine_id) for sr in storagerouters.values()) Configuration.set( "/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True ) thread_names = [ "scrub_{0}_{1}".format(vpool.guid, storagerouter.guid) for storagerouter in storagerouters.values() ] LockedClient.scrub_controller = { "possible_threads": thread_names, "volumes": {}, "waiter": Waiter(len(thread_names)), } LockedClient.thread_names = thread_names[:] for vdisk_id in sorted(vdisks): vdisk = vdisks[vdisk_id] LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": True, "scrub_work": range(vdisk_id)} GenericController.execute_scrub() self.assertEqual( first=len(LockedClient.thread_names), second=0, msg="Not all threads have been used in the process" ) ############## # Scenario 4 # ############## self.volatile.clean() self.persistent.clean() structure = Helper.build_service_structure( { "vpools": [1, 2, 3], "vdisks": [ (1, 1, 1, 1), (2, 1, 1, 1), (3, 1, 1, 1), (4, 2, 2, 2), (5, 2, 2, 2), (6, 2, 2, 2), (7, 3, 3, 3), (8, 3, 3, 3), (9, 3, 3, 3), ], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>) "mds_services": [(1, 1), (2, 2), (3, 3)], # (<id>, <storagedriver_id>) "storagerouters": [1, 2, 3, 4, 5], "storagedrivers": [(1, 1, 1), (2, 2, 1), (3, 3, 1)], } # (<id>, <vpool_id>, <storagerouter_id>) ) vpools = structure["vpools"] vdisks = structure["vdisks"] storagerouters = structure["storagerouters"] thread_names = [] for vpool in vpools.values(): Configuration.set( "/ovs/vpools/{0}/proxies/scrub/generic_scrub".format(vpool.guid), json.dumps({}, indent=4), raw=True ) for storagerouter in storagerouters.values(): thread_names.append("scrub_{0}_{1}".format(vpool.guid, storagerouter.guid)) LockedClient.scrub_controller = { "possible_threads": thread_names, "volumes": {}, "waiter": Waiter(len(thread_names) - 9), } LockedClient.thread_names = thread_names[:] for vdisk_id in sorted(vdisks): vdisk = vdisks[vdisk_id] LockedClient.scrub_controller["volumes"][vdisk.volume_id] = {"success": True, "scrub_work": range(vdisk_id)} GenericController.execute_scrub() self.assertEqual( first=len(LockedClient.thread_names), second=9, # 5 srs * 3 vps = 15 threads, but only 2 will be spawned per vPool --> 15 - 6 = 9 left msg="Not all threads have been used in the process", ) # 3 vPools will cause the scrubber to only launch 2 threads per vPool --> 1 possible thread should be unused per vPool for vpool in vpools.values(): threads_left = [thread_name for thread_name in LockedClient.thread_names if vpool.guid in thread_name] self.assertEqual( first=len(threads_left), second=3, msg="Unexpected amount of threads left for vPool {0}".format(vpool.name), )
def check_scrubbing_test(): """ Check scrubbing of vdisks test """ initial_counter = 100 step = 5 vdisk = None vpool_name = General.get_config().get('vpool', 'name') vpool = GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) assert vpool, "No vpool found where one was expected" template_folder = GeneralVMachine.template_target_folder image_name = GeneralVMachine.template_image disk_name = "scrubdisk" GeneralVMachine.logger.info("Starting RAW disk creation") out, err, _ = General.execute_command('qemu-img convert -O raw {0}{1} /mnt/{2}/{3}.raw'.format(template_folder, image_name, vpool_name, disk_name)) if err: GeneralVMachine.logger.error("Error while creating raw disk: {0}".format(err)) def snapshot_vdisk(vdisk): metadata = {'label': 'snap-' + vdisk.name, 'is_consistent': True, 'timestamp': time.time(), 'is_automatic': False, 'is_sticky': False} VDiskController.create_snapshot(vdisk.guid, metadata) counter = initial_counter while counter and vdisk is None: time.sleep(step) vdisk = VDiskList.get_by_devicename_and_vpool('/' + disk_name + '.raw', vpool) counter -= step assert counter > 0, "Vdisk with name {0} didn't appear in the model after 60 seconds".format(disk_name) # snapshot disks for the first time snapshot_vdisk(vdisk) counter = initial_counter while counter > 0: time.sleep(step) out, err, _ = General.execute_command('dd if=/dev/zero of=/mnt/{0}/{1}.raw bs=10K count=1000 conv=notrunc'.format(vpool_name, disk_name)) counter -= step snapshot_vdisk(vdisk) # saving disk 'stored' info / the only attribute that is lowered after scrubbing vdisk.invalidate_dynamics(['statistics']) disk_backend_data = vdisk.statistics['stored'] # deleting middle snapshots for snapshot in vdisk.snapshots[1:-1]: VDiskController.delete_snapshot(vdisk.guid, snapshot['guid']) # starting scrubber try: GenericController.execute_scrub() # waiting for model to catch up counter = initial_counter while counter > 0: time.sleep(step) vdisk.invalidate_dynamics(['statistics']) # checking result of scrub work if vdisk.statistics['stored'] < disk_backend_data: GeneralVMachine.logger.info("It took {0} seconds for the value to change from {1} to {2}\n".format((initial_counter - counter) * step, disk_backend_data, vdisk.statistics['stored'])) break counter -= step finally: # removing vdisk GeneralVMachine.logger.info("Removing vpool vdisks from {0} vpool".format(vpool_name)) out, err, _ = General.execute_command("rm -rf /mnt/{0}/*.raw".format(vpool_name)) if err: GeneralVMachine.logger.error("Error while removing vdisk: {0}".format(err)) assert counter > 0, "Scrubbing didn't run as expected, backend size of vdisk remained at {0}:\n".format(disk_backend_data)