def test_delete_snapshot(self): """ Test the delete snapshot functionality - Create a vDisk and take a snapshot - Attempt to delete a non-existing snapshot """ structure = Helper.build_service_structure( {'vpools': [1], 'storagerouters': [1], 'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>) 'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>) ) storagedrivers = structure['storagedrivers'] vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid)) VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': int(time.time()), 'label': 'label1', 'is_consistent': True, 'is_automatic': True, 'is_sticky': False}) snapshot = vdisk1.snapshots[0] self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find 1 snapshot') with self.assertRaises(RuntimeError): VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid, snapshot_id='non-existing') VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid, snapshot_id=snapshot['guid']) self.assertTrue(expr=len(vdisk1.snapshots) == 0, msg='Expected to find no more snapshots')
def test_delete_snapshot(self): """ Test the delete snapshot functionality - Create a vDisk and take a snapshot - Attempt to delete a non-existing snapshot """ structure = DalHelper.build_dal_structure( {'vpools': [1], 'storagerouters': [1], 'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>) 'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>) ) storagedrivers = structure['storagedrivers'] vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid)) VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': int(time.time()), 'label': 'label1', 'is_consistent': True, 'is_automatic': True, 'is_sticky': False}) self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find 1 snapshot') self.assertTrue(expr=len(vdisk1.snapshot_ids) == 1, msg='Expected to find 1 snapshot ID') with self.assertRaises(RuntimeError): VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid, snapshot_id='non-existing') VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid, snapshot_id=vdisk1.snapshot_ids[0]) self.assertTrue(expr=len(vdisk1.snapshots) == 0, msg='Expected to find no more snapshots') self.assertTrue(expr=len(vdisk1.snapshot_ids) == 0, msg='Expected to find no more snapshot IDs')
def snapshot(machineguid, label=None, is_consistent=False, timestamp=None, is_automatic=False): """ Snapshot VMachine disks @param machineguid: guid of the machine @param label: label to give the snapshots @param is_consistent: flag indicating the snapshot was consistent or not @param timestamp: override timestamp, if required. Should be a unix timestamp """ timestamp = timestamp if timestamp is not None else time.time() timestamp = str(int(float(timestamp))) metadata = { 'label': label, 'is_consistent': is_consistent, 'timestamp': timestamp, 'machineguid': machineguid, 'is_automatic': is_automatic } machine = VMachine(machineguid) # @todo: we now skip creating a snapshot when a vmachine's disks # is missing a mandatory property: volume_id # subtask will now raise an exception earlier in the workflow for disk in machine.vdisks: if not disk.volume_id: message = 'Missing volume_id on disk {0} - unable to create snapshot for vm {1}'.format( disk.guid, machine.guid) logger.info('Error: {0}'.format(message)) raise RuntimeError(message) snapshots = {} success = True try: for disk in machine.vdisks: snapshots[disk.guid] = VDiskController.create_snapshot( diskguid=disk.guid, metadata=metadata) except Exception as ex: logger.info('Error snapshotting disk {0}: {1}'.format( disk.name, str(ex))) success = False for diskguid, snapshotid in snapshots.iteritems(): VDiskController.delete_snapshot(diskguid=diskguid, snapshotid=snapshotid) logger.info('Create snapshot for vMachine {0}: {1}'.format( machine.name, 'Success' if success else 'Failure')) machine.invalidate_dynamics(['snapshots']) if not success: raise RuntimeError('Failed to snapshot vMachine {0}'.format( machine.name))
def snapshot(machineguid, label=None, is_consistent=False, timestamp=None, is_automatic=False, is_sticky=False): """ Snapshot VMachine disks :param machineguid: guid of the machine :param label: label to give the snapshots :param is_consistent: flag indicating the snapshot was consistent or not :param timestamp: override timestamp, if required. Should be a unix timestamp :param is_automatic: Flag to determine automated snapshots :param is_sticky: Flag indicating the snapshot is not to be deleted automatically """ timestamp = timestamp if timestamp is not None else time.time() timestamp = str(int(float(timestamp))) if is_automatic is True and is_sticky is True: raise ValueError('Snapshot {0} cannot be both automatic and sticky'.format(label)) metadata = {'label': label, 'is_consistent': is_consistent, 'timestamp': timestamp, 'machineguid': machineguid, 'is_automatic': is_automatic, 'is_sticky': is_sticky} machine = VMachine(machineguid) # @todo: we now skip creating a snapshot when a vmachine's disks # is missing a mandatory property: volume_id # sub-task will now raise an exception earlier in the workflow for disk in machine.vdisks: if not disk.volume_id: message = 'Missing volume_id on disk {0} - unable to create snapshot for vm {1}'.format( disk.guid, machine.guid ) VMachineController._logger.info('Error: {0}'.format(message)) raise RuntimeError(message) snapshots = {} success = True for disk in machine.vdisks: try: snapshots[disk.guid] = VDiskController.create_snapshot(diskguid=disk.guid, metadata=metadata) except Exception as ex: VMachineController._logger.info('Error taking snapshot of disk {0}: {1}'.format(disk.name, str(ex))) success = False for diskguid, snapshotid in snapshots.iteritems(): VDiskController.delete_snapshot(diskguid=diskguid, snapshotid=snapshotid) VMachineController._logger.info('Create snapshot for vMachine {0}: {1}'.format( machine.name, 'Success' if success else 'Failure' )) machine.invalidate_dynamics(['snapshots']) if not success: raise RuntimeError('Failed to snapshot vMachine {0}'.format(machine.name))
def delete_snapshot(self, snapshot): """Deletes a snapshot. :param snapshot: snapshot reference (sqlalchemy Model) """ _debug_vol_info("DELETE_SNAP", snapshot) volume = snapshot.volume hostname = volume.host location = volume.provider_location ovs_disk = self._find_ovs_model_disk_by_location(location, hostname) LOG.debug("DELETE_SNAP %s" % snapshot.id) VDiskController.delete_snapshot(diskguid=ovs_disk.guid, snapshotid=str(snapshot.id)) LOG.debug("DELETE_SNAP OK")
def delete_snapshot(self, snapshot): """Deletes a snapshot. :param snapshot: snapshot reference (sqlalchemy Model) """ _debug_vol_info('DELETE_SNAP', snapshot) volume = snapshot.volume hostname = volume.host location = volume.provider_location ovs_disk = self._find_ovs_model_disk_by_location(location, hostname) LOG.debug('DELETE_SNAP %s' % snapshot.id) VDiskController.delete_snapshot(diskguid=ovs_disk.guid, snapshotid=str(snapshot.id)) LOG.debug('DELETE_SNAP OK')
def delete_snapshot(vmachineguid, timestamp): """ Remove a snapshot from the vmachine @param vmachineguid: Guid of the virtual machine @param timestamp: timestamp of the snapshot """ vmachine = VMachine(vmachineguid) vmachine_snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == str(timestamp)] if len(vmachine_snapshots) != 1: raise RuntimeError('Snapshot {0} does not belong to vmachine {1}'.format(timestamp, vmachine.name)) vmachine_snapshot = vmachine_snapshots[0] VMachineController._logger.info('Deleting snapshot {0} from vmachine {1}'.format(timestamp, vmachine.name)) for diskguid, snapshotid in vmachine_snapshot['snapshots'].items(): VDiskController.delete_snapshot(diskguid, snapshotid) VMachineController._logger.info('Deleted snapshot {0}'.format(timestamp)) vmachine.invalidate_dynamics(['snapshots'])
def delete_snapshot(vmachineguid, timestamp): """ Remove a snapshot from the vmachine @param vmachineguid: Guid of the virtual machine @param timestamp: timestamp of the snapshot """ vmachine = VMachine(vmachineguid) vmachine_snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == str(timestamp)] if len(vmachine_snapshots) != 1: raise RuntimeError('Snapshot {0} does not belong to vmachine {1}'.format(timestamp, vmachine.name)) vmachine_snapshot = vmachine_snapshots[0] VMachineController._logger.info('Deleting snapshot {0} from vmachine {1}'.format(timestamp, vmachine.name)) for diskguid, snapshotid in vmachine_snapshot['snapshots'].items(): VDiskController.delete_snapshot(diskguid, snapshotid) VMachineController._logger.info('Deleted snapshot {0}'.format(timestamp)) vmachine.invalidate_dynamics(['snapshots'])
def deletescrubsnapshots(timestamp=None): """ Delete snapshots & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete """ logger.info('Delete snapshots started') day = 60 * 60 * 24 week = day * 7 # Calculate bucket structure if timestamp is None: timestamp = time.time() offset = int(mktime(datetime.fromtimestamp(timestamp).date().timetuple())) - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append({'start': offset - (day * i), 'end': offset - (day * (i + 1)), 'type': '1d', 'snapshots': []}) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append({'start': offset - (week * i), 'end': offset - (week * (i + 1)), 'type': '1w', 'snapshots': []}) buckets.append({'start': offset - (week * 4), 'end': 0, 'type': 'rest', 'snapshots': []}) # Place all snapshots in bucket_chains bucket_chains = [] for vmachine in VMachineList.get_customer_vmachines(): if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks): bucket_chain = copy.deepcopy(buckets) for snapshot in vmachine.snapshots: timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: for diskguid, snapshotguid in snapshot['snapshots'].iteritems(): bucket['snapshots'].append({'timestamp': timestamp, 'snapshotid': snapshotguid, 'diskguid': diskguid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE']: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: bucket['snapshots'].append({'timestamp': timestamp, 'snapshotid': snapshot['guid'], 'diskguid': vdisk.guid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket['snapshots']: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot['is_consistent'] and not best['is_consistent']: best = snapshot # Newer (larger timestamp) is better than older snapshots elif snapshot['is_consistent'] == best['is_consistent'] and \ snapshot['timestamp'] > best['timestamp']: best = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != best['timestamp']] first = False elif bucket['end'] > 0: oldest = None for snapshot in bucket['snapshots']: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot['timestamp'] < oldest['timestamp']: oldest = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != oldest['timestamp']] # Delete obsolete snapshots for bucket_chain in bucket_chains: for bucket in bucket_chain: for snapshot in bucket['snapshots']: VDiskController.delete_snapshot(diskguid=snapshot['diskguid'], snapshotid=snapshot['snapshotid']) logger.info('Delete snapshots finished') logger.info('Scrubbing started') vdisks = [] for vmachine in VMachineList.get_customer_vmachines(): for vdisk in vmachine.vdisks: if vdisk.info['object_type'] in ['BASE']: vdisks.append(vdisk) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE']: vdisks.append(vdisk) total = 0 failed = 0 for vdisk in vdisks: work_units = vdisk.storagedriver_client.get_scrubbing_workunits(str(vdisk.volume_id)) for work_unit in work_units: try: total += 1 scrubbing_result = _storagedriver_scrubber.scrub(work_unit, vdisk.vpool.mountpoint_temp) vdisk.storagedriver_client.apply_scrubbing_result(scrubbing_result) except: failed += 1 logger.info('Failed scrubbing work unit for volume {}'.format( vdisk.volume_id )) logger.info('Scrubbing finished. {} out of {} items failed.'.format( failed, total ))
def delete_snapshot_while_scrubbing(*args, **kwargs): _ = args, kwargs try: snapshot_while_scrub_results.append(VDiskController.delete_snapshot(vdisk_1.guid, vdisk_1.snapshot_ids[0])) except RuntimeError as ex: snapshot_while_scrub_results.append(ex)
def deletescrubsnapshots(timestamp=None): """ Delete snapshots & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete """ logger.info('Delete snapshots started') day = 60 * 60 * 24 week = day * 7 # Calculate bucket structure if timestamp is None: timestamp = time.time() offset = int( mktime(datetime.fromtimestamp(timestamp).date().timetuple())) - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append({ 'start': offset - (day * i), 'end': offset - (day * (i + 1)), 'type': '1d', 'snapshots': [] }) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append({ 'start': offset - (week * i), 'end': offset - (week * (i + 1)), 'type': '1w', 'snapshots': [] }) buckets.append({ 'start': offset - (week * 4), 'end': 0, 'type': 'rest', 'snapshots': [] }) # Place all snapshots in bucket_chains bucket_chains = [] for vmachine in VMachineList.get_customer_vmachines(): if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks): bucket_chain = copy.deepcopy(buckets) for snapshot in vmachine.snapshots: timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: for diskguid, snapshotguid in snapshot[ 'snapshots'].iteritems(): bucket['snapshots'].append({ 'timestamp': timestamp, 'snapshotid': snapshotguid, 'diskguid': diskguid, 'is_consistent': snapshot['is_consistent'] }) bucket_chains.append(bucket_chain) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE']: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: bucket['snapshots'].append({ 'timestamp': timestamp, 'snapshotid': snapshot['guid'], 'diskguid': vdisk.guid, 'is_consistent': snapshot['is_consistent'] }) bucket_chains.append(bucket_chain) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket['snapshots']: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot[ 'is_consistent'] and not best['is_consistent']: best = snapshot # Newer (larger timestamp) is better than older snapshots elif snapshot['is_consistent'] == best['is_consistent'] and \ snapshot['timestamp'] > best['timestamp']: best = snapshot bucket['snapshots'] = [ s for s in bucket['snapshots'] if s['timestamp'] != best['timestamp'] ] first = False elif bucket['end'] > 0: oldest = None for snapshot in bucket['snapshots']: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot['timestamp'] < oldest['timestamp']: oldest = snapshot bucket['snapshots'] = [ s for s in bucket['snapshots'] if s['timestamp'] != oldest['timestamp'] ] # Delete obsolete snapshots for bucket_chain in bucket_chains: for bucket in bucket_chain: for snapshot in bucket['snapshots']: VDiskController.delete_snapshot( diskguid=snapshot['diskguid'], snapshotid=snapshot['snapshotid']) logger.info('Delete snapshots finished') logger.info('Scrubbing started') vdisks = [] for vmachine in VMachineList.get_customer_vmachines(): for vdisk in vmachine.vdisks: if vdisk.info['object_type'] in ['BASE'] and len( vdisk.child_vdisks) == 0: vdisks.append(vdisk) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE'] and len( vdisk.child_vdisks) == 0: vdisks.append(vdisk) total = 0 failed = 0 skipped = 0 storagedrivers = {} for vdisk in vdisks: try: total += 1 # Load the vDisk's StorageDriver vdisk.invalidate_dynamics(['info', 'storagedriver_id']) if vdisk.storagedriver_id not in storagedrivers: storagedrivers[ vdisk. storagedriver_id] = StorageDriverList.get_by_storagedriver_id( vdisk.storagedriver_id) storagedriver = storagedrivers[vdisk.storagedriver_id] # Load the vDisk's MDS configuration vdisk.invalidate_dynamics(['info']) configs = vdisk.info['metadata_backend_config'] if len(configs) == 0: raise RuntimeError('Could not load MDS configuration') if configs[0]['ip'] != storagedriver.storagerouter.ip: # The MDS master is not local. Trigger an MDS handover and try again logger.debug( 'MDS for volume {0} is not local. Trigger handover'. format(vdisk.volume_id)) MDSServiceController.ensure_safety(vdisk) vdisk.invalidate_dynamics(['info']) configs = vdisk.info['metadata_backend_config'] if len(configs) == 0: raise RuntimeError('Could not load MDS configuration') if configs[0]['ip'] != storagedriver.storagerouter.ip: skipped += 1 logger.info( 'Skipping scrubbing work unit for volume {0}: MDS master is not local' .format(vdisk.volume_id)) continue work_units = vdisk.storagedriver_client.get_scrubbing_workunits( str(vdisk.volume_id)) for work_unit in work_units: scrubbing_result = _storagedriver_scrubber.scrub( work_unit, str(storagedriver.mountpoint_temp)) vdisk.storagedriver_client.apply_scrubbing_result( scrubbing_result) except Exception, ex: failed += 1 logger.info( 'Failed scrubbing work unit for volume {0}: {1}'.format( vdisk.volume_id, ex))
def delete_snapshots(timestamp=None): """ Delete snapshots & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete :param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used """ logger.info("Delete snapshots started") day = timedelta(1) week = day * 7 def make_timestamp(offset): return int(mktime((base - offset).timetuple())) # Calculate bucket structure if timestamp is None: timestamp = time.time() base = datetime.fromtimestamp(timestamp).date() - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append( {"start": make_timestamp(day * i), "end": make_timestamp(day * (i + 1)), "type": "1d", "snapshots": []} ) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append( { "start": make_timestamp(week * i), "end": make_timestamp(week * (i + 1)), "type": "1w", "snapshots": [], } ) buckets.append({"start": make_timestamp(week * 4), "end": 0, "type": "rest", "snapshots": []}) # Place all snapshots in bucket_chains bucket_chains = [] for vmachine in VMachineList.get_customer_vmachines(): if any(vd.info["object_type"] in ["BASE"] for vd in vmachine.vdisks): bucket_chain = copy.deepcopy(buckets) for snapshot in vmachine.snapshots: timestamp = int(snapshot["timestamp"]) for bucket in bucket_chain: if bucket["start"] >= timestamp > bucket["end"]: for diskguid, snapshotguid in snapshot["snapshots"].iteritems(): bucket["snapshots"].append( { "timestamp": timestamp, "snapshotid": snapshotguid, "diskguid": diskguid, "is_consistent": snapshot["is_consistent"], } ) bucket_chains.append(bucket_chain) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info["object_type"] in ["BASE"]: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: timestamp = int(snapshot["timestamp"]) for bucket in bucket_chain: if bucket["start"] >= timestamp > bucket["end"]: bucket["snapshots"].append( { "timestamp": timestamp, "snapshotid": snapshot["guid"], "diskguid": vdisk.guid, "is_consistent": snapshot["is_consistent"], } ) bucket_chains.append(bucket_chain) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket["snapshots"]: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot["is_consistent"] and not best["is_consistent"]: best = snapshot # Newer (larger timestamp) is better than older snapshots elif ( snapshot["is_consistent"] == best["is_consistent"] and snapshot["timestamp"] > best["timestamp"] ): best = snapshot bucket["snapshots"] = [s for s in bucket["snapshots"] if s["timestamp"] != best["timestamp"]] first = False elif bucket["end"] > 0: oldest = None for snapshot in bucket["snapshots"]: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot["timestamp"] < oldest["timestamp"]: oldest = snapshot bucket["snapshots"] = [s for s in bucket["snapshots"] if s["timestamp"] != oldest["timestamp"]] # Delete obsolete snapshots for bucket_chain in bucket_chains: for bucket in bucket_chain: for snapshot in bucket["snapshots"]: VDiskController.delete_snapshot(diskguid=snapshot["diskguid"], snapshotid=snapshot["snapshotid"]) logger.info("Delete snapshots finished")
def check_scrubbing_test(): """ Check scrubbing of vdisks test """ initial_counter = 100 step = 5 vdisk = None vpool_name = General.get_config().get('vpool', 'name') vpool = GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) assert vpool, "No vpool found where one was expected" template_folder = GeneralVMachine.template_target_folder image_name = GeneralVMachine.template_image disk_name = "scrubdisk" GeneralVMachine.logger.info("Starting RAW disk creation") out, err, _ = General.execute_command('qemu-img convert -O raw {0}{1} /mnt/{2}/{3}.raw'.format(template_folder, image_name, vpool_name, disk_name)) if err: GeneralVMachine.logger.error("Error while creating raw disk: {0}".format(err)) def snapshot_vdisk(vdisk): metadata = {'label': 'snap-' + vdisk.name, 'is_consistent': True, 'timestamp': time.time(), 'is_automatic': False, 'is_sticky': False} VDiskController.create_snapshot(vdisk.guid, metadata) counter = initial_counter while counter and vdisk is None: time.sleep(step) vdisk = VDiskList.get_by_devicename_and_vpool('/' + disk_name + '.raw', vpool) counter -= step assert counter > 0, "Vdisk with name {0} didn't appear in the model after 60 seconds".format(disk_name) # snapshot disks for the first time snapshot_vdisk(vdisk) counter = initial_counter while counter > 0: time.sleep(step) out, err, _ = General.execute_command('dd if=/dev/zero of=/mnt/{0}/{1}.raw bs=10K count=1000 conv=notrunc'.format(vpool_name, disk_name)) counter -= step snapshot_vdisk(vdisk) # saving disk 'stored' info / the only attribute that is lowered after scrubbing vdisk.invalidate_dynamics(['statistics']) disk_backend_data = vdisk.statistics['stored'] # deleting middle snapshots for snapshot in vdisk.snapshots[1:-1]: VDiskController.delete_snapshot(vdisk.guid, snapshot['guid']) # starting scrubber try: GenericController.execute_scrub() # waiting for model to catch up counter = initial_counter while counter > 0: time.sleep(step) vdisk.invalidate_dynamics(['statistics']) # checking result of scrub work if vdisk.statistics['stored'] < disk_backend_data: GeneralVMachine.logger.info("It took {0} seconds for the value to change from {1} to {2}\n".format((initial_counter - counter) * step, disk_backend_data, vdisk.statistics['stored'])) break counter -= step finally: # removing vdisk GeneralVMachine.logger.info("Removing vpool vdisks from {0} vpool".format(vpool_name)) out, err, _ = General.execute_command("rm -rf /mnt/{0}/*.raw".format(vpool_name)) if err: GeneralVMachine.logger.error("Error while removing vdisk: {0}".format(err)) assert counter > 0, "Scrubbing didn't run as expected, backend size of vdisk remained at {0}:\n".format(disk_backend_data)
def delete_snapshots(timestamp=None): """ Delete snapshots & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete :param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used """ logger.info('Delete snapshots started') day = timedelta(1) week = day * 7 def make_timestamp(offset): """ Create an integer based timestamp :param offset: Offset in days :return: Timestamp """ return int(mktime((base - offset).timetuple())) # Calculate bucket structure if timestamp is None: timestamp = time.time() base = datetime.fromtimestamp(timestamp).date() - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append({'start': make_timestamp(day * i), 'end': make_timestamp(day * (i + 1)), 'type': '1d', 'snapshots': []}) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append({'start': make_timestamp(week * i), 'end': make_timestamp(week * (i + 1)), 'type': '1w', 'snapshots': []}) buckets.append({'start': make_timestamp(week * 4), 'end': 0, 'type': 'rest', 'snapshots': []}) # Place all snapshots in bucket_chains bucket_chains = [] for vmachine in VMachineList.get_customer_vmachines(): if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks): bucket_chain = copy.deepcopy(buckets) for snapshot in vmachine.snapshots: if snapshot.get('is_sticky') is True: continue timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: for diskguid, snapshotguid in snapshot['snapshots'].iteritems(): bucket['snapshots'].append({'timestamp': timestamp, 'snapshotid': snapshotguid, 'diskguid': diskguid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE']: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: if snapshot.get('is_sticky') is True: continue timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: bucket['snapshots'].append({'timestamp': timestamp, 'snapshotid': snapshot['guid'], 'diskguid': vdisk.guid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket['snapshots']: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot['is_consistent'] and not best['is_consistent']: best = snapshot # Newer (larger timestamp) is better than older snapshots elif snapshot['is_consistent'] == best['is_consistent'] and \ snapshot['timestamp'] > best['timestamp']: best = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != best['timestamp']] first = False elif bucket['end'] > 0: oldest = None for snapshot in bucket['snapshots']: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot['timestamp'] < oldest['timestamp']: oldest = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != oldest['timestamp']] # Delete obsolete snapshots for bucket_chain in bucket_chains: for bucket in bucket_chain: for snapshot in bucket['snapshots']: VDiskController.delete_snapshot(diskguid=snapshot['diskguid'], snapshotid=snapshot['snapshotid']) logger.info('Delete snapshots finished')
def deletescrubsnapshots(timestamp=None): """ Delete snapshots & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete """ logger.info('Delete snapshots started') day = 60 * 60 * 24 week = day * 7 # Calculate bucket structure if timestamp is None: timestamp = time.time() offset = int(mktime(datetime.fromtimestamp(timestamp).date().timetuple())) - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append({'start': offset - (day * i), 'end': offset - (day * (i + 1)), 'type': '1d', 'snapshots': []}) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append({'start': offset - (week * i), 'end': offset - (week * (i + 1)), 'type': '1w', 'snapshots': []}) buckets.append({'start': offset - (week * 4), 'end': 0, 'type': 'rest', 'snapshots': []}) # Place all snapshots in bucket_chains bucket_chains = [] for vmachine in VMachineList.get_customer_vmachines(): if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks): bucket_chain = copy.deepcopy(buckets) for snapshot in vmachine.snapshots: timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: for diskguid, snapshotguid in snapshot['snapshots'].iteritems(): bucket['snapshots'].append({'timestamp': timestamp, 'snapshotid': snapshotguid, 'diskguid': diskguid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE']: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: bucket['snapshots'].append({'timestamp': timestamp, 'snapshotid': snapshot['guid'], 'diskguid': vdisk.guid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket['snapshots']: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot['is_consistent'] and not best['is_consistent']: best = snapshot # Newer (larger timestamp) is better than older snapshots elif snapshot['is_consistent'] == best['is_consistent'] and \ snapshot['timestamp'] > best['timestamp']: best = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != best['timestamp']] first = False elif bucket['end'] > 0: oldest = None for snapshot in bucket['snapshots']: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot['timestamp'] < oldest['timestamp']: oldest = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != oldest['timestamp']] # Delete obsolete snapshots for bucket_chain in bucket_chains: for bucket in bucket_chain: for snapshot in bucket['snapshots']: VDiskController.delete_snapshot(diskguid=snapshot['diskguid'], snapshotid=snapshot['snapshotid']) logger.info('Delete snapshots finished') logger.info('Scrubbing started') vdisks = [] for vmachine in VMachineList.get_customer_vmachines(): for vdisk in vmachine.vdisks: if vdisk.info['object_type'] in ['BASE'] and len(vdisk.child_vdisks) == 0: vdisks.append(vdisk) for vdisk in VDiskList.get_without_vmachine(): if vdisk.info['object_type'] in ['BASE'] and len(vdisk.child_vdisks) == 0: vdisks.append(vdisk) total = 0 failed = 0 skipped = 0 storagedrivers = {} for vdisk in vdisks: try: total += 1 # Load the vDisk's StorageDriver vdisk.invalidate_dynamics(['info', 'storagedriver_id']) if vdisk.storagedriver_id not in storagedrivers: storagedrivers[vdisk.storagedriver_id] = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id) storagedriver = storagedrivers[vdisk.storagedriver_id] # Load the vDisk's MDS configuration vdisk.invalidate_dynamics(['info']) configs = vdisk.info['metadata_backend_config'] if len(configs) == 0: raise RuntimeError('Could not load MDS configuration') if configs[0]['ip'] != storagedriver.storagerouter.ip: # The MDS master is not local. Trigger an MDS handover and try again logger.debug('MDS for volume {0} is not local. Trigger handover'.format(vdisk.volume_id)) MDSServiceController.ensure_safety(vdisk) vdisk.invalidate_dynamics(['info']) configs = vdisk.info['metadata_backend_config'] if len(configs) == 0: raise RuntimeError('Could not load MDS configuration') if configs[0]['ip'] != storagedriver.storagerouter.ip: skipped += 1 logger.info('Skipping scrubbing work unit for volume {0}: MDS master is not local'.format( vdisk.volume_id )) continue work_units = vdisk.storagedriver_client.get_scrubbing_workunits(str(vdisk.volume_id)) for work_unit in work_units: scrubbing_result = _storagedriver_scrubber.scrub(work_unit, str(storagedriver.mountpoint_temp)) vdisk.storagedriver_client.apply_scrubbing_result(scrubbing_result) except Exception, ex: failed += 1 logger.info('Failed scrubbing work unit for volume {0}: {1}'.format( vdisk.volume_id, ex ))
def delete_snapshots(timestamp=None): """ Delete snapshots & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete :param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used :type timestamp: float :return: None """ GenericController._logger.info('Delete snapshots started') day = timedelta(1) week = day * 7 def make_timestamp(offset): """ Create an integer based timestamp :param offset: Offset in days :return: Timestamp """ return int(mktime((base - offset).timetuple())) # Calculate bucket structure if timestamp is None: timestamp = time.time() base = datetime.fromtimestamp(timestamp).date() - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append({ 'start': make_timestamp(day * i), 'end': make_timestamp(day * (i + 1)), 'type': '1d', 'snapshots': [] }) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append({ 'start': make_timestamp(week * i), 'end': make_timestamp(week * (i + 1)), 'type': '1w', 'snapshots': [] }) buckets.append({ 'start': make_timestamp(week * 4), 'end': 0, 'type': 'rest', 'snapshots': [] }) # Get a list of all snapshots that are used as parents for clones parent_snapshots = set( [vd.parentsnapshot for vd in VDiskList.get_with_parent_snaphots()]) # Place all snapshots in bucket_chains bucket_chains = [] for vdisk in VDiskList.get_vdisks(): if vdisk.info['object_type'] in ['BASE']: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: if snapshot.get('is_sticky') is True: continue if snapshot['guid'] in parent_snapshots: GenericController._logger.info( 'Not deleting snapshot {0} because it has clones'. format(snapshot['guid'])) continue timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: bucket['snapshots'].append({ 'timestamp': timestamp, 'snapshot_id': snapshot['guid'], 'vdisk_guid': vdisk.guid, 'is_consistent': snapshot['is_consistent'] }) bucket_chains.append(bucket_chain) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket['snapshots']: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot[ 'is_consistent'] and not best['is_consistent']: best = snapshot # Newer (larger timestamp) is better than older snapshots elif snapshot['is_consistent'] == best['is_consistent'] and \ snapshot['timestamp'] > best['timestamp']: best = snapshot bucket['snapshots'] = [ s for s in bucket['snapshots'] if s['timestamp'] != best['timestamp'] ] first = False elif bucket['end'] > 0: oldest = None for snapshot in bucket['snapshots']: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot['timestamp'] < oldest['timestamp']: oldest = snapshot bucket['snapshots'] = [ s for s in bucket['snapshots'] if s['timestamp'] != oldest['timestamp'] ] # Delete obsolete snapshots for bucket_chain in bucket_chains: for bucket in bucket_chain: for snapshot in bucket['snapshots']: VDiskController.delete_snapshot( vdisk_guid=snapshot['vdisk_guid'], snapshot_id=snapshot['snapshot_id']) GenericController._logger.info('Delete snapshots finished')
def delete_snapshots_storagedriver(storagedriver_guid, timestamp=None, group_id=None): """ Delete snapshots per storagedriver & scrubbing policy Implemented delete snapshot policy: < 1d | 1d bucket | 1 | best of bucket | 1d < 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w < 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m > 1m | delete :param storagedriver_guid: Guid of the StorageDriver to remove snapshots on :type storagedriver_guid: str :param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used :type timestamp: float :param group_id: ID of the group task. Used to identify which snapshot deletes were called during the scheduled task :type group_id: str :return: None """ if group_id: log_id = 'Group job {} - '.format(group_id) else: log_id = '' def format_log(message): return '{}{}'.format(log_id, message) GenericController._logger.info(format_log('Delete snapshots started for StorageDriver {0}'.format(storagedriver_guid))) storagedriver = StorageDriver(storagedriver_guid) exceptions = [] day = timedelta(1) week = day * 7 def make_timestamp(offset): """ Create an integer based timestamp :param offset: Offset in days :return: Timestamp """ return int(mktime((base - offset).timetuple())) # Calculate bucket structure if timestamp is None: timestamp = time.time() base = datetime.fromtimestamp(timestamp).date() - day buckets = [] # Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[ for i in xrange(0, 7): buckets.append({'start': make_timestamp(day * i), 'end': make_timestamp(day * (i + 1)), 'type': '1d', 'snapshots': []}) # Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[ for i in xrange(1, 4): buckets.append({'start': make_timestamp(week * i), 'end': make_timestamp(week * (i + 1)), 'type': '1w', 'snapshots': []}) buckets.append({'start': make_timestamp(week * 4), 'end': 0, 'type': 'rest', 'snapshots': []}) # Get a list of all snapshots that are used as parents for clones parent_snapshots = set([vd.parentsnapshot for vd in VDiskList.get_with_parent_snaphots()]) # Place all snapshots in bucket_chains bucket_chains = [] for vdisk_guid in storagedriver.vdisks_guids: try: vdisk = VDisk(vdisk_guid) vdisk.invalidate_dynamics('being_scrubbed') if vdisk.being_scrubbed: continue if vdisk.info['object_type'] in ['BASE']: bucket_chain = copy.deepcopy(buckets) for snapshot in vdisk.snapshots: if snapshot.get('is_sticky') is True: continue if snapshot['guid'] in parent_snapshots: GenericController._logger.info(format_log('Not deleting snapshot {0} because it has clones'.format(snapshot['guid']))) continue timestamp = int(snapshot['timestamp']) for bucket in bucket_chain: if bucket['start'] >= timestamp > bucket['end']: bucket['snapshots'].append({'timestamp': timestamp, 'snapshot_id': snapshot['guid'], 'vdisk_guid': vdisk.guid, 'is_consistent': snapshot['is_consistent']}) bucket_chains.append(bucket_chain) except Exception as ex: exceptions.append(ex) # Clean out the snapshot bucket_chains, we delete the snapshots we want to keep # And we'll remove all snapshots that remain in the buckets for bucket_chain in bucket_chains: first = True for bucket in bucket_chain: if first is True: best = None for snapshot in bucket['snapshots']: if best is None: best = snapshot # Consistent is better than inconsistent elif snapshot['is_consistent'] and not best['is_consistent']: best = snapshot # Newer (larger timestamp) is better than older snapshots elif snapshot['is_consistent'] == best['is_consistent'] and \ snapshot['timestamp'] > best['timestamp']: best = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != best['timestamp']] first = False elif bucket['end'] > 0: oldest = None for snapshot in bucket['snapshots']: if oldest is None: oldest = snapshot # Older (smaller timestamp) is the one we want to keep elif snapshot['timestamp'] < oldest['timestamp']: oldest = snapshot bucket['snapshots'] = [s for s in bucket['snapshots'] if s['timestamp'] != oldest['timestamp']] # Delete obsolete snapshots for bucket_chain in bucket_chains: # Each bucket chain represents one vdisk's snapshots try: for bucket in bucket_chain: for snapshot in bucket['snapshots']: VDiskController.delete_snapshot(vdisk_guid=snapshot['vdisk_guid'], snapshot_id=snapshot['snapshot_id']) except RuntimeError as ex: vdisk_guid = next((snapshot['vdisk_guid'] for bucket in bucket_chain for snapshot in bucket['snapshots']), '') vdisk_id_log = '' if vdisk_guid: vdisk_id_log = ' for VDisk with guid {}'.format(vdisk_guid) if SCRUB_VDISK_EXCEPTION_MESSAGE in ex.message: GenericController._logger.warning(format_log('Being scrubbed exception occurred while deleting snapshots{}'.format(vdisk_id_log))) else: GenericController._logger.exception(format_log('Exception occurred while deleting snapshots{}'.format(vdisk_id_log))) exceptions.append(ex) if exceptions: raise RuntimeError('Exceptions occurred while deleting snapshots: \n- {}'.format('\n- '.join((str(ex) for ex in exceptions)))) GenericController._logger.info(format_log('Delete snapshots finished for StorageDriver {0}'))
def test_clone(self): """ Test the clone functionality - Create a vDisk with name 'clone1' - Clone the vDisk and make some assertions - Attempt to clone again using same name and same devicename - Attempt to clone on Storage Router which is not linked to the vPool on which the original vDisk is hosted - Attempt to clone on Storage Driver without MDS service - Attempt to clone from snapshot which is not yet completely synced to backend - Attempt to delete the snapshot from which a clone was made - Clone the vDisk on another Storage Router - Clone another vDisk with name 'clone1' linked to another vPool """ structure = DalHelper.build_dal_structure({ 'vpools': [1, 2], 'storagerouters': [1, 2, 3], 'storagedrivers': [(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>) 'mds_services': [(1, 1), (2, 2)] } # (<id>, <storagedriver_id>) ) vpools = structure['vpools'] mds_services = structure['mds_services'] service_type = structure['service_types']['MetadataServer'] storagedrivers = structure['storagedrivers'] storagerouters = structure['storagerouters'] self._roll_out_dtl_services(vpool=vpools[1], storagerouters=storagerouters) self._roll_out_dtl_services(vpool=vpools[2], storagerouters=storagerouters) # Basic clone scenario vdisk1 = VDisk( VDiskController.create_new( volume_name='vdisk_1', volume_size=1024**3, storagedriver_guid=storagedrivers[1].guid)) clone1_info = VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1') vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks') clones = VDiskList.get_by_parentsnapshot(vdisk1.snapshot_ids[0]) self.assertTrue(expr=len(clones) == 1, msg='Expected to find 1 vDisk with parent snapshot') self.assertTrue(expr=len(vdisk1.child_vdisks) == 1, msg='Expected to find 1 child vDisk') for expected_key in ['vdisk_guid', 'name', 'backingdevice']: self.assertTrue( expr=expected_key in clone1_info, msg='Expected to find key "{0}" in clone_info'.format( expected_key)) self.assertTrue(expr=clones[0].guid == clone1_info['vdisk_guid'], msg='Guids do not match') self.assertTrue(expr=clones[0].name == clone1_info['name'], msg='Names do not match') self.assertTrue( expr=clones[0].devicename == clone1_info['backingdevice'], msg='Device names do not match') # Attempt to clone again with same name with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1') vdisks = VDiskList.get_vdisks() self.assertTrue( expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 1') # Attempt to clone again with a name which will have identical devicename with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1%') vdisks = VDiskList.get_vdisks() self.assertTrue( expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 2') # Attempt to clone on Storage Router on which vPool is not extended with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2', storagerouter_guid=storagerouters[2].guid) vdisks = VDiskList.get_vdisks() self.assertTrue( expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 3') # Attempt to clone on non-existing Storage Driver storagedrivers[1].storagedriver_id = 'non-existing' storagedrivers[1].save() with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2') vdisks = VDiskList.get_vdisks() self.assertTrue( expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 4') storagedrivers[1].storagedriver_id = '1' storagedrivers[1].save() # Attempt to clone on Storage Driver without MDS service mds_services[1].service.storagerouter = storagerouters[3] mds_services[1].service.save() with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2') vdisks = VDiskList.get_vdisks() self.assertTrue( expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 5') mds_services[1].service.storagerouter = storagerouters[1] mds_services[1].service.save() # Attempt to clone by providing snapshot_id not synced to backend self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find only 1 snapshot before cloning') self.assertTrue( expr=len(vdisk1.snapshot_ids) == 1, msg='Expected to find only 1 snapshot ID before cloning') metadata = { 'label': 'label1', 'timestamp': int(time.time()), 'is_sticky': False, 'in_backend': False, 'is_automatic': True, 'is_consistent': True } snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata=metadata) self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots') self.assertTrue(expr=len(vdisk1.snapshot_ids) == 2, msg='Expected to find 2 snapshot IDs') with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2', snapshot_id=snapshot_id) vdisks = VDiskList.get_vdisks() self.assertTrue( expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 6') # Update backend synced flag and retry vdisk1.storagedriver_client._set_snapshot_in_backend( vdisk1.volume_id, snapshot_id, True) vdisk1.invalidate_dynamics(['snapshots', 'snapshot_ids']) VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2', snapshot_id=snapshot_id) vdisks = VDiskList.get_vdisks() vdisk1.invalidate_dynamics() self.assertTrue(expr=len(vdisks) == 3, msg='Expected to find 3 vDisks') self.assertTrue(expr=len(vdisk1.child_vdisks) == 2, msg='Expected to find 2 child vDisks') self.assertTrue( expr=len(vdisk1.snapshots) == 2, msg= 'Expected to find 2 snapshots after cloning from a specified snapshot' ) self.assertTrue( expr=len(vdisk1.snapshot_ids) == 2, msg= 'Expected to find 2 snapshot IDs after cloning from a specified snapshot' ) # Attempt to delete the snapshot that has clones with self.assertRaises(RuntimeError): VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid, snapshot_id=snapshot_id) # Clone on specific Storage Router storagedriver = StorageDriver() storagedriver.vpool = vpools[1] storagedriver.storagerouter = storagerouters[2] storagedriver.name = '3' storagedriver.mountpoint = '/' storagedriver.cluster_ip = storagerouters[2].ip storagedriver.storage_ip = '127.0.0.1' storagedriver.storagedriver_id = '3' storagedriver.ports = { 'management': 1, 'xmlrpc': 2, 'dtl': 3, 'edge': 4 } storagedriver.save() s_id = '{0}-1'.format(storagedriver.storagerouter.name) service = Service() service.name = s_id service.storagerouter = storagedriver.storagerouter service.ports = [3] service.type = service_type service.save() mds_service = MDSService() mds_service.service = service mds_service.number = 0 mds_service.capacity = 10 mds_service.vpool = storagedriver.vpool mds_service.save() clone3 = VDisk( VDiskController.clone( vdisk_guid=vdisk1.guid, name='clone3', storagerouter_guid=storagerouters[2].guid)['vdisk_guid']) self.assertTrue( expr=clone3.storagerouter_guid == storagerouters[2].guid, msg='Incorrect Storage Router on which the clone is attached') # Clone vDisk with existing name on another vPool vdisk2 = VDisk( VDiskController.create_new( volume_name='vdisk_1', volume_size=1024**3, storagedriver_guid=storagedrivers[2].guid)) clone_vdisk2 = VDisk( VDiskController.clone(vdisk_guid=vdisk2.guid, name='clone1')['vdisk_guid']) self.assertTrue( expr=clone_vdisk2.vpool == vpools[2], msg='Cloned vDisk with name "clone1" was created on incorrect vPool' ) self.assertTrue(expr=len([ vdisk for vdisk in VDiskList.get_vdisks() if vdisk.name == 'clone1' ]) == 2, msg='Expected to find 2 vDisks with name "clone1"') # Attempt to clone without specifying snapshot and snapshot fails to sync to backend StorageRouterClient.synced = False vdisk2 = VDisk( VDiskController.create_new( volume_name='vdisk_2', volume_size=1024**3, storagedriver_guid=storagedrivers[1].guid)) with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk2.guid, name='clone4') vdisk2.invalidate_dynamics() self.assertTrue(expr=len(vdisk2.snapshots) == 0, msg='Expected to find 0 snapshots after clone failure') self.assertTrue( expr=len(vdisk2.snapshot_ids) == 0, msg='Expected to find 0 snapshot IDs after clone failure') self.assertTrue(expr=len(vdisk2.child_vdisks) == 0, msg='Expected to find 0 children after clone failure') StorageRouterClient.synced = True
def test_clone(self): """ Test the clone functionality - Create a vDisk with name 'clone1' - Clone the vDisk and make some assertions - Attempt to clone again using same name and same devicename - Attempt to clone on Storage Router which is not linked to the vPool on which the original vDisk is hosted - Attempt to clone on Storage Driver without MDS service - Attempt to clone from snapshot which is not yet completely synced to backend - Attempt to delete the snapshot from which a clone was made - Clone the vDisk on another Storage Router - Clone another vDisk with name 'clone1' linked to another vPool """ structure = Helper.build_service_structure( {'vpools': [1, 2], 'storagerouters': [1, 2, 3], 'storagedrivers': [(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>) 'mds_services': [(1, 1), (2, 2)]} # (<id>, <storagedriver_id>) ) vpools = structure['vpools'] mds_services = structure['mds_services'] service_type = structure['service_type'] storagedrivers = structure['storagedrivers'] storagerouters = structure['storagerouters'] self._roll_out_dtl_services(vpool=vpools[1], storagerouters=storagerouters) self._roll_out_dtl_services(vpool=vpools[2], storagerouters=storagerouters) # Basic clone scenario vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid)) clone1_info = VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1') vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks') clones = VDiskList.get_by_parentsnapshot(vdisk1.snapshots[0]['guid']) self.assertTrue(expr=len(clones) == 1, msg='Expected to find 1 vDisk with parent snapshot') self.assertTrue(expr=len(vdisk1.child_vdisks) == 1, msg='Expected to find 1 child vDisk') for expected_key in ['vdisk_guid', 'name', 'backingdevice']: self.assertTrue(expr=expected_key in clone1_info, msg='Expected to find key "{0}" in clone_info'.format(expected_key)) self.assertTrue(expr=clones[0].guid == clone1_info['vdisk_guid'], msg='Guids do not match') self.assertTrue(expr=clones[0].name == clone1_info['name'], msg='Names do not match') self.assertTrue(expr=clones[0].devicename == clone1_info['backingdevice'], msg='Device names do not match') # Attempt to clone again with same name with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1') vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 1') # Attempt to clone again with a name which will have identical devicename with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1%') vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 2') # Attempt to clone on Storage Router on which vPool is not extended with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2', storagerouter_guid=storagerouters[2].guid) vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 3') # Attempt to clone on non-existing Storage Driver storagedrivers[1].storagedriver_id = 'non-existing' storagedrivers[1].save() with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2') vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 4') storagedrivers[1].storagedriver_id = '1' storagedrivers[1].save() # Attempt to clone on Storage Driver without MDS service mds_services[1].service.storagerouter = storagerouters[3] mds_services[1].service.save() with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2') vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 5') mds_services[1].service.storagerouter = storagerouters[1] mds_services[1].service.save() # Attempt to clone by providing snapshot_id not synced to backend self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find only 1 snapshot before cloning') metadata = {'label': 'label1', 'timestamp': int(time.time()), 'is_sticky': False, 'in_backend': False, 'is_automatic': True, 'is_consistent': True} snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata=metadata) self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots') with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2', snapshot_id=snapshot_id) vdisks = VDiskList.get_vdisks() self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 6') # Update backend synced flag and retry vdisk1.storagedriver_client._set_snapshot_in_backend(vdisk1.volume_id, snapshot_id, True) vdisk1.invalidate_dynamics('snapshots') VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2', snapshot_id=snapshot_id) vdisks = VDiskList.get_vdisks() vdisk1.invalidate_dynamics() self.assertTrue(expr=len(vdisks) == 3, msg='Expected to find 3 vDisks') self.assertTrue(expr=len(vdisk1.child_vdisks) == 2, msg='Expected to find 2 child vDisks') self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots after cloning from a specified snapshot') # Attempt to delete the snapshot that has clones with self.assertRaises(RuntimeError): VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid, snapshot_id=snapshot_id) # Clone on specific Storage Router storagedriver = StorageDriver() storagedriver.vpool = vpools[1] storagedriver.storagerouter = storagerouters[2] storagedriver.name = '3' storagedriver.mountpoint = '/' storagedriver.cluster_ip = storagerouters[2].ip storagedriver.storage_ip = '127.0.0.1' storagedriver.storagedriver_id = '3' storagedriver.ports = {'management': 1, 'xmlrpc': 2, 'dtl': 3, 'edge': 4} storagedriver.save() s_id = '{0}-1'.format(storagedriver.storagerouter.name) service = Service() service.name = s_id service.storagerouter = storagedriver.storagerouter service.ports = [3] service.type = service_type service.save() mds_service = MDSService() mds_service.service = service mds_service.number = 0 mds_service.capacity = 10 mds_service.vpool = storagedriver.vpool mds_service.save() clone3 = VDisk(VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone3', storagerouter_guid=storagerouters[2].guid)['vdisk_guid']) self.assertTrue(expr=clone3.storagerouter_guid == storagerouters[2].guid, msg='Incorrect Storage Router on which the clone is attached') # Clone vDisk with existing name on another vPool vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[2].guid)) clone_vdisk2 = VDisk(VDiskController.clone(vdisk_guid=vdisk2.guid, name='clone1')['vdisk_guid']) self.assertTrue(expr=clone_vdisk2.vpool == vpools[2], msg='Cloned vDisk with name "clone1" was created on incorrect vPool') self.assertTrue(expr=len([vdisk for vdisk in VDiskList.get_vdisks() if vdisk.name == 'clone1']) == 2, msg='Expected to find 2 vDisks with name "clone1"') # Attempt to clone without specifying snapshot and snapshot fails to sync to backend StorageRouterClient.synced = False vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_2', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid)) with self.assertRaises(RuntimeError): VDiskController.clone(vdisk_guid=vdisk2.guid, name='clone4') vdisk2.invalidate_dynamics() self.assertTrue(expr=len(vdisk2.snapshots) == 0, msg='Expected to find 0 snapshots after clone failure') self.assertTrue(expr=len(vdisk2.child_vdisks) == 0, msg='Expected to find 0 children after clone failure') StorageRouterClient.synced = True