def test_delete_snapshot(self):
"""
Test the delete snapshot functionality
- Create a vDisk and take a snapshot
- Attempt to delete a non-existing snapshot
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': int(time.time()),
'label': 'label1',
'is_consistent': True,
'is_automatic': True,
'is_sticky': False})
snapshot = vdisk1.snapshots[0]
self.assertTrue(expr=len(vdisk1.snapshots) == 1,
msg='Expected to find 1 snapshot')
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id='non-existing')
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=snapshot['guid'])
self.assertTrue(expr=len(vdisk1.snapshots) == 0,
msg='Expected to find no more snapshots')
def test_delete_snapshot(self):
"""
Test the delete snapshot functionality
- Create a vDisk and take a snapshot
- Attempt to delete a non-existing snapshot
"""
structure = DalHelper.build_dal_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': int(time.time()),
'label': 'label1',
'is_consistent': True,
'is_automatic': True,
'is_sticky': False})
self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find 1 snapshot')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 1, msg='Expected to find 1 snapshot ID')
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id='non-existing')
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=vdisk1.snapshot_ids[0])
self.assertTrue(expr=len(vdisk1.snapshots) == 0, msg='Expected to find no more snapshots')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 0, msg='Expected to find no more snapshot IDs')
def snapshot(machineguid,
label=None,
is_consistent=False,
timestamp=None,
is_automatic=False):
"""
Snapshot VMachine disks
@param machineguid: guid of the machine
@param label: label to give the snapshots
@param is_consistent: flag indicating the snapshot was consistent or not
@param timestamp: override timestamp, if required. Should be a unix timestamp
"""
timestamp = timestamp if timestamp is not None else time.time()
timestamp = str(int(float(timestamp)))
metadata = {
'label': label,
'is_consistent': is_consistent,
'timestamp': timestamp,
'machineguid': machineguid,
'is_automatic': is_automatic
}
machine = VMachine(machineguid)
# @todo: we now skip creating a snapshot when a vmachine's disks
# is missing a mandatory property: volume_id
# subtask will now raise an exception earlier in the workflow
for disk in machine.vdisks:
if not disk.volume_id:
message = 'Missing volume_id on disk {0} - unable to create snapshot for vm {1}'.format(
disk.guid, machine.guid)
logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
snapshots = {}
success = True
try:
for disk in machine.vdisks:
snapshots[disk.guid] = VDiskController.create_snapshot(
diskguid=disk.guid, metadata=metadata)
except Exception as ex:
logger.info('Error snapshotting disk {0}: {1}'.format(
disk.name, str(ex)))
success = False
for diskguid, snapshotid in snapshots.iteritems():
VDiskController.delete_snapshot(diskguid=diskguid,
snapshotid=snapshotid)
logger.info('Create snapshot for vMachine {0}: {1}'.format(
machine.name, 'Success' if success else 'Failure'))
machine.invalidate_dynamics(['snapshots'])
if not success:
raise RuntimeError('Failed to snapshot vMachine {0}'.format(
machine.name))
def snapshot(machineguid, label=None, is_consistent=False, timestamp=None, is_automatic=False, is_sticky=False):
"""
Snapshot VMachine disks
:param machineguid: guid of the machine
:param label: label to give the snapshots
:param is_consistent: flag indicating the snapshot was consistent or not
:param timestamp: override timestamp, if required. Should be a unix timestamp
:param is_automatic: Flag to determine automated snapshots
:param is_sticky: Flag indicating the snapshot is not to be deleted automatically
"""
timestamp = timestamp if timestamp is not None else time.time()
timestamp = str(int(float(timestamp)))
if is_automatic is True and is_sticky is True:
raise ValueError('Snapshot {0} cannot be both automatic and sticky'.format(label))
metadata = {'label': label,
'is_consistent': is_consistent,
'timestamp': timestamp,
'machineguid': machineguid,
'is_automatic': is_automatic,
'is_sticky': is_sticky}
machine = VMachine(machineguid)
# @todo: we now skip creating a snapshot when a vmachine's disks
# is missing a mandatory property: volume_id
# sub-task will now raise an exception earlier in the workflow
for disk in machine.vdisks:
if not disk.volume_id:
message = 'Missing volume_id on disk {0} - unable to create snapshot for vm {1}'.format(
disk.guid, machine.guid
)
VMachineController._logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
snapshots = {}
success = True
for disk in machine.vdisks:
try:
snapshots[disk.guid] = VDiskController.create_snapshot(diskguid=disk.guid,
metadata=metadata)
except Exception as ex:
VMachineController._logger.info('Error taking snapshot of disk {0}: {1}'.format(disk.name, str(ex)))
success = False
for diskguid, snapshotid in snapshots.iteritems():
VDiskController.delete_snapshot(diskguid=diskguid,
snapshotid=snapshotid)
VMachineController._logger.info('Create snapshot for vMachine {0}: {1}'.format(
machine.name, 'Success' if success else 'Failure'
))
machine.invalidate_dynamics(['snapshots'])
if not success:
raise RuntimeError('Failed to snapshot vMachine {0}'.format(machine.name))
def delete_snapshot(self, snapshot):
"""Deletes a snapshot.
:param snapshot: snapshot reference (sqlalchemy Model)
"""
_debug_vol_info("DELETE_SNAP", snapshot)
volume = snapshot.volume
hostname = volume.host
location = volume.provider_location
ovs_disk = self._find_ovs_model_disk_by_location(location, hostname)
LOG.debug("DELETE_SNAP %s" % snapshot.id)
VDiskController.delete_snapshot(diskguid=ovs_disk.guid, snapshotid=str(snapshot.id))
LOG.debug("DELETE_SNAP OK")
def delete_snapshot(self, snapshot):
"""Deletes a snapshot.
:param snapshot: snapshot reference (sqlalchemy Model)
"""
_debug_vol_info('DELETE_SNAP', snapshot)
volume = snapshot.volume
hostname = volume.host
location = volume.provider_location
ovs_disk = self._find_ovs_model_disk_by_location(location, hostname)
LOG.debug('DELETE_SNAP %s' % snapshot.id)
VDiskController.delete_snapshot(diskguid=ovs_disk.guid,
snapshotid=str(snapshot.id))
LOG.debug('DELETE_SNAP OK')
def delete_snapshot(vmachineguid, timestamp):
"""
Remove a snapshot from the vmachine
@param vmachineguid: Guid of the virtual machine
@param timestamp: timestamp of the snapshot
"""
vmachine = VMachine(vmachineguid)
vmachine_snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == str(timestamp)]
if len(vmachine_snapshots) != 1:
raise RuntimeError('Snapshot {0} does not belong to vmachine {1}'.format(timestamp, vmachine.name))
vmachine_snapshot = vmachine_snapshots[0]
VMachineController._logger.info('Deleting snapshot {0} from vmachine {1}'.format(timestamp, vmachine.name))
for diskguid, snapshotid in vmachine_snapshot['snapshots'].items():
VDiskController.delete_snapshot(diskguid, snapshotid)
VMachineController._logger.info('Deleted snapshot {0}'.format(timestamp))
vmachine.invalidate_dynamics(['snapshots'])
def delete_snapshot(vmachineguid, timestamp):
"""
Remove a snapshot from the vmachine
@param vmachineguid: Guid of the virtual machine
@param timestamp: timestamp of the snapshot
"""
vmachine = VMachine(vmachineguid)
vmachine_snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == str(timestamp)]
if len(vmachine_snapshots) != 1:
raise RuntimeError('Snapshot {0} does not belong to vmachine {1}'.format(timestamp, vmachine.name))
vmachine_snapshot = vmachine_snapshots[0]
VMachineController._logger.info('Deleting snapshot {0} from vmachine {1}'.format(timestamp, vmachine.name))
for diskguid, snapshotid in vmachine_snapshot['snapshots'].items():
VDiskController.delete_snapshot(diskguid, snapshotid)
VMachineController._logger.info('Deleted snapshot {0}'.format(timestamp))
vmachine.invalidate_dynamics(['snapshots'])
def deletescrubsnapshots(timestamp=None):
"""
Delete snapshots & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
"""
logger.info('Delete snapshots started')
day = 60 * 60 * 24
week = day * 7
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
offset = int(mktime(datetime.fromtimestamp(timestamp).date().timetuple())) - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({'start': offset - (day * i),
'end': offset - (day * (i + 1)),
'type': '1d',
'snapshots': []})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({'start': offset - (week * i),
'end': offset - (week * (i + 1)),
'type': '1w',
'snapshots': []})
buckets.append({'start': offset - (week * 4),
'end': 0,
'type': 'rest',
'snapshots': []})
# Place all snapshots in bucket_chains
bucket_chains = []
for vmachine in VMachineList.get_customer_vmachines():
if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks):
bucket_chain = copy.deepcopy(buckets)
for snapshot in vmachine.snapshots:
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems():
bucket['snapshots'].append({'timestamp': timestamp,
'snapshotid': snapshotguid,
'diskguid': diskguid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({'timestamp': timestamp,
'snapshotid': snapshot['guid'],
'diskguid': vdisk.guid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot['is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != best['timestamp']]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != oldest['timestamp']]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(diskguid=snapshot['diskguid'],
snapshotid=snapshot['snapshotid'])
logger.info('Delete snapshots finished')
logger.info('Scrubbing started')
vdisks = []
for vmachine in VMachineList.get_customer_vmachines():
for vdisk in vmachine.vdisks:
if vdisk.info['object_type'] in ['BASE']:
vdisks.append(vdisk)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE']:
vdisks.append(vdisk)
total = 0
failed = 0
for vdisk in vdisks:
work_units = vdisk.storagedriver_client.get_scrubbing_workunits(str(vdisk.volume_id))
for work_unit in work_units:
try:
total += 1
scrubbing_result = _storagedriver_scrubber.scrub(work_unit, vdisk.vpool.mountpoint_temp)
vdisk.storagedriver_client.apply_scrubbing_result(scrubbing_result)
except:
failed += 1
logger.info('Failed scrubbing work unit for volume {}'.format(
vdisk.volume_id
))
logger.info('Scrubbing finished. {} out of {} items failed.'.format(
failed, total
))
def delete_snapshot_while_scrubbing(*args, **kwargs):
_ = args, kwargs
try:
snapshot_while_scrub_results.append(VDiskController.delete_snapshot(vdisk_1.guid, vdisk_1.snapshot_ids[0]))
except RuntimeError as ex:
snapshot_while_scrub_results.append(ex)
def deletescrubsnapshots(timestamp=None):
"""
Delete snapshots & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
"""
logger.info('Delete snapshots started')
day = 60 * 60 * 24
week = day * 7
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
offset = int(
mktime(datetime.fromtimestamp(timestamp).date().timetuple())) - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({
'start': offset - (day * i),
'end': offset - (day * (i + 1)),
'type': '1d',
'snapshots': []
})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({
'start': offset - (week * i),
'end': offset - (week * (i + 1)),
'type': '1w',
'snapshots': []
})
buckets.append({
'start': offset - (week * 4),
'end': 0,
'type': 'rest',
'snapshots': []
})
# Place all snapshots in bucket_chains
bucket_chains = []
for vmachine in VMachineList.get_customer_vmachines():
if any(vd.info['object_type'] in ['BASE']
for vd in vmachine.vdisks):
bucket_chain = copy.deepcopy(buckets)
for snapshot in vmachine.snapshots:
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
for diskguid, snapshotguid in snapshot[
'snapshots'].iteritems():
bucket['snapshots'].append({
'timestamp':
timestamp,
'snapshotid':
snapshotguid,
'diskguid':
diskguid,
'is_consistent':
snapshot['is_consistent']
})
bucket_chains.append(bucket_chain)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({
'timestamp':
timestamp,
'snapshotid':
snapshot['guid'],
'diskguid':
vdisk.guid,
'is_consistent':
snapshot['is_consistent']
})
bucket_chains.append(bucket_chain)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot[
'is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [
s for s in bucket['snapshots']
if s['timestamp'] != best['timestamp']
]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [
s for s in bucket['snapshots']
if s['timestamp'] != oldest['timestamp']
]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(
diskguid=snapshot['diskguid'],
snapshotid=snapshot['snapshotid'])
logger.info('Delete snapshots finished')
logger.info('Scrubbing started')
vdisks = []
for vmachine in VMachineList.get_customer_vmachines():
for vdisk in vmachine.vdisks:
if vdisk.info['object_type'] in ['BASE'] and len(
vdisk.child_vdisks) == 0:
vdisks.append(vdisk)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE'] and len(
vdisk.child_vdisks) == 0:
vdisks.append(vdisk)
total = 0
failed = 0
skipped = 0
storagedrivers = {}
for vdisk in vdisks:
try:
total += 1
# Load the vDisk's StorageDriver
vdisk.invalidate_dynamics(['info', 'storagedriver_id'])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[
vdisk.
storagedriver_id] = StorageDriverList.get_by_storagedriver_id(
vdisk.storagedriver_id)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
vdisk.invalidate_dynamics(['info'])
configs = vdisk.info['metadata_backend_config']
if len(configs) == 0:
raise RuntimeError('Could not load MDS configuration')
if configs[0]['ip'] != storagedriver.storagerouter.ip:
# The MDS master is not local. Trigger an MDS handover and try again
logger.debug(
'MDS for volume {0} is not local. Trigger handover'.
format(vdisk.volume_id))
MDSServiceController.ensure_safety(vdisk)
vdisk.invalidate_dynamics(['info'])
configs = vdisk.info['metadata_backend_config']
if len(configs) == 0:
raise RuntimeError('Could not load MDS configuration')
if configs[0]['ip'] != storagedriver.storagerouter.ip:
skipped += 1
logger.info(
'Skipping scrubbing work unit for volume {0}: MDS master is not local'
.format(vdisk.volume_id))
continue
work_units = vdisk.storagedriver_client.get_scrubbing_workunits(
str(vdisk.volume_id))
for work_unit in work_units:
scrubbing_result = _storagedriver_scrubber.scrub(
work_unit, str(storagedriver.mountpoint_temp))
vdisk.storagedriver_client.apply_scrubbing_result(
scrubbing_result)
except Exception, ex:
failed += 1
logger.info(
'Failed scrubbing work unit for volume {0}: {1}'.format(
vdisk.volume_id, ex))
def delete_snapshots(timestamp=None):
"""
Delete snapshots & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
:param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used
"""
logger.info("Delete snapshots started")
day = timedelta(1)
week = day * 7
def make_timestamp(offset):
return int(mktime((base - offset).timetuple()))
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
base = datetime.fromtimestamp(timestamp).date() - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append(
{"start": make_timestamp(day * i), "end": make_timestamp(day * (i + 1)), "type": "1d", "snapshots": []}
)
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append(
{
"start": make_timestamp(week * i),
"end": make_timestamp(week * (i + 1)),
"type": "1w",
"snapshots": [],
}
)
buckets.append({"start": make_timestamp(week * 4), "end": 0, "type": "rest", "snapshots": []})
# Place all snapshots in bucket_chains
bucket_chains = []
for vmachine in VMachineList.get_customer_vmachines():
if any(vd.info["object_type"] in ["BASE"] for vd in vmachine.vdisks):
bucket_chain = copy.deepcopy(buckets)
for snapshot in vmachine.snapshots:
timestamp = int(snapshot["timestamp"])
for bucket in bucket_chain:
if bucket["start"] >= timestamp > bucket["end"]:
for diskguid, snapshotguid in snapshot["snapshots"].iteritems():
bucket["snapshots"].append(
{
"timestamp": timestamp,
"snapshotid": snapshotguid,
"diskguid": diskguid,
"is_consistent": snapshot["is_consistent"],
}
)
bucket_chains.append(bucket_chain)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info["object_type"] in ["BASE"]:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
timestamp = int(snapshot["timestamp"])
for bucket in bucket_chain:
if bucket["start"] >= timestamp > bucket["end"]:
bucket["snapshots"].append(
{
"timestamp": timestamp,
"snapshotid": snapshot["guid"],
"diskguid": vdisk.guid,
"is_consistent": snapshot["is_consistent"],
}
)
bucket_chains.append(bucket_chain)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket["snapshots"]:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot["is_consistent"] and not best["is_consistent"]:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif (
snapshot["is_consistent"] == best["is_consistent"]
and snapshot["timestamp"] > best["timestamp"]
):
best = snapshot
bucket["snapshots"] = [s for s in bucket["snapshots"] if s["timestamp"] != best["timestamp"]]
first = False
elif bucket["end"] > 0:
oldest = None
for snapshot in bucket["snapshots"]:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot["timestamp"] < oldest["timestamp"]:
oldest = snapshot
bucket["snapshots"] = [s for s in bucket["snapshots"] if s["timestamp"] != oldest["timestamp"]]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
for bucket in bucket_chain:
for snapshot in bucket["snapshots"]:
VDiskController.delete_snapshot(diskguid=snapshot["diskguid"], snapshotid=snapshot["snapshotid"])
logger.info("Delete snapshots finished")
def check_scrubbing_test():
"""
Check scrubbing of vdisks test
"""
initial_counter = 100
step = 5
vdisk = None
vpool_name = General.get_config().get('vpool', 'name')
vpool = GeneralVPool.get_vpool_by_name(vpool_name=vpool_name)
assert vpool, "No vpool found where one was expected"
template_folder = GeneralVMachine.template_target_folder
image_name = GeneralVMachine.template_image
disk_name = "scrubdisk"
GeneralVMachine.logger.info("Starting RAW disk creation")
out, err, _ = General.execute_command('qemu-img convert -O raw {0}{1} /mnt/{2}/{3}.raw'.format(template_folder, image_name, vpool_name, disk_name))
if err:
GeneralVMachine.logger.error("Error while creating raw disk: {0}".format(err))
def snapshot_vdisk(vdisk):
metadata = {'label': 'snap-' + vdisk.name,
'is_consistent': True,
'timestamp': time.time(),
'is_automatic': False,
'is_sticky': False}
VDiskController.create_snapshot(vdisk.guid, metadata)
counter = initial_counter
while counter and vdisk is None:
time.sleep(step)
vdisk = VDiskList.get_by_devicename_and_vpool('/' + disk_name + '.raw', vpool)
counter -= step
assert counter > 0, "Vdisk with name {0} didn't appear in the model after 60 seconds".format(disk_name)
# snapshot disks for the first time
snapshot_vdisk(vdisk)
counter = initial_counter
while counter > 0:
time.sleep(step)
out, err, _ = General.execute_command('dd if=/dev/zero of=/mnt/{0}/{1}.raw bs=10K count=1000 conv=notrunc'.format(vpool_name, disk_name))
counter -= step
snapshot_vdisk(vdisk)
# saving disk 'stored' info / the only attribute that is lowered after scrubbing
vdisk.invalidate_dynamics(['statistics'])
disk_backend_data = vdisk.statistics['stored']
# deleting middle snapshots
for snapshot in vdisk.snapshots[1:-1]:
VDiskController.delete_snapshot(vdisk.guid, snapshot['guid'])
# starting scrubber
try:
GenericController.execute_scrub()
# waiting for model to catch up
counter = initial_counter
while counter > 0:
time.sleep(step)
vdisk.invalidate_dynamics(['statistics'])
# checking result of scrub work
if vdisk.statistics['stored'] < disk_backend_data:
GeneralVMachine.logger.info("It took {0} seconds for the value to change from {1} to {2}\n".format((initial_counter - counter) * step,
disk_backend_data,
vdisk.statistics['stored']))
break
counter -= step
finally:
# removing vdisk
GeneralVMachine.logger.info("Removing vpool vdisks from {0} vpool".format(vpool_name))
out, err, _ = General.execute_command("rm -rf /mnt/{0}/*.raw".format(vpool_name))
if err:
GeneralVMachine.logger.error("Error while removing vdisk: {0}".format(err))
assert counter > 0, "Scrubbing didn't run as expected, backend size of vdisk remained at {0}:\n".format(disk_backend_data)
def delete_snapshots(timestamp=None):
"""
Delete snapshots & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
:param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used
"""
logger.info('Delete snapshots started')
day = timedelta(1)
week = day * 7
def make_timestamp(offset):
"""
Create an integer based timestamp
:param offset: Offset in days
:return: Timestamp
"""
return int(mktime((base - offset).timetuple()))
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
base = datetime.fromtimestamp(timestamp).date() - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({'start': make_timestamp(day * i),
'end': make_timestamp(day * (i + 1)),
'type': '1d',
'snapshots': []})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({'start': make_timestamp(week * i),
'end': make_timestamp(week * (i + 1)),
'type': '1w',
'snapshots': []})
buckets.append({'start': make_timestamp(week * 4),
'end': 0,
'type': 'rest',
'snapshots': []})
# Place all snapshots in bucket_chains
bucket_chains = []
for vmachine in VMachineList.get_customer_vmachines():
if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks):
bucket_chain = copy.deepcopy(buckets)
for snapshot in vmachine.snapshots:
if snapshot.get('is_sticky') is True:
continue
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems():
bucket['snapshots'].append({'timestamp': timestamp,
'snapshotid': snapshotguid,
'diskguid': diskguid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
if snapshot.get('is_sticky') is True:
continue
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({'timestamp': timestamp,
'snapshotid': snapshot['guid'],
'diskguid': vdisk.guid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot['is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != best['timestamp']]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != oldest['timestamp']]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(diskguid=snapshot['diskguid'],
snapshotid=snapshot['snapshotid'])
logger.info('Delete snapshots finished')
def deletescrubsnapshots(timestamp=None):
"""
Delete snapshots & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
"""
logger.info('Delete snapshots started')
day = 60 * 60 * 24
week = day * 7
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
offset = int(mktime(datetime.fromtimestamp(timestamp).date().timetuple())) - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({'start': offset - (day * i),
'end': offset - (day * (i + 1)),
'type': '1d',
'snapshots': []})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({'start': offset - (week * i),
'end': offset - (week * (i + 1)),
'type': '1w',
'snapshots': []})
buckets.append({'start': offset - (week * 4),
'end': 0,
'type': 'rest',
'snapshots': []})
# Place all snapshots in bucket_chains
bucket_chains = []
for vmachine in VMachineList.get_customer_vmachines():
if any(vd.info['object_type'] in ['BASE'] for vd in vmachine.vdisks):
bucket_chain = copy.deepcopy(buckets)
for snapshot in vmachine.snapshots:
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems():
bucket['snapshots'].append({'timestamp': timestamp,
'snapshotid': snapshotguid,
'diskguid': diskguid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({'timestamp': timestamp,
'snapshotid': snapshot['guid'],
'diskguid': vdisk.guid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot['is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != best['timestamp']]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != oldest['timestamp']]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(diskguid=snapshot['diskguid'],
snapshotid=snapshot['snapshotid'])
logger.info('Delete snapshots finished')
logger.info('Scrubbing started')
vdisks = []
for vmachine in VMachineList.get_customer_vmachines():
for vdisk in vmachine.vdisks:
if vdisk.info['object_type'] in ['BASE'] and len(vdisk.child_vdisks) == 0:
vdisks.append(vdisk)
for vdisk in VDiskList.get_without_vmachine():
if vdisk.info['object_type'] in ['BASE'] and len(vdisk.child_vdisks) == 0:
vdisks.append(vdisk)
total = 0
failed = 0
skipped = 0
storagedrivers = {}
for vdisk in vdisks:
try:
total += 1
# Load the vDisk's StorageDriver
vdisk.invalidate_dynamics(['info', 'storagedriver_id'])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[vdisk.storagedriver_id] = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
vdisk.invalidate_dynamics(['info'])
configs = vdisk.info['metadata_backend_config']
if len(configs) == 0:
raise RuntimeError('Could not load MDS configuration')
if configs[0]['ip'] != storagedriver.storagerouter.ip:
# The MDS master is not local. Trigger an MDS handover and try again
logger.debug('MDS for volume {0} is not local. Trigger handover'.format(vdisk.volume_id))
MDSServiceController.ensure_safety(vdisk)
vdisk.invalidate_dynamics(['info'])
configs = vdisk.info['metadata_backend_config']
if len(configs) == 0:
raise RuntimeError('Could not load MDS configuration')
if configs[0]['ip'] != storagedriver.storagerouter.ip:
skipped += 1
logger.info('Skipping scrubbing work unit for volume {0}: MDS master is not local'.format(
vdisk.volume_id
))
continue
work_units = vdisk.storagedriver_client.get_scrubbing_workunits(str(vdisk.volume_id))
for work_unit in work_units:
scrubbing_result = _storagedriver_scrubber.scrub(work_unit, str(storagedriver.mountpoint_temp))
vdisk.storagedriver_client.apply_scrubbing_result(scrubbing_result)
except Exception, ex:
failed += 1
logger.info('Failed scrubbing work unit for volume {0}: {1}'.format(
vdisk.volume_id, ex
))
def delete_snapshots(timestamp=None):
"""
Delete snapshots & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
:param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used
:type timestamp: float
:return: None
"""
GenericController._logger.info('Delete snapshots started')
day = timedelta(1)
week = day * 7
def make_timestamp(offset):
"""
Create an integer based timestamp
:param offset: Offset in days
:return: Timestamp
"""
return int(mktime((base - offset).timetuple()))
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
base = datetime.fromtimestamp(timestamp).date() - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({
'start': make_timestamp(day * i),
'end': make_timestamp(day * (i + 1)),
'type': '1d',
'snapshots': []
})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({
'start': make_timestamp(week * i),
'end': make_timestamp(week * (i + 1)),
'type': '1w',
'snapshots': []
})
buckets.append({
'start': make_timestamp(week * 4),
'end': 0,
'type': 'rest',
'snapshots': []
})
# Get a list of all snapshots that are used as parents for clones
parent_snapshots = set(
[vd.parentsnapshot for vd in VDiskList.get_with_parent_snaphots()])
# Place all snapshots in bucket_chains
bucket_chains = []
for vdisk in VDiskList.get_vdisks():
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
if snapshot.get('is_sticky') is True:
continue
if snapshot['guid'] in parent_snapshots:
GenericController._logger.info(
'Not deleting snapshot {0} because it has clones'.
format(snapshot['guid']))
continue
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({
'timestamp':
timestamp,
'snapshot_id':
snapshot['guid'],
'vdisk_guid':
vdisk.guid,
'is_consistent':
snapshot['is_consistent']
})
bucket_chains.append(bucket_chain)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot[
'is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [
s for s in bucket['snapshots']
if s['timestamp'] != best['timestamp']
]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [
s for s in bucket['snapshots']
if s['timestamp'] != oldest['timestamp']
]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(
vdisk_guid=snapshot['vdisk_guid'],
snapshot_id=snapshot['snapshot_id'])
GenericController._logger.info('Delete snapshots finished')
def delete_snapshots_storagedriver(storagedriver_guid, timestamp=None, group_id=None):
"""
Delete snapshots per storagedriver & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
:param storagedriver_guid: Guid of the StorageDriver to remove snapshots on
:type storagedriver_guid: str
:param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used
:type timestamp: float
:param group_id: ID of the group task. Used to identify which snapshot deletes were called during the scheduled task
:type group_id: str
:return: None
"""
if group_id:
log_id = 'Group job {} - '.format(group_id)
else:
log_id = ''
def format_log(message):
return '{}{}'.format(log_id, message)
GenericController._logger.info(format_log('Delete snapshots started for StorageDriver {0}'.format(storagedriver_guid)))
storagedriver = StorageDriver(storagedriver_guid)
exceptions = []
day = timedelta(1)
week = day * 7
def make_timestamp(offset):
"""
Create an integer based timestamp
:param offset: Offset in days
:return: Timestamp
"""
return int(mktime((base - offset).timetuple()))
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
base = datetime.fromtimestamp(timestamp).date() - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({'start': make_timestamp(day * i),
'end': make_timestamp(day * (i + 1)),
'type': '1d',
'snapshots': []})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({'start': make_timestamp(week * i),
'end': make_timestamp(week * (i + 1)),
'type': '1w',
'snapshots': []})
buckets.append({'start': make_timestamp(week * 4),
'end': 0,
'type': 'rest',
'snapshots': []})
# Get a list of all snapshots that are used as parents for clones
parent_snapshots = set([vd.parentsnapshot for vd in VDiskList.get_with_parent_snaphots()])
# Place all snapshots in bucket_chains
bucket_chains = []
for vdisk_guid in storagedriver.vdisks_guids:
try:
vdisk = VDisk(vdisk_guid)
vdisk.invalidate_dynamics('being_scrubbed')
if vdisk.being_scrubbed:
continue
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
if snapshot.get('is_sticky') is True:
continue
if snapshot['guid'] in parent_snapshots:
GenericController._logger.info(format_log('Not deleting snapshot {0} because it has clones'.format(snapshot['guid'])))
continue
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({'timestamp': timestamp,
'snapshot_id': snapshot['guid'],
'vdisk_guid': vdisk.guid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
except Exception as ex:
exceptions.append(ex)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot['is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != best['timestamp']]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != oldest['timestamp']]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
# Each bucket chain represents one vdisk's snapshots
try:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(vdisk_guid=snapshot['vdisk_guid'],
snapshot_id=snapshot['snapshot_id'])
except RuntimeError as ex:
vdisk_guid = next((snapshot['vdisk_guid'] for bucket in bucket_chain for snapshot in bucket['snapshots']), '')
vdisk_id_log = ''
if vdisk_guid:
vdisk_id_log = ' for VDisk with guid {}'.format(vdisk_guid)
if SCRUB_VDISK_EXCEPTION_MESSAGE in ex.message:
GenericController._logger.warning(format_log('Being scrubbed exception occurred while deleting snapshots{}'.format(vdisk_id_log)))
else:
GenericController._logger.exception(format_log('Exception occurred while deleting snapshots{}'.format(vdisk_id_log)))
exceptions.append(ex)
if exceptions:
raise RuntimeError('Exceptions occurred while deleting snapshots: \n- {}'.format('\n- '.join((str(ex) for ex in exceptions))))
GenericController._logger.info(format_log('Delete snapshots finished for StorageDriver {0}'))
def test_clone(self):
"""
Test the clone functionality
- Create a vDisk with name 'clone1'
- Clone the vDisk and make some assertions
- Attempt to clone again using same name and same devicename
- Attempt to clone on Storage Router which is not linked to the vPool on which the original vDisk is hosted
- Attempt to clone on Storage Driver without MDS service
- Attempt to clone from snapshot which is not yet completely synced to backend
- Attempt to delete the snapshot from which a clone was made
- Clone the vDisk on another Storage Router
- Clone another vDisk with name 'clone1' linked to another vPool
"""
structure = DalHelper.build_dal_structure({
'vpools': [1, 2],
'storagerouters': [1, 2, 3],
'storagedrivers':
[(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]
} # (<id>, <storagedriver_id>)
)
vpools = structure['vpools']
mds_services = structure['mds_services']
service_type = structure['service_types']['MetadataServer']
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpools[1],
storagerouters=storagerouters)
self._roll_out_dtl_services(vpool=vpools[2],
storagerouters=storagerouters)
# Basic clone scenario
vdisk1 = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[1].guid))
clone1_info = VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks')
clones = VDiskList.get_by_parentsnapshot(vdisk1.snapshot_ids[0])
self.assertTrue(expr=len(clones) == 1,
msg='Expected to find 1 vDisk with parent snapshot')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 1,
msg='Expected to find 1 child vDisk')
for expected_key in ['vdisk_guid', 'name', 'backingdevice']:
self.assertTrue(
expr=expected_key in clone1_info,
msg='Expected to find key "{0}" in clone_info'.format(
expected_key))
self.assertTrue(expr=clones[0].guid == clone1_info['vdisk_guid'],
msg='Guids do not match')
self.assertTrue(expr=clones[0].name == clone1_info['name'],
msg='Names do not match')
self.assertTrue(
expr=clones[0].devicename == clone1_info['backingdevice'],
msg='Device names do not match')
# Attempt to clone again with same name
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 1')
# Attempt to clone again with a name which will have identical devicename
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1%')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 2')
# Attempt to clone on Storage Router on which vPool is not extended
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
storagerouter_guid=storagerouters[2].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 3')
# Attempt to clone on non-existing Storage Driver
storagedrivers[1].storagedriver_id = 'non-existing'
storagedrivers[1].save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 4')
storagedrivers[1].storagedriver_id = '1'
storagedrivers[1].save()
# Attempt to clone on Storage Driver without MDS service
mds_services[1].service.storagerouter = storagerouters[3]
mds_services[1].service.save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 5')
mds_services[1].service.storagerouter = storagerouters[1]
mds_services[1].service.save()
# Attempt to clone by providing snapshot_id not synced to backend
self.assertTrue(expr=len(vdisk1.snapshots) == 1,
msg='Expected to find only 1 snapshot before cloning')
self.assertTrue(
expr=len(vdisk1.snapshot_ids) == 1,
msg='Expected to find only 1 snapshot ID before cloning')
metadata = {
'label': 'label1',
'timestamp': int(time.time()),
'is_sticky': False,
'in_backend': False,
'is_automatic': True,
'is_consistent': True
}
snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid,
metadata=metadata)
self.assertTrue(expr=len(vdisk1.snapshots) == 2,
msg='Expected to find 2 snapshots')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 2,
msg='Expected to find 2 snapshot IDs')
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 6')
# Update backend synced flag and retry
vdisk1.storagedriver_client._set_snapshot_in_backend(
vdisk1.volume_id, snapshot_id, True)
vdisk1.invalidate_dynamics(['snapshots', 'snapshot_ids'])
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
vdisk1.invalidate_dynamics()
self.assertTrue(expr=len(vdisks) == 3, msg='Expected to find 3 vDisks')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 2,
msg='Expected to find 2 child vDisks')
self.assertTrue(
expr=len(vdisk1.snapshots) == 2,
msg=
'Expected to find 2 snapshots after cloning from a specified snapshot'
)
self.assertTrue(
expr=len(vdisk1.snapshot_ids) == 2,
msg=
'Expected to find 2 snapshot IDs after cloning from a specified snapshot'
)
# Attempt to delete the snapshot that has clones
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=snapshot_id)
# Clone on specific Storage Router
storagedriver = StorageDriver()
storagedriver.vpool = vpools[1]
storagedriver.storagerouter = storagerouters[2]
storagedriver.name = '3'
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[2].ip
storagedriver.storage_ip = '127.0.0.1'
storagedriver.storagedriver_id = '3'
storagedriver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storagedriver.save()
s_id = '{0}-1'.format(storagedriver.storagerouter.name)
service = Service()
service.name = s_id
service.storagerouter = storagedriver.storagerouter
service.ports = [3]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storagedriver.vpool
mds_service.save()
clone3 = VDisk(
VDiskController.clone(
vdisk_guid=vdisk1.guid,
name='clone3',
storagerouter_guid=storagerouters[2].guid)['vdisk_guid'])
self.assertTrue(
expr=clone3.storagerouter_guid == storagerouters[2].guid,
msg='Incorrect Storage Router on which the clone is attached')
# Clone vDisk with existing name on another vPool
vdisk2 = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[2].guid))
clone_vdisk2 = VDisk(
VDiskController.clone(vdisk_guid=vdisk2.guid,
name='clone1')['vdisk_guid'])
self.assertTrue(
expr=clone_vdisk2.vpool == vpools[2],
msg='Cloned vDisk with name "clone1" was created on incorrect vPool'
)
self.assertTrue(expr=len([
vdisk for vdisk in VDiskList.get_vdisks() if vdisk.name == 'clone1'
]) == 2,
msg='Expected to find 2 vDisks with name "clone1"')
# Attempt to clone without specifying snapshot and snapshot fails to sync to backend
StorageRouterClient.synced = False
vdisk2 = VDisk(
VDiskController.create_new(
volume_name='vdisk_2',
volume_size=1024**3,
storagedriver_guid=storagedrivers[1].guid))
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk2.guid, name='clone4')
vdisk2.invalidate_dynamics()
self.assertTrue(expr=len(vdisk2.snapshots) == 0,
msg='Expected to find 0 snapshots after clone failure')
self.assertTrue(
expr=len(vdisk2.snapshot_ids) == 0,
msg='Expected to find 0 snapshot IDs after clone failure')
self.assertTrue(expr=len(vdisk2.child_vdisks) == 0,
msg='Expected to find 0 children after clone failure')
StorageRouterClient.synced = True
def test_clone(self):
"""
Test the clone functionality
- Create a vDisk with name 'clone1'
- Clone the vDisk and make some assertions
- Attempt to clone again using same name and same devicename
- Attempt to clone on Storage Router which is not linked to the vPool on which the original vDisk is hosted
- Attempt to clone on Storage Driver without MDS service
- Attempt to clone from snapshot which is not yet completely synced to backend
- Attempt to delete the snapshot from which a clone was made
- Clone the vDisk on another Storage Router
- Clone another vDisk with name 'clone1' linked to another vPool
"""
structure = Helper.build_service_structure(
{'vpools': [1, 2],
'storagerouters': [1, 2, 3],
'storagedrivers': [(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]} # (<id>, <storagedriver_id>)
)
vpools = structure['vpools']
mds_services = structure['mds_services']
service_type = structure['service_type']
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpools[1], storagerouters=storagerouters)
self._roll_out_dtl_services(vpool=vpools[2], storagerouters=storagerouters)
# Basic clone scenario
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
clone1_info = VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks')
clones = VDiskList.get_by_parentsnapshot(vdisk1.snapshots[0]['guid'])
self.assertTrue(expr=len(clones) == 1, msg='Expected to find 1 vDisk with parent snapshot')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 1, msg='Expected to find 1 child vDisk')
for expected_key in ['vdisk_guid', 'name', 'backingdevice']:
self.assertTrue(expr=expected_key in clone1_info, msg='Expected to find key "{0}" in clone_info'.format(expected_key))
self.assertTrue(expr=clones[0].guid == clone1_info['vdisk_guid'], msg='Guids do not match')
self.assertTrue(expr=clones[0].name == clone1_info['name'], msg='Names do not match')
self.assertTrue(expr=clones[0].devicename == clone1_info['backingdevice'], msg='Device names do not match')
# Attempt to clone again with same name
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 1')
# Attempt to clone again with a name which will have identical devicename
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1%')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 2')
# Attempt to clone on Storage Router on which vPool is not extended
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
storagerouter_guid=storagerouters[2].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 3')
# Attempt to clone on non-existing Storage Driver
storagedrivers[1].storagedriver_id = 'non-existing'
storagedrivers[1].save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 4')
storagedrivers[1].storagedriver_id = '1'
storagedrivers[1].save()
# Attempt to clone on Storage Driver without MDS service
mds_services[1].service.storagerouter = storagerouters[3]
mds_services[1].service.save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 5')
mds_services[1].service.storagerouter = storagerouters[1]
mds_services[1].service.save()
# Attempt to clone by providing snapshot_id not synced to backend
self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find only 1 snapshot before cloning')
metadata = {'label': 'label1',
'timestamp': int(time.time()),
'is_sticky': False,
'in_backend': False,
'is_automatic': True,
'is_consistent': True}
snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata=metadata)
self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots')
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 6')
# Update backend synced flag and retry
vdisk1.storagedriver_client._set_snapshot_in_backend(vdisk1.volume_id, snapshot_id, True)
vdisk1.invalidate_dynamics('snapshots')
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
vdisk1.invalidate_dynamics()
self.assertTrue(expr=len(vdisks) == 3, msg='Expected to find 3 vDisks')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 2, msg='Expected to find 2 child vDisks')
self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots after cloning from a specified snapshot')
# Attempt to delete the snapshot that has clones
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=snapshot_id)
# Clone on specific Storage Router
storagedriver = StorageDriver()
storagedriver.vpool = vpools[1]
storagedriver.storagerouter = storagerouters[2]
storagedriver.name = '3'
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[2].ip
storagedriver.storage_ip = '127.0.0.1'
storagedriver.storagedriver_id = '3'
storagedriver.ports = {'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4}
storagedriver.save()
s_id = '{0}-1'.format(storagedriver.storagerouter.name)
service = Service()
service.name = s_id
service.storagerouter = storagedriver.storagerouter
service.ports = [3]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storagedriver.vpool
mds_service.save()
clone3 = VDisk(VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone3',
storagerouter_guid=storagerouters[2].guid)['vdisk_guid'])
self.assertTrue(expr=clone3.storagerouter_guid == storagerouters[2].guid, msg='Incorrect Storage Router on which the clone is attached')
# Clone vDisk with existing name on another vPool
vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[2].guid))
clone_vdisk2 = VDisk(VDiskController.clone(vdisk_guid=vdisk2.guid,
name='clone1')['vdisk_guid'])
self.assertTrue(expr=clone_vdisk2.vpool == vpools[2], msg='Cloned vDisk with name "clone1" was created on incorrect vPool')
self.assertTrue(expr=len([vdisk for vdisk in VDiskList.get_vdisks() if vdisk.name == 'clone1']) == 2, msg='Expected to find 2 vDisks with name "clone1"')
# Attempt to clone without specifying snapshot and snapshot fails to sync to backend
StorageRouterClient.synced = False
vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_2', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk2.guid,
name='clone4')
vdisk2.invalidate_dynamics()
self.assertTrue(expr=len(vdisk2.snapshots) == 0, msg='Expected to find 0 snapshots after clone failure')
self.assertTrue(expr=len(vdisk2.child_vdisks) == 0, msg='Expected to find 0 children after clone failure')
StorageRouterClient.synced = True
