def devicename_exists(self, vpool, name=None, names=None):
"""
Checks whether a given name can be created on the vpool
:param vpool: vPool object
:type vpool: VPool
:param name: Candidate name
:type name: str
:param names: Candidate names
:type names: list
:return: Whether the devicename exists
:rtype: bool
"""
error_message = None
if not (name is None) ^ (names is None):
error_message = 'Either the name (string) or the names (list of strings) parameter must be passed'
if name is not None and not isinstance(name, basestring):
error_message = 'The name parameter must be a string'
if names is not None and not isinstance(names, list):
error_message = 'The names parameter must be a list of strings'
if error_message is not None:
raise HttpNotAcceptableException(error_description=error_message,
error='impossible_request')
if name is not None:
devicename = VDiskController.clean_devicename(name)
return VDiskList.get_by_devicename_and_vpool(devicename,
vpool) is not None
for name in names:
devicename = VDiskController.clean_devicename(name)
if VDiskList.get_by_devicename_and_vpool(devicename,
vpool) is not None:
return True
return False
def devicename_exists(self, vpool, name=None, names=None):
"""
Checks whether a given name can be created on the vpool
:param vpool: vPool object
:type vpool: VPool
:param name: Candidate name
:type name: str
:param names: Candidate names
:type names: list
:return: Whether the devicename exists
:rtype: bool
"""
error_message = None
if not (name is None) ^ (names is None):
error_message = 'Either the name (string) or the names (list of strings) parameter must be passed'
if name is not None and not isinstance(name, basestring):
error_message = 'The name parameter must be a string'
if names is not None and not isinstance(names, list):
error_message = 'The names parameter must be a list of strings'
if error_message is not None:
raise HttpNotAcceptableException(error_description=error_message,
error='impossible_request')
if name is not None:
devicename = VDiskController.clean_devicename(name)
return VDiskList.get_by_devicename_and_vpool(devicename, vpool) is not None
for name in names:
devicename = VDiskController.clean_devicename(name)
if VDiskList.get_by_devicename_and_vpool(devicename, vpool) is not None:
return True
return False
def set_as_template(machineguid):
"""
Set a vmachine as template
@param machineguid: guid of the machine
@return: vmachine template conversion successful: True|False
"""
# Do some magic on the storage layer?
# This is most likely required as extra security measure
# Suppose the template is set back to a real machine
# it can be deleted within vmware which should be blocked.
# This might also require a storagerouter internal check
# to be implemented to discourage volumes from being deleted
# when clones were made from it.
vmachine = VMachine(machineguid)
if vmachine.hypervisor_status == 'RUNNING':
raise RuntimeError(
'vMachine {0} may not be running to set it as vTemplate'.
format(vmachine.name))
for disk in vmachine.vdisks:
VDiskController.set_as_template(diskguid=disk.guid)
vmachine.is_vtemplate = True
vmachine.invalidate_dynamics(['snapshots'])
vmachine.save()
def create_volume(self, volume):
"""Creates a volume.
Called on "cinder create ..." or "nova volume-create ..."
:param volume: volume reference (sqlalchemy Model)
"""
_debug_vol_info("CREATE", volume)
hostname = str(volume.host)
name = volume.display_name
if not name:
name = volume.name
mountpoint = self._get_hostname_mountpoint(hostname)
location = '{}/{}.raw'.format(mountpoint, name)
size = volume.size
LOG.info('DO_CREATE_VOLUME %s %s' % (location, size))
VDiskController.create_volume(location = location,
size = size)
volume['provider_location'] = location
try:
ovs_disk = self._find_ovs_model_disk_by_location(location,
hostname)
except RuntimeError:
VDiskController.delete_volume(location = location)
raise
ovs_disk.cinder_id = volume.id
ovs_disk.name = name
ovs_disk.save()
return {'provider_location': volume['provider_location']}
def _run_and_validate_dtl_checkup(self, vdisk, validations):
"""
Execute the DTL checkup for a vDisk and validate the settings afterwards
"""
single_node = len(StorageRouterList.get_storagerouters()) == 1
VDiskController.dtl_checkup(vdisk_guid=vdisk.guid)
config = vdisk.storagedriver_client.get_dtl_config(vdisk.volume_id)
config_mode = vdisk.storagedriver_client.get_dtl_config_mode(vdisk.volume_id)
msg = '{0} node - {{0}} - Actual: {{1}} - Expected: {{2}}'.format('Single' if single_node is True else 'Multi')
validations.append({'key': 'config_mode', 'value': DTLConfigMode.MANUAL})
for validation in validations:
key = validation['key']
value = validation['value']
if key == 'config':
actual_value = config
elif key == 'host':
actual_value = config.host
elif key == 'port':
actual_value = config.port
elif key == 'mode':
actual_value = config.mode
else:
actual_value = config_mode
if isinstance(value, list):
self.assertTrue(expr=actual_value in value,
msg=msg.format(key.capitalize(), actual_value, ', '.join(value)))
else:
self.assertEqual(first=actual_value,
second=value,
msg=msg.format(key.capitalize(), actual_value, value))
return config
def test_event_migrate_from_volumedriver(self):
"""
Test migrate from volumedriver event
"""
_ = self
structure = DalHelper.build_dal_structure({
'vpools': [1],
'storagerouters': [1, 2],
'storagedrivers':
[(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]
} # (<id>, <storagedriver_id>)
)
vpool = structure['vpools'][1]
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpool, storagerouters=storagerouters)
vdisk = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**4,
storagedriver_guid=storagedrivers[1].guid))
vdisk.storagedriver_client.migrate(vdisk.volume_id,
storagedrivers[2].storagedriver_id,
False)
VDiskController.migrate_from_voldrv(
volume_id=vdisk.volume_id,
new_owner_id=storagedrivers[2].storagedriver_id)
self.assertEqual(vdisk.storagedriver_id,
storagedrivers[2].storagedriver_id)
def setAsTemplate(templatepath):
disk = getVDisk(templatepath, timeout=60)
if not disk:
raise RuntimeError("Template did not become available on OVS at %s" % templatepath)
if disk.info['object_type'] != 'TEMPLATE':
VDiskController.set_as_template(disk.guid)
return disk.guid
def snapshot_vdisk(vdisk):
metadata = {'label': 'snap-' + vdisk.name,
'is_consistent': True,
'timestamp': time.time(),
'is_automatic': False,
'is_sticky': False}
VDiskController.create_snapshot(vdisk.guid, metadata)
def create_volume(self, volume):
"""Creates a volume.
Called on "cinder create ..." or "nova volume-create ..."
:param volume: volume reference (sqlalchemy Model)
"""
_debug_vol_info("CREATE", volume)
hostname = str(volume.host)
name = volume.display_name
if not name:
name = volume.name
mountpoint = self._get_hostname_mountpoint(hostname)
location = '{}/{}.raw'.format(mountpoint, name)
size = volume.size
LOG.info('DO_CREATE_VOLUME %s %s' % (location, size))
VDiskController.create_volume(location=location, size=size)
volume['provider_location'] = location
try:
ovs_disk = self._find_ovs_model_disk_by_location(
location, hostname)
except RuntimeError:
VDiskController.delete_volume(location=location)
raise
ovs_disk.cinder_id = volume.id
ovs_disk.name = name
ovs_disk.save()
return {'provider_location': volume['provider_location']}
def _run_and_validate_dtl_checkup(self, vdisk, validations):
"""
Execute the DTL checkup for a vDisk and validate the settings afterwards
"""
single_node = len(StorageRouterList.get_storagerouters()) == 1
VDiskController.dtl_checkup(vdisk_guid=vdisk.guid)
config = vdisk.storagedriver_client.get_dtl_config(vdisk.volume_id)
config_mode = vdisk.storagedriver_client.get_dtl_config_mode(vdisk.volume_id)
msg = '{0} node - {{0}} - Actual: {{1}} - Expected: {{2}}'.format('Single' if single_node is True else 'Multi')
validations.append({'key': 'config_mode', 'value': DTLConfigMode.MANUAL})
for validation in validations:
key = validation['key']
value = validation['value']
if key == 'config':
actual_value = config
elif key == 'host':
actual_value = config.host
elif key == 'port':
actual_value = config.port
elif key == 'mode':
actual_value = config.mode
else:
actual_value = config_mode
if isinstance(value, list):
self.assertTrue(expr=actual_value in value,
msg=msg.format(key.capitalize(), actual_value, ', '.join(value)))
else:
self.assertEqual(first=actual_value,
second=value,
msg=msg.format(key.capitalize(), actual_value, value))
return config
def rollback(machineguid, timestamp):
"""
Rolls back a VM based on a given disk snapshot timestamp
:param machineguid: Guid of the machine to rollback
:param timestamp: Timestamp to rollback to
"""
vmachine = VMachine(machineguid)
if vmachine.hypervisor_status == 'RUNNING':
raise RuntimeError(
'vMachine {0} may not be running to set it as vTemplate'.
format(vmachine.name))
snapshots = [
snap for snap in vmachine.snapshots
if snap['timestamp'] == timestamp
]
if not snapshots:
raise ValueError(
'No vmachine snapshots found for timestamp {0}'.format(
timestamp))
for disk in vmachine.vdisks:
VDiskController.rollback(diskguid=disk.guid, timestamp=timestamp)
vmachine.invalidate_dynamics(['snapshots'])
def create_snapshot(self, snapshot):
"""Creates a snapshot.
Called on "nova image-create " or "cinder snapshot-create "
:param snapshot: snapshot reference (sqlalchemy Model)
"""
_debug_vol_info('CREATE_SNAP', snapshot)
volume = snapshot.volume
_debug_vol_info('CREATE_SNAP_VOL', volume)
hostname = volume.host
location = volume.provider_location
ovs_disk = self._find_ovs_model_disk_by_location(location, hostname)
metadata = {
'label': "{0} (OpenStack)".format(snapshot.display_name),
'is_consistent': False,
'timestamp': time.time(),
'machineguid': ovs_disk.vmachine_guid,
'is_automatic': False
}
LOG.debug('CREATE_SNAP %s %s' % (snapshot.display_name, str(metadata)))
VDiskController.create_snapshot(diskguid=ovs_disk.guid,
metadata=metadata,
snapshotid=str(snapshot.id))
LOG.debug('CREATE_SNAP OK')
def copy_image_to_volume(self, context, volume, image_service, image_id):
"""Copy image to volume
Called on "nova volume-create --image-id ..."
or "cinder create --image-id"
Downloads image from glance server into local .raw
:param volume: volume reference (sqlalchemy Model)
"""
_debug_vol_info("CP_IMG_TO_VOL", volume)
LOG.info("CP_IMG_TO_VOL %s %s" % (image_service, image_id))
name = volume.display_name
if not name:
name = volume.name
volume.display_name = volume.name
# downloading from an existing image
destination_path = volume.provider_location
if destination_path:
try:
LOG.info("CP_IMG_TO_VOL Deleting existing empty raw file %s " % destination_path)
VDiskController.delete_volume(location=destination_path)
LOG.info("CP_IMG_TO_VOL Downloading image to %s" % destination_path)
image_utils.fetch_to_raw(context, image_service, image_id, destination_path, "1M", size=volume["size"])
LOG.info("CP_IMG_TO_VOL Resizing volume to size %s" % volume["size"])
self.extend_volume(volume=volume, size_gb=volume["size"])
except Exception as ex:
LOG.error("CP_IMG_TO_VOL Internal error %s " % unicode(ex))
self.delete_volume(volume)
raise
ovs_disk = self._find_ovs_model_disk_by_location(volume.provider_location, str(volume.host))
ovs_disk.name = name
ovs_disk.save()
def download_to_vpool(url, path, overwrite_if_exists=False):
"""
Special method to download to vpool because voldrv does not support extending file at write
:param url: URL to download from
:type url: str
:param path: Path to download to
:type path: str
:param overwrite_if_exists: Overwrite if file already exists
:type overwrite_if_exists: bool
:return: None
"""
print url
print path
if os.path.exists(path) and not overwrite_if_exists:
return
u = urllib.urlopen(url)
file_size = u.info()['Content-Length']
bsize = 4096 * 1024
VDiskController.create_volume(path, 0)
with open(path, "wb") as f:
size_written = 0
os.ftruncate(f.fileno(), int(file_size))
while 1:
s = u.read(bsize)
size_written += len(s)
f.write(s)
if len(s) < bsize:
break
u.close()
def set_as_template(machineguid):
"""
Set a vmachine as template
@param machineguid: guid of the machine
@return: vmachine template conversion successful: True|False
"""
# Do some magic on the storage layer?
# This is most likely required as extra security measure
# Suppose the template is set back to a real machine
# it can be deleted within vmware which should be blocked.
# This might also require a storagerouter internal check
# to be implemented to discourage volumes from being deleted
# when clones were made from it.
vmachine = VMachine(machineguid)
if vmachine.hypervisor_status == 'RUNNING':
raise RuntimeError('vMachine {0} may not be running to set it as vTemplate'.format(vmachine.name))
for disk in vmachine.vdisks:
VDiskController.set_as_template(diskguid=disk.guid)
vmachine.is_vtemplate = True
vmachine.invalidate_dynamics(['snapshots'])
vmachine.save()
def snapshot_all_vdisks():
"""
Snapshots all vDisks
"""
GenericController._logger.info('[SSA] started')
success = []
fail = []
for vdisk in VDiskList.get_vdisks():
if vdisk.is_vtemplate is True:
continue
try:
metadata = {
'label': '',
'is_consistent': False,
'timestamp': str(int(time.time())),
'is_automatic': True,
'is_sticky': False
}
VDiskController.create_snapshot(vdisk_guid=vdisk.guid,
metadata=metadata)
success.append(vdisk.guid)
except Exception:
GenericController._logger.exception(
'Error taking snapshot for vDisk {0}'.format(vdisk.guid))
fail.append(vdisk.guid)
GenericController._logger.info(
'[SSA] Snapshot has been taken for {0} vDisks, {1} failed.'.format(
len(success), len(fail)))
return success, fail
def validate(self, storagerouter=None, storagedriver=None):
"""
Perform some validations before creating or extending a vPool
:param storagerouter: StorageRouter on which the vPool will be created or extended
:type storagerouter: ovs.dal.hybrids.storagerouter.StorageRouter
:param storagedriver: When passing a StorageDriver, perform validations when shrinking a vPool
:type storagedriver: ovs.dal.hybrids.storagedriver.StorageDriver
:raises ValueError: If extending a vPool which status is not RUNNING
RuntimeError: If this vPool's configuration does not meet the requirements
If the vPool has already been extended on the specified StorageRouter
:return: None
:rtype: NoneType
"""
if self.vpool is not None:
if self.vpool.status != VPool.STATUSES.RUNNING:
raise ValueError('vPool should be in {0} status'.format(
VPool.STATUSES.RUNNING))
ExtensionsToolbox.verify_required_params(
actual_params=self.vpool.configuration,
required_params={
'sco_size':
(int, StorageDriverClient.TLOG_MULTIPLIER_MAP.keys()),
'dtl_mode':
(str, StorageDriverClient.VPOOL_DTL_MODE_MAP.keys()),
'write_buffer': (float, None),
'dtl_transport':
(str, StorageDriverClient.VPOOL_DTL_TRANSPORT_MAP.keys()),
'tlog_multiplier':
(int, StorageDriverClient.TLOG_MULTIPLIER_MAP.values())
})
if storagerouter is not None:
for vpool_storagedriver in self.vpool.storagedrivers:
if vpool_storagedriver.storagerouter_guid == storagerouter.guid:
raise RuntimeError(
'A StorageDriver is already linked to this StorageRouter for vPool {0}'
.format(self.vpool.name))
if storagedriver is not None:
VDiskController.sync_with_reality(vpool_guid=self.vpool.guid)
storagedriver.invalidate_dynamics('vdisks_guids')
if len(storagedriver.vdisks_guids) > 0:
raise RuntimeError(
'There are still vDisks served from the given StorageDriver'
)
self.mds_services = [
mds_service for mds_service in self.vpool.mds_services
if mds_service.service.storagerouter_guid ==
storagedriver.storagerouter_guid
]
for mds_service in self.mds_services:
if len(mds_service.storagedriver_partitions
) == 0 or mds_service.storagedriver_partitions[
0].storagedriver is None:
raise RuntimeError(
'Failed to retrieve the linked StorageDriver to this MDS Service {0}'
.format(mds_service.service.name))
def extend_volume(self, volume, size_gb):
"""Extend volume to new size size_gb
"""
_debug_vol_info("EXTEND_VOL", volume)
LOG.info("EXTEND_VOL Size %s" % size_gb)
location = volume.provider_location
if location is not None:
LOG.info("DO_EXTEND_VOLUME %s" % (location))
VDiskController.extend_volume(location=location, size=size_gb)
def extend_volume(self, volume, size_gb):
"""Extend volume to new size size_gb
"""
_debug_vol_info('EXTEND_VOL', volume)
LOG.info('EXTEND_VOL Size %s' % size_gb)
location = volume.provider_location
if location is not None:
LOG.info('DO_EXTEND_VOLUME %s' % (location))
VDiskController.extend_volume(location=location, size=size_gb)
def test_sync_vdisk_with_voldrv(self):
clone_depth = 3
def _make_clones(vdisks_map, depth=clone_depth):
for level in range(depth):
previous_vd = list(vdisks_map.itervalues())[-1]
new_name = previous_vd.name + '_clone'
new_guid = VDiskController.clone(previous_vd.guid,
new_name).get('vdisk_guid')
vdisks_map[new_name] = VDisk(new_guid)
structure = DalHelper.build_dal_structure({
'vpools': [1],
'storagerouters': [1],
'storagedrivers':
[(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]
} # (<id>, <storagedriver_id>)
)
vdisk_name = 'vdisk_1'
storagedriver = structure['storagedrivers'][1]
vdisk_1 = VDisk(
VDiskController.create_new(volume_name=vdisk_name,
volume_size=1024**4,
storagedriver_guid=storagedriver.guid))
vdisks = OrderedDict()
vdisks[vdisk_name] = vdisk_1
_make_clones(vdisks)
self.assertEquals(clone_depth + 1, len(list(VDiskList.get_vdisks())))
delete_list = list(vdisks.itervalues(
))[::-1][:-1] # These vDisks are clones and ought to be deleted
for vdisk in delete_list:
for mds_service in vdisk.mds_services:
mds_service.delete()
vdisk.delete()
self.assertEquals(1, len(list(VDiskList.get_vdisks()))
) # Make sure vDisk clones are properly removed
self.assertEquals(
VDiskList.get_vdisks()[0].name,
vdisk_name) # Make sure only item left is original vDisk
VDiskController.sync_with_reality()
self.assertEquals(clone_depth + 1, len(list(
VDiskList.get_vdisks()))) # The clones should be in place now
parents = 0
for vdisk in VDiskList.get_vdisks():
try:
if vdisk.parent_vdisk.name:
parents += 1
except AttributeError:
pass
self.assertEquals(
clone_depth, parents
) # As much parents should be detected as the depth of the clones
def test_event_resize_from_volumedriver(self):
"""
Test resize from volumedriver event
- Create a vDisk using the resize event
- Resize the created vDisk using the same resize event
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
vpools = structure['vpools']
storagedrivers = structure['storagedrivers']
mds_service = structure['mds_services'][1]
# Create volume using resize from voldrv
device_name = '/vdisk.raw'
srclient = StorageRouterClient(vpools[1].guid, None)
mds_backend_config = Helper._generate_mdsmetadatabackendconfig([mds_service])
volume_id = srclient.create_volume(device_name, mds_backend_config, 1024 ** 4, str(storagedrivers[1].storagedriver_id))
VDiskController.resize_from_voldrv(volume_id=volume_id,
volume_size=1024 ** 4,
volume_path=device_name,
storagedriver_id=storagedrivers[1].storagedriver_id)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 1,
msg='Expected to find 1 vDisk in model')
self.assertEqual(first=vdisks[0].name,
second='vdisk',
msg='Volume name should be vdisk')
self.assertEqual(first=vdisks[0].volume_id,
second=volume_id,
msg='Volume ID should be {0}'.format(volume_id))
self.assertEqual(first=vdisks[0].devicename,
second=device_name,
msg='Device name should be {0}'.format(device_name))
self.assertEqual(first=vdisks[0].size,
second=1024 ** 4,
msg='Size should be 1 TiB')
# Resize volume using resize from voldrv
VDiskController.resize_from_voldrv(volume_id=volume_id,
volume_size=2 * 1024 ** 4,
volume_path=device_name,
storagedriver_id=storagedrivers[1].storagedriver_id)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 1,
msg='Expected to find 1 vDisk in model')
self.assertEqual(first=vdisks[0].name,
second='vdisk',
msg='Volume name should be vdisk')
self.assertEqual(first=vdisks[0].size,
second=2 * 1024 ** 4,
msg='Size should be 2 TiB')
def snapshot(machineguid,
label=None,
is_consistent=False,
timestamp=None,
is_automatic=False):
"""
Snapshot VMachine disks
@param machineguid: guid of the machine
@param label: label to give the snapshots
@param is_consistent: flag indicating the snapshot was consistent or not
@param timestamp: override timestamp, if required. Should be a unix timestamp
"""
timestamp = timestamp if timestamp is not None else time.time()
timestamp = str(int(float(timestamp)))
metadata = {
'label': label,
'is_consistent': is_consistent,
'timestamp': timestamp,
'machineguid': machineguid,
'is_automatic': is_automatic
}
machine = VMachine(machineguid)
# @todo: we now skip creating a snapshot when a vmachine's disks
# is missing a mandatory property: volume_id
# subtask will now raise an exception earlier in the workflow
for disk in machine.vdisks:
if not disk.volume_id:
message = 'Missing volume_id on disk {0} - unable to create snapshot for vm {1}'.format(
disk.guid, machine.guid)
logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
snapshots = {}
success = True
try:
for disk in machine.vdisks:
snapshots[disk.guid] = VDiskController.create_snapshot(
diskguid=disk.guid, metadata=metadata)
except Exception as ex:
logger.info('Error snapshotting disk {0}: {1}'.format(
disk.name, str(ex)))
success = False
for diskguid, snapshotid in snapshots.iteritems():
VDiskController.delete_snapshot(diskguid=diskguid,
snapshotid=snapshotid)
logger.info('Create snapshot for vMachine {0}: {1}'.format(
machine.name, 'Success' if success else 'Failure'))
machine.invalidate_dynamics(['snapshots'])
if not success:
raise RuntimeError('Failed to snapshot vMachine {0}'.format(
machine.name))
def delete_volume(self, volume):
"""Deletes a logical volume.
Called on "cinder delete ... "
:param volume: volume reference (sqlalchemy Model)
"""
_debug_vol_info("DELETE", volume)
location = volume.provider_location
if location is not None:
LOG.info('DO_DELETE_VOLUME %s' % (location))
VDiskController.delete_volume(location = location)
def delete_volume(self, volume):
"""Deletes a logical volume.
Called on "cinder delete ... "
:param volume: volume reference (sqlalchemy Model)
"""
_debug_vol_info("DELETE", volume)
location = volume.provider_location
if location is not None:
LOG.info('DO_DELETE_VOLUME %s' % (location))
VDiskController.delete_volume(location=location)
def snapshot(machineguid, label=None, is_consistent=False, timestamp=None, is_automatic=False, is_sticky=False):
"""
Snapshot VMachine disks
:param machineguid: guid of the machine
:param label: label to give the snapshots
:param is_consistent: flag indicating the snapshot was consistent or not
:param timestamp: override timestamp, if required. Should be a unix timestamp
:param is_automatic: Flag to determine automated snapshots
:param is_sticky: Flag indicating the snapshot is not to be deleted automatically
"""
timestamp = timestamp if timestamp is not None else time.time()
timestamp = str(int(float(timestamp)))
if is_automatic is True and is_sticky is True:
raise ValueError('Snapshot {0} cannot be both automatic and sticky'.format(label))
metadata = {'label': label,
'is_consistent': is_consistent,
'timestamp': timestamp,
'machineguid': machineguid,
'is_automatic': is_automatic,
'is_sticky': is_sticky}
machine = VMachine(machineguid)
# @todo: we now skip creating a snapshot when a vmachine's disks
# is missing a mandatory property: volume_id
# sub-task will now raise an exception earlier in the workflow
for disk in machine.vdisks:
if not disk.volume_id:
message = 'Missing volume_id on disk {0} - unable to create snapshot for vm {1}'.format(
disk.guid, machine.guid
)
VMachineController._logger.info('Error: {0}'.format(message))
raise RuntimeError(message)
snapshots = {}
success = True
for disk in machine.vdisks:
try:
snapshots[disk.guid] = VDiskController.create_snapshot(diskguid=disk.guid,
metadata=metadata)
except Exception as ex:
VMachineController._logger.info('Error taking snapshot of disk {0}: {1}'.format(disk.name, str(ex)))
success = False
for diskguid, snapshotid in snapshots.iteritems():
VDiskController.delete_snapshot(diskguid=diskguid,
snapshotid=snapshotid)
VMachineController._logger.info('Create snapshot for vMachine {0}: {1}'.format(
machine.name, 'Success' if success else 'Failure'
))
machine.invalidate_dynamics(['snapshots'])
if not success:
raise RuntimeError('Failed to snapshot vMachine {0}'.format(machine.name))
def delete_snapshot(self, snapshot):
"""Deletes a snapshot.
:param snapshot: snapshot reference (sqlalchemy Model)
"""
_debug_vol_info("DELETE_SNAP", snapshot)
volume = snapshot.volume
hostname = volume.host
location = volume.provider_location
ovs_disk = self._find_ovs_model_disk_by_location(location, hostname)
LOG.debug("DELETE_SNAP %s" % snapshot.id)
VDiskController.delete_snapshot(diskguid=ovs_disk.guid, snapshotid=str(snapshot.id))
LOG.debug("DELETE_SNAP OK")
def test_clone_from_template_happypath(self):
"""
Test clone from template - happy path
"""
StorageDriverModule.use_good_client()
vdisk_1_1, pmachine = self._prepare()
VDiskController.create_from_template(vdisk_1_1.guid, 'vmachine_2', 'vdisk_1_1-clone', pmachine.guid)
vdisks = VDiskList.get_vdisk_by_name('vdisk_1_1')
self.assertEqual(len(vdisks), 1, 'Vdisk not modeled')
clones = VDiskList.get_vdisk_by_name('vdisk_1_1-clone')
self.assertEqual(len(clones), 1, 'Clone not modeled')
def test_clone_from_template_happypath(self):
"""
Test clone from template - happy path
"""
StorageDriverModule.use_good_client()
vdisk_1_1, pmachine = self._prepare()
VDiskController.create_from_template(vdisk_1_1.guid, 'vmachine_2',
'vdisk_1_1-clone', pmachine.guid)
vdisks = VDiskList.get_vdisk_by_name('vdisk_1_1')
self.assertEqual(len(vdisks), 1, 'Vdisk not modeled')
clones = VDiskList.get_vdisk_by_name('vdisk_1_1-clone')
self.assertEqual(len(clones), 1, 'Clone not modeled')
def test_folder_renames(self):
"""
Validates whether folder renames are correctly processed
"""
structure = DalHelper.build_dal_structure({
'vpools': [1],
'storagerouters': [1],
'storagedrivers':
[(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]
} # (<id>, <storagedriver_id>)
)
storagedriver = structure['storagedrivers'][1]
vdisk1 = VDisk(
VDiskController.create_new('foo/one.raw', 1024**3,
storagedriver.guid))
vdisk2 = VDisk(
VDiskController.create_new('bar/two.raw', 1024**3,
storagedriver.guid))
vdisk3 = VDisk(
VDiskController.create_new('three.raw', 1024**3,
storagedriver.guid))
VDiskController.rename_from_voldrv(
old_path='/thr',
new_path='/test',
storagedriver_id=storagedriver.storagedriver_id)
vdisk1.discard()
vdisk2.discard()
vdisk3.discard()
self.assertEqual(vdisk1.devicename, '/foo/one.raw')
self.assertEqual(vdisk2.devicename, '/bar/two.raw')
self.assertEqual(vdisk3.devicename, '/three.raw')
VDiskController.rename_from_voldrv(
old_path='/foo',
new_path='/bar',
storagedriver_id=storagedriver.storagedriver_id)
vdisk1.discard()
vdisk2.discard()
vdisk3.discard()
self.assertEqual(vdisk1.devicename, '/bar/one.raw')
self.assertEqual(vdisk2.devicename, '/bar/two.raw')
self.assertEqual(vdisk3.devicename, '/three.raw')
VDiskController.rename_from_voldrv(
old_path='/bar',
new_path='/foo',
storagedriver_id=storagedriver.storagedriver_id)
vdisk1.discard()
vdisk2.discard()
vdisk3.discard()
self.assertEqual(vdisk1.devicename, '/foo/one.raw')
self.assertEqual(vdisk2.devicename, '/foo/two.raw')
self.assertEqual(vdisk3.devicename, '/three.raw')
def delete_snapshot(self, snapshot):
"""Deletes a snapshot.
:param snapshot: snapshot reference (sqlalchemy Model)
"""
_debug_vol_info('DELETE_SNAP', snapshot)
volume = snapshot.volume
hostname = volume.host
location = volume.provider_location
ovs_disk = self._find_ovs_model_disk_by_location(location, hostname)
LOG.debug('DELETE_SNAP %s' % snapshot.id)
VDiskController.delete_snapshot(diskguid=ovs_disk.guid,
snapshotid=str(snapshot.id))
LOG.debug('DELETE_SNAP OK')
def test_clean_devicename(self):
"""
Validates whether a devicename is properly cleaned
* Test several names and validate the returned devicename
"""
test = {
'Foo Bar':
'/Foo_Bar.raw',
'/Foo Bar .raw':
'/Foo_Bar_.raw',
'foo-bar.rawtest':
'/foo-bar.rawtest.raw',
'test///folder':
'/test/folder.raw',
'foobar-flat.vmdk':
'/foobar-flat.vmdk.raw',
'//test.raw':
'/test.raw',
'test/.raw':
'/test/.raw.raw',
'//d\'!@#%xfoo Bar/te_b --asdfS SA AS lolz///f.wrv.':
'/dxfoo_Bar/te_b_--asdfS_SA_AS_lolz/f.wrv..raw'
}
for raw, expected in test.iteritems():
result = VDiskController.clean_devicename(raw)
self.assertEqual(result, expected)
def delete_snapshot(vmachineguid, timestamp):
"""
Remove a snapshot from the vmachine
@param vmachineguid: Guid of the virtual machine
@param timestamp: timestamp of the snapshot
"""
vmachine = VMachine(vmachineguid)
vmachine_snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == str(timestamp)]
if len(vmachine_snapshots) != 1:
raise RuntimeError('Snapshot {0} does not belong to vmachine {1}'.format(timestamp, vmachine.name))
vmachine_snapshot = vmachine_snapshots[0]
VMachineController._logger.info('Deleting snapshot {0} from vmachine {1}'.format(timestamp, vmachine.name))
for diskguid, snapshotid in vmachine_snapshot['snapshots'].items():
VDiskController.delete_snapshot(diskguid, snapshotid)
VMachineController._logger.info('Deleted snapshot {0}'.format(timestamp))
vmachine.invalidate_dynamics(['snapshots'])
def _make_clones(vdisks_map, depth=clone_depth):
for level in range(depth):
previous_vd = list(vdisks_map.itervalues())[-1]
new_name = previous_vd.name + '_clone'
new_guid = VDiskController.clone(previous_vd.guid,
new_name).get('vdisk_guid')
vdisks_map[new_name] = VDisk(new_guid)
def delete_snapshot(vmachineguid, timestamp):
"""
Remove a snapshot from the vmachine
@param vmachineguid: Guid of the virtual machine
@param timestamp: timestamp of the snapshot
"""
vmachine = VMachine(vmachineguid)
vmachine_snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == str(timestamp)]
if len(vmachine_snapshots) != 1:
raise RuntimeError('Snapshot {0} does not belong to vmachine {1}'.format(timestamp, vmachine.name))
vmachine_snapshot = vmachine_snapshots[0]
VMachineController._logger.info('Deleting snapshot {0} from vmachine {1}'.format(timestamp, vmachine.name))
for diskguid, snapshotid in vmachine_snapshot['snapshots'].items():
VDiskController.delete_snapshot(diskguid, snapshotid)
VMachineController._logger.info('Deleted snapshot {0}'.format(timestamp))
vmachine.invalidate_dynamics(['snapshots'])
def test_delete(self):
"""
Test the delete of a vDisk
- Create 2 vDisks with identical names on 2 different vPools
- Delete 1st vDisk and verify other still remains on correct vPool
- Delete 2nd vDisk and verify no more volumes left
"""
structure = DalHelper.build_dal_structure({
'vpools': [1, 2],
'domains': [1],
'storagerouters': [1],
'storagedrivers':
[(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]
} # (<id>, <storagedriver_id>)
)
domains = structure['domains']
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[1].guid))
vdisk2 = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[2].guid))
vdisk_domain = VDiskDomain()
vdisk_domain.domain = domains[1]
vdisk_domain.vdisk = vdisk1
vdisk_domain.save()
# Delete vDisk1 and make some assertions
VDiskController.delete(vdisk_guid=vdisk1.guid)
with self.assertRaises(ObjectNotFoundException):
VDisk(vdisk1.guid)
self.assertEqual(
first=len(VDiskController.list_volumes()),
second=1,
msg='Expected to find only 1 volume in Storage Driver list_volumes'
)
self.assertIn(member=vdisk2,
container=VDiskList.get_vdisks(),
msg='vDisk2 should still be modeled')
# Delete vDisk2 and make some assertions
VDiskController.delete(vdisk_guid=vdisk2.guid)
with self.assertRaises(ObjectNotFoundException):
VDisk(vdisk2.guid)
self.assertEqual(
first=len(VDiskController.list_volumes()),
second=0,
msg=
'Expected to find no more volumes in Storage Driver list_volumes')
def test_create_snapshot(self):
"""
Test the create snapshot functionality
- Create a vDisk
- Attempt to create a snapshot providing incorrect parameters
- Create a snapshot and make some assertions
"""
structure = DalHelper.build_dal_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
with self.assertRaises(ValueError):
# noinspection PyTypeChecker
VDiskController.create_snapshot(vdisk_guid=vdisk1.guid,
metadata='')
now = int(time.time())
snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': now,
'label': 'label1',
'is_consistent': True,
'is_automatic': True,
'is_sticky': False})
self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find 1 snapshot')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 1, msg='Expected to find 1 snapshot ID')
snapshot = vdisk1.snapshots[0]
expected_keys = {'guid', 'timestamp', 'label', 'is_consistent', 'is_automatic', 'is_sticky', 'in_backend', 'stored'}
self.assertEqual(first=expected_keys,
second=set(snapshot.keys()),
msg='Set of expected keys differs from reality. Expected: {0} - Reality: {1}'.format(expected_keys, set(snapshot.keys())))
for key, value in {'guid': snapshot_id,
'label': 'label1',
'stored': 0,
'is_sticky': False,
'timestamp': now,
'in_backend': True,
'is_automatic': True,
'is_consistent': True}.iteritems():
self.assertEqual(first=value,
second=snapshot[key],
msg='Value for key "{0}" does not match reality. Expected: {1} - Reality: {2}'.format(key, value, snapshot[key]))
def test_create_snapshot(self):
"""
Test the create snapshot functionality
- Create a vDisk
- Attempt to create a snapshot providing incorrect parameters
- Create a snapshot and make some assertions
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
with self.assertRaises(ValueError):
# noinspection PyTypeChecker
VDiskController.create_snapshot(vdisk_guid=vdisk1.guid,
metadata='')
now = int(time.time())
snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': now,
'label': 'label1',
'is_consistent': True,
'is_automatic': True,
'is_sticky': False})
self.assertTrue(expr=len(vdisk1.snapshots) == 1,
msg='Expected to find 1 snapshot')
snapshot = vdisk1.snapshots[0]
expected_keys = {'guid', 'timestamp', 'label', 'is_consistent', 'is_automatic', 'is_sticky', 'in_backend', 'stored'}
self.assertEqual(first=expected_keys,
second=set(snapshot.keys()),
msg='Set of expected keys differs from reality. Expected: {0} - Reality: {1}'.format(expected_keys, set(snapshot.keys())))
for key, value in {'guid': snapshot_id,
'label': 'label1',
'stored': 0,
'is_sticky': False,
'timestamp': now,
'in_backend': True,
'is_automatic': True,
'is_consistent': True}.iteritems():
self.assertEqual(first=value,
second=snapshot[key],
msg='Value for key "{0}" does not match reality. Expected: {1} - Reality: {2}'.format(key, value, snapshot[key]))
def test_delete_snapshot(self):
"""
Test the delete snapshot functionality
- Create a vDisk and take a snapshot
- Attempt to delete a non-existing snapshot
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': int(time.time()),
'label': 'label1',
'is_consistent': True,
'is_automatic': True,
'is_sticky': False})
snapshot = vdisk1.snapshots[0]
self.assertTrue(expr=len(vdisk1.snapshots) == 1,
msg='Expected to find 1 snapshot')
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id='non-existing')
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=snapshot['guid'])
self.assertTrue(expr=len(vdisk1.snapshots) == 0,
msg='Expected to find no more snapshots')
def test_delete_snapshot(self):
"""
Test the delete snapshot functionality
- Create a vDisk and take a snapshot
- Attempt to delete a non-existing snapshot
"""
structure = DalHelper.build_dal_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata={'timestamp': int(time.time()),
'label': 'label1',
'is_consistent': True,
'is_automatic': True,
'is_sticky': False})
self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find 1 snapshot')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 1, msg='Expected to find 1 snapshot ID')
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id='non-existing')
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=vdisk1.snapshot_ids[0])
self.assertTrue(expr=len(vdisk1.snapshots) == 0, msg='Expected to find no more snapshots')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 0, msg='Expected to find no more snapshot IDs')
def test_set_as_template(self):
"""
Test the set as template functionality
- Create a vDisk
- Set it as template and make some assertions
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 4, storagedriver_guid=storagedrivers[1].guid))
metadata = {'is_consistent': True,
'is_automatic': True,
'is_sticky': False}
for x in range(5):
metadata['label'] = 'label{0}'.format(x)
metadata['timestamp'] = int(time.time())
VDiskController.create_snapshot(vdisk_guid=vdisk.guid, metadata=metadata)
self.assertTrue(expr=len(vdisk.snapshots) == 5, msg='Expected to find 5 snapshots')
# Set as template and validate the model
self.assertFalse(expr=vdisk.is_vtemplate, msg='Dynamic property "is_vtemplate" should be False')
VDiskController.set_as_template(vdisk.guid)
vdisk.invalidate_dynamics('snapshots')
self.assertTrue(expr=vdisk.is_vtemplate, msg='Dynamic property "is_vtemplate" should be True')
self.assertTrue(expr=len(vdisk.snapshots) == 1, msg='Expected to find only 1 snapshot after converting to template')
# Try again and verify job succeeds, previously we raised error when setting as template an additional time
VDiskController.set_as_template(vdisk.guid)
self.assertTrue(expr=vdisk.is_vtemplate, msg='Dynamic property "is_vtemplate" should still be True')
def rollback(machineguid, timestamp):
"""
Rolls back a VM based on a given disk snapshot timestamp
"""
vmachine = VMachine(machineguid)
if vmachine.hypervisor_status == 'RUNNING':
raise RuntimeError('vMachine {0} may not be running to set it as vTemplate'.format(
vmachine.name
))
snapshots = [snap for snap in vmachine.snapshots if snap['timestamp'] == timestamp]
if not snapshots:
raise ValueError('No vmachine snapshots found for timestamp {0}'.format(timestamp))
for disk in vmachine.vdisks:
VDiskController.rollback(diskguid=disk.guid,
timestamp=timestamp)
vmachine.invalidate_dynamics(['snapshots'])
def test_event_migrate_from_volumedriver(self):
"""
Test migrate from volumedriver event
"""
_ = self
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1, 2],
'storagedrivers': [(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]} # (<id>, <storagedriver_id>)
)
vpool = structure['vpools'][1]
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpool, storagerouters=storagerouters)
vdisk = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 4, storagedriver_guid=storagedrivers[1].guid))
vdisk.storagedriver_client.migrate(vdisk.volume_id, storagedrivers[2].storagedriver_id, False)
VDiskController.migrate_from_voldrv(volume_id=vdisk.volume_id, new_owner_id=storagedrivers[2].storagedriver_id)
self.assertEqual(vdisk.storagedriver_id, storagedrivers[2].storagedriver_id)
def create_volume_from_snapshot(self, volume, snapshot):
"""Creates a volume from a snapshot.
Called on "cinder create --snapshot-id ..."
:param snapshot: snapshot reference (sqlalchemy Model)
:param volume: volume reference (sqlalchemy Model)
Volume here is just a ModelObject, it doesn't exist physically,
it will be created by OVS.
Diskguid to be passed to the clone method is the ovs diskguid of the
parent of the snapshot with snapshot.id
OVS: Clone from arbitrary volume,
requires volumedriver 3.6 release > 15.08.2014
"""
_debug_vol_info('CLONE_VOL', volume)
_debug_vol_info('CLONE_SNAP', snapshot)
mountpoint = self._get_hostname_mountpoint(str(volume.host))
ovs_snap_disk = self._find_ovs_model_disk_by_snapshot_id(snapshot.id)
devicename = volume.display_name
if not devicename:
devicename = volume.name
pmachineguid = self._find_ovs_model_pmachine_guid_by_hostname(
str(volume.host))
LOG.info('[CLONE FROM SNAP] %s %s %s %s' %
(ovs_snap_disk.guid, snapshot.id, devicename, pmachineguid))
try:
disk_meta = VDiskController.clone(diskguid=ovs_snap_disk.guid,
snapshotid=snapshot.id,
devicename=devicename,
pmachineguid=pmachineguid,
machinename="",
machineguid=None)
volume['provider_location'] = '{}{}'.format(
mountpoint, disk_meta['backingdevice'])
LOG.debug('[CLONE FROM SNAP] Meta: %s' % str(disk_meta))
LOG.debug('[CLONE FROM SNAP] New volume %s' %
volume['provider_location'])
vdisk = VDisk(disk_meta['diskguid'])
vdisk.cinder_id = volume.id
vdisk.name = devicename
vdisk.save()
except Exception as ex:
LOG.error('CLONE FROM SNAP: Internal error %s ' % str(ex))
self.delete_volume(volume)
self.delete_snapshot(snapshot)
raise
return {
'provider_location': volume['provider_location'],
'display_name': volume['display_name']
}
def test_exception_handling(self):
"""
Test if the scheduled job can handle exceptions
"""
def raise_an_exception(*args, **kwargs):
raise RuntimeError('Emulated snapshot delete error')
structure = DalHelper.build_dal_structure({
'vpools': [1],
'vdisks': [
(1, 1, 1, 1), (2, 1, 1, 1)
], # (<id>, <storagedriver_id>, <vpool_id>, <mds_service_id>)
'mds_services': [(1, 1)],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)]
} # (<id>, <vpool_id>, <storagerouter_id>)
)
vdisk_1, vdisk_2 = structure['vdisks'].values()
storagedriver_1 = structure['storagedrivers'][1]
vdisks = [vdisk_1, vdisk_2]
for vdisk in vdisks:
[
dynamic for dynamic in vdisk._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
for i in xrange(0, 2):
metadata = {
'label':
str(i),
'is_consistent':
False,
'is_sticky':
False,
'timestamp':
str((int(time.time() -
datetime.timedelta(2).total_seconds() - i)))
}
snapshot_id = VDiskController.create_snapshot(
vdisk.guid, metadata)
if vdisk == vdisk_1:
StorageRouterClient.delete_snapshot_callbacks[
vdisk.volume_id] = {
snapshot_id: raise_an_exception
}
with self.assertRaises(RuntimeError):
GenericController.delete_snapshots_storagedriver(
storagedriver_guid=storagedriver_1.guid)
self.assertEqual(1, len(vdisk_2.snapshot_ids),
'One snapshot should be removed for vdisk 2')
self.assertEqual(2, len(vdisk_1.snapshot_ids),
'No snapshots should be removed for vdisk 1')
def copy_image_to_volume(self, context, volume, image_service, image_id):
"""Copy image to volume
Called on "nova volume-create --image-id ..."
or "cinder create --image-id"
Downloads image from glance server into local .raw
:param volume: volume reference (sqlalchemy Model)
"""
_debug_vol_info("CP_IMG_TO_VOL", volume)
LOG.info("CP_IMG_TO_VOL %s %s" % (image_service, image_id))
name = volume.display_name
if not name:
name = volume.name
volume.display_name = volume.name
# downloading from an existing image
destination_path = volume.provider_location
if destination_path:
try:
LOG.info('CP_IMG_TO_VOL Deleting existing empty raw file %s ' %
destination_path)
VDiskController.delete_volume(location=destination_path)
LOG.info('CP_IMG_TO_VOL Downloading image to %s' %
destination_path)
image_utils.fetch_to_raw(context,
image_service,
image_id,
destination_path,
'1M',
size=volume['size'])
LOG.info('CP_IMG_TO_VOL Resizing volume to size %s' %
volume['size'])
self.extend_volume(volume=volume, size_gb=volume['size'])
except Exception as ex:
LOG.error('CP_IMG_TO_VOL Internal error %s ' % unicode(ex))
self.delete_volume(volume)
raise
ovs_disk = self._find_ovs_model_disk_by_location(
volume.provider_location, str(volume.host))
ovs_disk.name = name
ovs_disk.save()
def test_extract_volumename(self):
"""
Validates whether a correct volumename is yielded from a given devicename
* Test several devicenames and validate the returned name
"""
test = {'/foo/Bar/something.raw': 'something',
'/Some.long.dotted.name.raw': 'Some.long.dotted.name',
'': '',
'f4qav@#$%sd.raw': 'f4qav@#$%sd'}
for devicename, expected in test.iteritems():
result = VDiskController.extract_volumename(devicename)
self.assertEqual(result, expected)
def snapshot_all_vdisks():
"""
Snapshots all vDisks
"""
ScheduledTaskController._logger.info('[SSA] started')
success = []
fail = []
for vdisk in VDiskList.get_vdisks():
try:
metadata = {'label': '',
'is_consistent': False,
'timestamp': str(int(time.time())),
'is_automatic': True,
'is_sticky': False}
VDiskController.create_snapshot(vdisk_guid=vdisk.guid,
metadata=metadata)
success.append(vdisk.guid)
except Exception:
ScheduledTaskController._logger.exception('Error taking snapshot for vDisk {0}'.format(vdisk.guid))
fail.append(vdisk.guid)
ScheduledTaskController._logger.info('[SSA] Snapshot has been taken for {0} vDisks, {1} failed.'.format(len(success), len(fail)))
def _snapshot_ids(self):
"""
Fetches the snapshot IDs for this vDisk
"""
if not self.volume_id or not self.vpool:
return []
from ovs.lib.vdisk import VDiskController
try:
return VDiskController.list_snapshot_ids(vdisk=self)
except:
return []
def clone(machineguid, timestamp, name):
"""
Clone a vmachine using the disk snapshot based on a snapshot timestamp
@param machineguid: guid of the machine to clone
@param timestamp: timestamp of the disk snapshots to use for the clone
@param name: name for the new machine
"""
machine = VMachine(machineguid)
disks = {}
for snapshot in machine.snapshots:
if snapshot['timestamp'] == timestamp:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems(
):
disks[diskguid] = snapshotguid
new_machine = VMachine()
new_machine.copy(machine)
new_machine.name = name
new_machine.pmachine = machine.pmachine
new_machine.save()
new_disk_guids = []
disks_by_order = sorted(machine.vdisks, key=lambda x: x.order)
for currentDisk in disks_by_order:
if machine.is_vtemplate and currentDisk.templatesnapshot:
snapshotid = currentDisk.templatesnapshot
else:
snapshotid = disks[currentDisk.guid]
prefix = '%s-clone' % currentDisk.name
result = VDiskController.clone(
diskguid=currentDisk.guid,
snapshotid=snapshotid,
devicename=prefix,
pmachineguid=new_machine.pmachine_guid,
machinename=new_machine.name,
machineguid=new_machine.guid)
new_disk_guids.append(result['diskguid'])
hv = Factory.get(machine.pmachine)
try:
result = hv.clone_vm(machine.hypervisor_id, name, disks, None,
True)
except:
VMachineController.delete(machineguid=new_machine.guid)
raise
new_machine.hypervisor_id = result
new_machine.save()
return new_machine.guid
def create_volume_from_snapshot(self, volume, snapshot):
"""Creates a volume from a snapshot.
Called on "cinder create --snapshot-id ..."
:param snapshot: snapshot reference (sqlalchemy Model)
:param volume: volume reference (sqlalchemy Model)
Volume here is just a ModelObject, it doesn't exist physically,
it will be created by OVS.
Diskguid to be passed to the clone method is the ovs diskguid of the
parent of the snapshot with snapshot.id
OVS: Clone from arbitrary volume,
requires volumedriver 3.6 release > 15.08.2014
"""
_debug_vol_info('CLONE_VOL', volume)
_debug_vol_info('CLONE_SNAP', snapshot)
mountpoint = self._get_hostname_mountpoint(str(volume.host))
ovs_snap_disk = self._find_ovs_model_disk_by_snapshot_id(snapshot.id)
devicename = volume.display_name
if not devicename:
devicename = volume.name
pmachineguid = self._find_ovs_model_pmachine_guid_by_hostname(
str(volume.host))
LOG.info('[CLONE FROM SNAP] %s %s %s %s'
% (ovs_snap_disk.guid, snapshot.id, devicename, pmachineguid))
try:
disk_meta = VDiskController.clone(diskguid = ovs_snap_disk.guid,
snapshotid = snapshot.id,
devicename = devicename,
pmachineguid = pmachineguid,
machinename = "",
machineguid=None)
volume['provider_location'] = '{}{}'.format(
mountpoint, disk_meta['backingdevice'])
LOG.debug('[CLONE FROM SNAP] Meta: %s' % str(disk_meta))
LOG.debug('[CLONE FROM SNAP] New volume %s'
% volume['provider_location'])
vdisk = VDisk(disk_meta['diskguid'])
vdisk.cinder_id = volume.id
vdisk.name = devicename
vdisk.save()
except Exception as ex:
LOG.error('CLONE FROM SNAP: Internal error %s ' % str(ex))
self.delete_volume(volume)
self.delete_snapshot(snapshot)
raise
return {'provider_location': volume['provider_location'],
'display_name': volume['display_name']}
def _check_volumedriver_remove(vpool_name, vdisk_name, present=True):
"""
Remove a vdisk from a vpool
:param vdisk_name: name of a vdisk (e.g. test.raw)
:type vdisk_name: str
:param vpool_name: name of a vpool
:type vpool_name: str
:param present: should the disk be present?
:type present: bool
:return: True if disk is not present anymore
:rtype: bool
"""
try:
vdisk = VDiskHelper.get_vdisk_by_name(vdisk_name=vdisk_name, vpool_name=vpool_name)
VDiskController.delete(vdisk.guid)
return True
except VDiskNotFoundError:
# not found, if it should be present, re-raise the exception
if present:
raise
else:
return True
def test_extract_volumename(self):
"""
Validates whether a correct volumename is yielded from a given devicename
* Test several devicenames and validate the returned name
"""
test = {
'/foo/Bar/something.raw': 'something',
'/Some.long.dotted.name.raw': 'Some.long.dotted.name',
'': '',
'f4qav@#$%sd.raw': 'f4qav@#$%sd'
}
for devicename, expected in test.iteritems():
result = VDiskController.extract_volumename(devicename)
self.assertEqual(result, expected)
def create_snapshot(self, snapshot):
"""Creates a snapshot.
Called on "nova image-create " or "cinder snapshot-create "
:param snapshot: snapshot reference (sqlalchemy Model)
"""
_debug_vol_info('CREATE_SNAP', snapshot)
volume = snapshot.volume
_debug_vol_info('CREATE_SNAP_VOL', volume)
hostname = volume.host
location = volume.provider_location
ovs_disk = self._find_ovs_model_disk_by_location(location, hostname)
metadata = {'label': "{0} (OpenStack)".format(snapshot.display_name),
'is_consistent': False,
'timestamp': time.time(),
'machineguid': ovs_disk.vmachine_guid,
'is_automatic': False}
LOG.debug('CREATE_SNAP %s %s' % (snapshot.display_name, str(metadata)))
VDiskController.create_snapshot(diskguid = ovs_disk.guid,
metadata = metadata,
snapshotid = str(snapshot.id))
LOG.debug('CREATE_SNAP OK')
def clone(machineguid, timestamp, name):
"""
Clone a vmachine using the disk snapshot based on a snapshot timestamp
@param machineguid: guid of the machine to clone
@param timestamp: timestamp of the disk snapshots to use for the clone
@param name: name for the new machine
"""
machine = VMachine(machineguid)
disks = {}
for snapshot in machine.snapshots:
if snapshot['timestamp'] == timestamp:
for diskguid, snapshotguid in snapshot['snapshots'].iteritems():
disks[diskguid] = snapshotguid
new_machine = VMachine()
new_machine.copy(machine)
new_machine.name = name
new_machine.pmachine = machine.pmachine
new_machine.save()
new_disk_guids = []
disks_by_order = sorted(machine.vdisks, key=lambda x: x.order)
for currentDisk in disks_by_order:
if machine.is_vtemplate and currentDisk.templatesnapshot:
snapshotid = currentDisk.templatesnapshot
else:
snapshotid = disks[currentDisk.guid]
prefix = '%s-clone' % currentDisk.name
result = VDiskController.clone(diskguid=currentDisk.guid,
snapshotid=snapshotid,
devicename=prefix,
pmachineguid=new_machine.pmachine_guid,
machinename=new_machine.name,
machineguid=new_machine.guid)
new_disk_guids.append(result['diskguid'])
hv = Factory.get(machine.pmachine)
try:
result = hv.clone_vm(machine.hypervisor_id, name, disks, None, True)
except:
VMachineController.delete(machineguid=new_machine.guid)
raise
new_machine.hypervisor_id = result
new_machine.save()
return new_machine.guid
def _check_volumedriver(vdisk_name, storagedriver_guid, logger, vdisk_size=VDISK_CHECK_SIZE):
"""
Checks if the volumedriver can create a new vdisk
:param vdisk_name: name of a vdisk (e.g. test.raw)
:type vdisk_name: str
:param storagedriver_guid: guid of a storagedriver
:type storagedriver_guid: str
:param vdisk_size: size of the volume in bytes (e.g. 10737418240 is 10GB in bytes)
:type vdisk_size: int
:param logger: logger instance
:type logger: ovs.extensions.healthcheck.result.HCResults
:return: True if succeeds
:rtype: bool
"""
try:
VDiskController.create_new(vdisk_name, vdisk_size, storagedriver_guid)
except FileExistsException:
# can be ignored until fixed in framework
# https://github.com/openvstorage/framework/issues/1247
return True
except Exception as ex:
logger.failure('Creation of the vdisk failed. Got {0}'.format(str(ex)))
return False
return True
def test_clean_devicename(self):
"""
Validates whether a devicename is properly cleaned
* Test several names and validate the returned devicename
"""
test = {'Foo Bar': '/Foo_Bar.raw',
'/Foo Bar .raw': '/Foo_Bar_.raw',
'foo-bar.rawtest': '/foo-bar.rawtest.raw',
'test///folder': '/test/folder.raw',
'foobar-flat.vmdk': '/foobar-flat.vmdk.raw',
'//test.raw': '/test.raw',
'test/.raw': '/test/.raw.raw',
'//d\'!@#%xfoo Bar/te_b --asdfS SA AS lolz///f.wrv.': '/dxfoo_Bar/te_b_--asdfS_SA_AS_lolz/f.wrv..raw'}
for raw, expected in test.iteritems():
result = VDiskController.clean_devicename(raw)
self.assertEqual(result, expected)
def test_delete(self):
"""
Test the delete of a vDisk
- Create 2 vDisks with identical names on 2 different vPools
- Delete 1st vDisk and verify other still remains on correct vPool
- Delete 2nd vDisk and verify no more volumes left
"""
structure = Helper.build_service_structure(
{'vpools': [1, 2],
'domains': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]} # (<id>, <storagedriver_id>)
)
domains = structure['domains']
storagedrivers = structure['storagedrivers']
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[2].guid))
vdisk_domain = VDiskDomain()
vdisk_domain.domain = domains[1]
vdisk_domain.vdisk = vdisk1
vdisk_domain.save()
# Delete vDisk1 and make some assertions
VDiskController.delete(vdisk_guid=vdisk1.guid)
with self.assertRaises(ObjectNotFoundException):
VDisk(vdisk1.guid)
self.assertEqual(first=len(VDiskController.list_volumes()),
second=1,
msg='Expected to find only 1 volume in Storage Driver list_volumes')
self.assertIn(member=vdisk2,
container=VDiskList.get_vdisks(),
msg='vDisk2 should still be modeled')
# Delete vDisk2 and make some assertions
VDiskController.delete(vdisk_guid=vdisk2.guid)
with self.assertRaises(ObjectNotFoundException):
VDisk(vdisk2.guid)
self.assertEqual(first=len(VDiskController.list_volumes()),
second=0,
msg='Expected to find no more volumes in Storage Driver list_volumes')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistents
every now an then. The delelete policy is exectued every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and an additional disk
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
debug = True
amount_of_days = 50
base = datetime.now().date()
day = timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = DeleteSnapshots._make_timestamp(base, day * d)
print ''
print 'Day cycle: {}: {}'.format(
d, datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')
)
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.deletescrubsnapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def create_cloned_volume(self, volume, src_vref):
"""Create a cloned volume from another volume.
Called on "cinder create --source-volid ... "
:param volume: volume reference - target volume (sqlalchemy Model)
:param src_vref: volume reference - source volume (sqlalchemy Model)
OVS: Create clone from template if the source is a template
Create volume from snapshot if the source is a volume
- create snapshot of source volume if it doesn't have snapshots
"""
_debug_vol_info('CREATE_CLONED_VOL', volume)
_debug_vol_info('CREATE_CLONED_VOL Source', src_vref)
mountpoint = self._get_hostname_mountpoint(str(volume.host))
name = volume.display_name
if not name:
name = volume.name
volume.display_name = volume.name
pmachineguid = self._find_ovs_model_pmachine_guid_by_hostname(
str(volume.host))
#source
source_ovs_disk = self._find_ovs_model_disk_by_location(
str(src_vref.provider_location), src_vref.host)
if source_ovs_disk.info['object_type'] == 'TEMPLATE':
LOG.info('[CREATE_FROM_TEMPLATE] VDisk %s is a template'
% source_ovs_disk.devicename)
# cloning from a template
LOG.debug('[CREATE FROM TEMPLATE] ovs_disk %s '
% (source_ovs_disk.devicename))
disk_meta = VDiskController.create_from_template(
diskguid = source_ovs_disk.guid,
machinename = "",
devicename = str(name),
pmachineguid = pmachineguid,
machineguid = None,
storagedriver_guid = None)
volume['provider_location'] = '{}{}'.format(
mountpoint, disk_meta['backingdevice'])
LOG.debug('[CREATE FROM TEMPLATE] New volume %s'
% volume['provider_location'])
vdisk = VDisk(disk_meta['diskguid'])
vdisk.cinder_id = volume.id
vdisk.name = name
LOG.debug('[CREATE FROM TEMPLATE] Updating meta %s %s'
% (volume.id, name))
vdisk.save()
else:
LOG.info('[THIN CLONE] VDisk %s is not a template'
% source_ovs_disk.devicename)
# We do not support yet full volume clone
# - requires "emancipate" functionality
# So for now we'll take a snapshot
# (or the latest snapshot existing) and clone from that snapshot
available_snapshots = [snapshot for snapshot in source_ovs_disk.snapshots
if 'in_backend' not in snapshot or snapshot['in_backend'] is True]
if len(available_snapshots) == 0:
metadata = {'label': "Cinder clone snapshot {0}".format(name),
'is_consistent': False,
'timestamp': time.time(),
'machineguid': source_ovs_disk.vmachine_guid,
'is_automatic': False}
LOG.debug('CREATE_SNAP %s %s' % (name, str(metadata)))
snapshotid = VDiskController.create_snapshot(
diskguid = source_ovs_disk.guid,
metadata = metadata,
snapshotid = None)
LOG.debug('CREATE_SNAP OK')
OVSVolumeDriver._wait_for_snapshot(source_ovs_disk, snapshotid)
else:
snapshotid = available_snapshots[-1]['guid']
LOG.debug('[CREATE CLONE FROM SNAP] %s ' % snapshotid)
disk_meta = VDiskController.clone(diskguid = source_ovs_disk.guid,
snapshotid = snapshotid,
devicename = str(name),
pmachineguid = pmachineguid,
machinename = "",
machineguid=None)
volume['provider_location'] = '{}{}'.format(
mountpoint, disk_meta['backingdevice'])
LOG.debug('[CLONE FROM SNAP] Meta: %s' % str(disk_meta))
LOG.debug('[CLONE FROM SNAP] New volume %s'
% volume['provider_location'])
vdisk = VDisk(disk_meta['diskguid'])
vdisk.cinder_id = volume.id
vdisk.name = name
vdisk.save()
return {'provider_location': volume['provider_location'],
'display_name': volume['display_name']}
