def check_dtl(result_handler):
"""
Checks the dtl for all vdisks on the local node
:param result_handler: logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:return: None
:rtype: NoneType
"""
# Fetch vdisks hosted on this machine
local_sr = System.get_my_storagerouter()
if len(local_sr.vdisks_guids) == 0:
return result_handler.skip('No VDisks present in cluster.')
for vdisk_guid in local_sr.vdisks_guids:
vdisk = VDisk(vdisk_guid)
vdisk.invalidate_dynamics(['dtl_status', 'info'])
if vdisk.dtl_status == 'ok_standalone' or vdisk.dtl_status == 'disabled':
result_handler.success('VDisk {0}s DTL is disabled'.format(vdisk.name), code=ErrorCodes.volume_dtl_standalone)
elif vdisk.dtl_status == 'ok_sync':
result_handler.success('VDisk {0}s DTL is enabled and running.'.format(vdisk.name), code=ErrorCodes.volume_dtl_ok)
elif vdisk.dtl_status == 'degraded':
result_handler.warning('VDisk {0}s DTL is degraded.'.format(vdisk.name), code=ErrorCodes.volume_dtl_degraded)
elif vdisk.dtl_status == 'checkup_required':
result_handler.warning('VDisk {0}s DTL should be configured.'.format(vdisk.name), code=ErrorCodes.volume_dtl_checkup_required)
elif vdisk.dtl_status == 'catch_up':
result_handler.warning('VDisk {0}s DTL is enabled but still syncing.'.format(vdisk.name), code=ErrorCodes.volume_dtl_catch_up)
else:
result_handler.warning('VDisk {0}s DTL has an unknown status: {1}.'.format(vdisk.name, vdisk.dtl_status), code=ErrorCodes.volume_dtl_unknown)
def test_set_as_template(self):
"""
Test the set as template functionality
- Create a vDisk
- Set it as template and make some assertions
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]} # (<id>, <storagedriver_id>)
)
storagedrivers = structure['storagedrivers']
vdisk = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 4, storagedriver_guid=storagedrivers[1].guid))
metadata = {'is_consistent': True,
'is_automatic': True,
'is_sticky': False}
for x in range(5):
metadata['label'] = 'label{0}'.format(x)
metadata['timestamp'] = int(time.time())
VDiskController.create_snapshot(vdisk_guid=vdisk.guid, metadata=metadata)
self.assertTrue(expr=len(vdisk.snapshots) == 5, msg='Expected to find 5 snapshots')
# Set as template and validate the model
self.assertFalse(expr=vdisk.is_vtemplate, msg='Dynamic property "is_vtemplate" should be False')
VDiskController.set_as_template(vdisk.guid)
vdisk.invalidate_dynamics('snapshots')
self.assertTrue(expr=vdisk.is_vtemplate, msg='Dynamic property "is_vtemplate" should be True')
self.assertTrue(expr=len(vdisk.snapshots) == 1, msg='Expected to find only 1 snapshot after converting to template')
# Try again and verify job succeeds, previously we raised error when setting as template an additional time
VDiskController.set_as_template(vdisk.guid)
self.assertTrue(expr=vdisk.is_vtemplate, msg='Dynamic property "is_vtemplate" should still be True')
def _execute_scrub_work(scrub_location, vdisk_guids):
def _verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics(['info'])
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
ScheduledTaskController._logger.info('Execute Scrub - Started')
ScheduledTaskController._logger.info('Execute Scrub - Scrub location - {0}'.format(scrub_location))
total = len(vdisk_guids)
skipped = 0
storagedrivers = {}
failures = []
for vdisk_guid in vdisk_guids:
vdisk = VDisk(vdisk_guid)
try:
# Load the vDisk's StorageDriver
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Started'.format(vdisk.guid, vdisk.name))
vdisk.invalidate_dynamics(['storagedriver_id'])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[vdisk.storagedriver_id] = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
configs = _verify_mds_config(current_vdisk=vdisk)
# Check MDS master is local. Trigger MDS handover if necessary
if configs[0].get('ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.debug('Execute Scrub - Virtual disk {0} - {1} - MDS master is not local, trigger handover'.format(vdisk.guid, vdisk.name))
MDSServiceController.ensure_safety(vdisk)
configs = _verify_mds_config(current_vdisk=vdisk)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
skipped += 1
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Skipping because master MDS still not local'.format(vdisk.guid, vdisk.name))
continue
with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client:
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Retrieve and apply scrub work'.format(vdisk.guid, vdisk.name))
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
scrubbing_result = locked_client.scrub(work_unit, scrub_location, log_sinks=[SCRUBBER_LOGFILE_LOCATION])
locked_client.apply_scrubbing_result(scrubbing_result)
if work_units:
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - Scrub successfully applied'.format(vdisk.guid, vdisk.name))
else:
ScheduledTaskController._logger.info('Execute Scrub - Virtual disk {0} - {1} - No scrubbing required'.format(vdisk.guid, vdisk.name))
except Exception as ex:
failures.append('Failed scrubbing work unit for volume {0} with guid {1}: {2}'.format(vdisk.name, vdisk.guid, ex))
failed = len(failures)
ScheduledTaskController._logger.info('Execute Scrub - Finished - Success: {0} - Failed: {1} - Skipped: {2}'.format((total - failed - skipped), failed, skipped))
if failed > 0:
raise Exception('\n - '.join(failures))
return vdisk_guids
def rollback(diskguid, timestamp):
"""
Rolls back a disk based on a given disk snapshot timestamp
"""
disk = VDisk(diskguid)
snapshots = [snap for snap in disk.snapshots if snap['timestamp'] == timestamp]
if not snapshots:
raise ValueError('No snapshot found for timestamp {}'.format(timestamp))
snapshotguid = snapshots[0]['guid']
disk.storagedriver_client.rollback_volume(str(disk.volume_id), snapshotguid)
disk.invalidate_dynamics(['snapshots'])
return True
def _execute_scrub_work(scrub_location, vdisk_guids):
def verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics(["info"])
vdisk_configs = current_vdisk.info["metadata_backend_config"]
if len(vdisk_configs) == 0:
raise RuntimeError("Could not load MDS configuration")
return vdisk_configs
logger.info("Scrub location: {0}".format(scrub_location))
total = len(vdisk_guids)
skipped = 0
storagedrivers = {}
failures = []
for vdisk_guid in vdisk_guids:
vdisk = VDisk(vdisk_guid)
try:
# Load the vDisk's StorageDriver
logger.info("Scrubbing virtual disk {0} with guid {1}".format(vdisk.name, vdisk.guid))
vdisk.invalidate_dynamics(["storagedriver_id"])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[vdisk.storagedriver_id] = StorageDriverList.get_by_storagedriver_id(
vdisk.storagedriver_id
)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
configs = verify_mds_config(current_vdisk=vdisk)
# Check MDS master is local. Trigger MDS handover if necessary
if configs[0].get("ip") != storagedriver.storagerouter.ip:
logger.debug("MDS for volume {0} is not local. Trigger handover".format(vdisk.volume_id))
MDSServiceController.ensure_safety(vdisk)
configs = verify_mds_config(current_vdisk=vdisk)
if configs[0].get("ip") != storagedriver.storagerouter.ip:
skipped += 1
logger.info(
"Skipping scrubbing work unit for volume {0}: MDS master is not local".format(
vdisk.volume_id
)
)
continue
with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client:
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
scrubbing_result = locked_client.scrub(work_unit, scrub_location)
locked_client.apply_scrubbing_result(scrubbing_result)
if work_units:
logger.info("Scrubbing successfully applied")
except Exception, ex:
failures.append(
"Failed scrubbing work unit for volume {0} with guid {1}: {2}".format(vdisk.name, vdisk.guid, ex)
)
def delete_snapshot(diskguid, snapshotid):
"""
Delete a disk snapshot
@param diskguid: guid of the disk
@param snapshotid: id of the snapshot
@todo: Check if new volumedriver storagedriver upon deletion
of a snapshot has built-in protection to block it from being deleted
if a clone was created from it.
"""
disk = VDisk(diskguid)
logger.info('Deleting snapshot {} from disk {}'.format(snapshotid, disk.name))
disk.storagedriver_client.delete_snapshot(str(disk.volume_id), str(snapshotid))
disk.invalidate_dynamics(['snapshots'])
def create_snapshot(diskguid, metadata, snapshotid=None):
"""
Create a disk snapshot
@param diskguid: guid of the disk
@param metadata: dict of metadata
"""
disk = VDisk(diskguid)
logger.info("Create snapshot for disk {}".format(disk.name))
if snapshotid is None:
snapshotid = str(uuid.uuid4())
metadata = pickle.dumps(metadata)
disk.storagedriver_client.create_snapshot(str(disk.volume_id), snapshot_id=snapshotid, metadata=metadata)
disk.invalidate_dynamics(["snapshots"])
return snapshotid
def rollback(diskguid, timestamp):
"""
Rolls back a disk based on a given disk snapshot timestamp
"""
disk = VDisk(diskguid)
snapshots = [
snap for snap in disk.snapshots if snap['timestamp'] == timestamp
]
if not snapshots:
raise ValueError(
'No snapshot found for timestamp {}'.format(timestamp))
snapshotguid = snapshots[0]['guid']
disk.storagedriver_client.rollback_volume(str(disk.volume_id),
snapshotguid)
disk.invalidate_dynamics(['snapshots'])
return True
def delete_snapshot(diskguid, snapshotid):
"""
Delete a disk snapshot
@param diskguid: guid of the disk
@param snapshotguid: guid of the snapshot
@todo: Check if new volumedriver storagedriver upon deletion
of a snapshot has built-in protection to block it from being deleted
if a clone was created from it.
"""
disk = VDisk(diskguid)
logger.info('Deleting snapshot {} from disk {}'.format(
snapshotid, disk.name))
disk.storagedriver_client.delete_snapshot(str(disk.volume_id),
str(snapshotid))
disk.invalidate_dynamics(['snapshots'])
def create_snapshot(diskguid, metadata, snapshotid=None):
"""
Create a disk snapshot
@param diskguid: guid of the disk
@param metadata: dict of metadata
"""
disk = VDisk(diskguid)
logger.info('Create snapshot for disk {}'.format(disk.name))
if snapshotid is None:
snapshotid = str(uuid.uuid4())
metadata = pickle.dumps(metadata)
disk.storagedriver_client.create_snapshot(str(disk.volume_id),
snapshot_id=snapshotid,
metadata=metadata)
disk.invalidate_dynamics(['snapshots'])
return snapshotid
def delete_snapshot(diskguid, snapshotid):
"""
Delete a disk snapshot
@param diskguid: guid of the disk
@param snapshotid: ID of the snapshot
@todo: Check if new volumedriver storagedriver upon deletion
of a snapshot has built-in protection to block it from being deleted
if a clone was created from it.
"""
disk = VDisk(diskguid)
if snapshotid not in [snap['guid'] for snap in disk.snapshots]:
raise RuntimeError('Snapshot {0} does not belong to disk {1}'.format(snapshotid, disk.name))
clones_of_snapshot = VDiskList.get_by_parentsnapshot(snapshotid)
if len(clones_of_snapshot) > 0:
raise RuntimeError('Snapshot {0} has {1} volumes cloned from it, cannot remove'.format(snapshotid, len(clones_of_snapshot)))
logger.info('Deleting snapshot {0} from disk {1}'.format(snapshotid, disk.name))
disk.storagedriver_client.delete_snapshot(str(disk.volume_id), str(snapshotid))
disk.invalidate_dynamics(['snapshots'])
def create_snapshot(diskguid, metadata, snapshotid=None):
"""
Create a disk snapshot
:param diskguid: guid of the disk
:param metadata: dict of metadata
:param snapshotid: ID of the snapshot
"""
if not isinstance(metadata, dict):
raise ValueError('Expected metadata as dict, got {0} instead'.format(type(metadata)))
disk = VDisk(diskguid)
logger.info('Create snapshot for disk {0}'.format(disk.name))
if snapshotid is None:
snapshotid = str(uuid.uuid4())
metadata = pickle.dumps(metadata)
disk.storagedriver_client.create_snapshot(str(disk.volume_id),
snapshot_id=snapshotid,
metadata=metadata)
disk.invalidate_dynamics(['snapshots'])
return snapshotid
def test_remove_snapshots(self):
"""
Validates whether the remove_snapshots call works as expected. Due to openvstorage/framework#1534
it needs to handle some backwards compatibiltiy.
"""
structure = DalHelper.build_dal_structure({
'vpools': [1],
'storagerouters': [1],
'storagedrivers':
[(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)]
} # (<id>, <storagedriver_id>)
)
storagedriver = structure['storagedrivers'][1]
vdisk = VDisk(
VDiskController.create_new(volume_name='vdisk_1',
volume_size=1024**4,
storagedriver_guid=storagedriver.guid))
snapshots = []
for i in xrange(10):
metadata = {
'label': 'label{0}'.format(i),
'timestamp': int(time.time()),
'is_sticky': False,
'in_backend': True,
'is_automatic': True,
'is_consistent': True
}
snapshots.append(
VDiskController.create_snapshot(vdisk_guid=vdisk.guid,
metadata=metadata))
vdisk.invalidate_dynamics(['snapshots', 'snapshot_ids'])
self.assertEqual(len(vdisk.snapshots), 10)
self.assertEqual(len(vdisk.snapshot_ids), 10)
snapshot_id = snapshots[0]
# Old format
results = VDiskController.delete_snapshots({vdisk.guid: snapshot_id})
expected = {vdisk.guid: [True, snapshot_id]}
self.assertDictEqual(results, expected)
self.assertEqual(len(vdisk.snapshots), 9)
self.assertEqual(len(vdisk.snapshot_ids), 9)
results = VDiskController.delete_snapshots({vdisk.guid: snapshot_id})
expected = {vdisk.guid: [False, results[vdisk.guid][1]]}
self.assertDictEqual(results, expected)
self.assertRegexpMatches(results[vdisk.guid][1],
'^Snapshot (.*?) does not belong to vDisk')
self.assertEqual(len(vdisk.snapshots), 9)
self.assertEqual(len(vdisk.snapshot_ids), 9)
results = VDiskController.delete_snapshots({'foo': snapshot_id})
expected = {'foo': [False, results['foo'][1]]}
self.assertDictEqual(results, expected)
self.assertRegexpMatches(results['foo'][1],
'VDisk with guid (.*?) could not be found')
# New format
snapshot_id1 = snapshots[1]
snapshot_id2 = snapshots[2]
results = VDiskController.delete_snapshots(
{vdisk.guid: [snapshot_id1, snapshot_id2]})
expected = {
vdisk.guid: {
'success': True,
'error': None,
'results': {
snapshot_id1: [True, snapshot_id1],
snapshot_id2: [True, snapshot_id2]
}
}
}
self.assertDictEqual(results, expected)
self.assertEqual(len(vdisk.snapshots), 7)
self.assertEqual(len(vdisk.snapshot_ids), 7)
snapshot_id2 = snapshots[3]
results = VDiskController.delete_snapshots(
{vdisk.guid: [snapshot_id1, snapshot_id2]})
expected = {
vdisk.guid: {
'success': False,
'error': results[vdisk.guid]['error'],
'results': {
snapshot_id1:
[False, results[vdisk.guid]['results'][snapshot_id1][1]],
snapshot_id2: [True, snapshot_id2]
}
}
}
self.assertDictEqual(results, expected)
self.assertEquals(results[vdisk.guid]['error'],
'One or more snapshots could not be removed')
self.assertRegexpMatches(
results[vdisk.guid]['results'][snapshot_id1][1],
'^Snapshot (.*?) does not belong to vDisk')
self.assertEqual(len(vdisk.snapshots), 6)
self.assertEqual(len(vdisk.snapshot_ids), 6)
results = VDiskController.delete_snapshots({'foo': [snapshot_id1]})
expected = {
'foo': {
'success': False,
'error': results['foo']['error'],
'results': {}
}
}
self.assertDictEqual(results, expected)
self.assertRegexpMatches(results['foo']['error'],
'VDisk with guid (.*?) could not be found')
snapshot_id = snapshots[4]
VDiskController.clone(vdisk.guid, 'clone', snapshot_id)
results = VDiskController.delete_snapshots({vdisk.guid: [snapshot_id]})
expected = {
vdisk.guid: {
'success': False,
'error': results[vdisk.guid]['error'],
'results': {
snapshot_id:
[False, results[vdisk.guid]['results'][snapshot_id][1]]
}
}
}
self.assertDictEqual(results, expected)
self.assertEquals(results[vdisk.guid]['error'],
'One or more snapshots could not be removed')
self.assertRegexpMatches(
results[vdisk.guid]['results'][snapshot_id][1],
'^Snapshot (.*?) has [0-9]+ volume(.?) cloned from it, cannot remove$'
)
class SafetyEnsurer(MDSShared):
"""
Class responsible to ensure the MDS Safety of a volume
"""
_logger = Logger('lib')
def __init__(self, vdisk_guid, excluded_storagerouter_guids=None):
"""
:param vdisk_guid: vDisk GUID to calculate a new safety for
:type vdisk_guid: str
:param excluded_storagerouter_guids: GUIDs of StorageRouters to leave out of calculation (Eg: When 1 is down or unavailable)
:type excluded_storagerouter_guids: list[str]
"""
if excluded_storagerouter_guids is None:
excluded_storagerouter_guids = []
self.vdisk = VDisk(vdisk_guid)
self.excluded_storagerouters = [
StorageRouter(sr_guid) for sr_guid in excluded_storagerouter_guids
]
self.sr_client_timeout = Configuration.get(
'ovs/vpools/{0}/mds_config|sr_client_connection_timeout'.format(
self.vdisk.vpool_guid),
default=300)
self.mds_client_timeout = Configuration.get(
'ovs/vpools/{0}/mds_config|mds_client_connection_timeout'.format(
self.vdisk.vpool_guid),
default=120)
self.tlogs, self.safety, self.max_load = self.get_mds_config()
# Filled in by functions
self.metadata_backend_config_start = {}
# Layout related
self.mds_layout = {
'primary': {
'used': [],
'loads': {},
'available': []
},
'secondary': {
'used': [],
'loads': {},
'available': []
}
}
self.services_load = {}
self.recommended_primary = None
self.recommended_secondary = None
self.master_service = None
self.slave_services = []
self.mds_client_cache = {}
def validate_vdisk(self):
"""
Validates if the vDisk is ready for ensuring the MDS safety
:raises SRCObjectNotFoundException: If the vDisk is no associated with a StorageRouter
:raises RuntimeError: if
- Current host is in the excluded storagerouters
- vDisk is in a different state than running
:return: None
:rtype: NoneType
"""
self.vdisk.invalidate_dynamics(['info', 'storagerouter_guid'])
if self.vdisk.storagerouter_guid is None:
raise SRCObjectNotFoundException(
'Cannot ensure MDS safety for vDisk {0} with guid {1} because vDisk is not attached to any StorageRouter'
.format(self.vdisk.name, self.vdisk.guid))
vdisk_storagerouter = StorageRouter(self.vdisk.storagerouter_guid)
if vdisk_storagerouter in self.excluded_storagerouters:
raise RuntimeError(
'Current host ({0}) of vDisk {1} is in the list of excluded StorageRouters'
.format(vdisk_storagerouter.ip, self.vdisk.guid))
if self.vdisk.info['live_status'] != VDisk.STATUSES.RUNNING:
raise RuntimeError(
'vDisk {0} is not {1}, cannot update MDS configuration'.format(
self.vdisk.guid, VDisk.STATUSES.RUNNING))
self.metadata_backend_config_start = self.vdisk.info[
'metadata_backend_config']
if self.vdisk.info['metadata_backend_config'] == {}:
raise RuntimeError(
'Configured MDS layout for vDisk {0} could not be retrieved}, cannot update MDS configuration'
.format(self.vdisk.guid))
def map_mds_services_by_socket(self):
"""
Maps the mds services related to the vpool by their socket
:return: A dict wth sockets as key, service as value
:rtype: Dict[str, ovs.dal.hybrids.j_mdsservice.MDSService
"""
return super(SafetyEnsurer,
self).map_mds_services_by_socket(self.vdisk)
def get_primary_and_secondary_storagerouters(self):
# type: () -> Tuple[List[StorageRouter], List[StorageRouter]]
"""
Retrieve the primary and secondary storagerouters for MDS deployment
:return: Both primary and secondary storagerouters
:rtype: Tuple[List[StorageRouter], List[StorageRouter]]
"""
# Create a pool of StorageRouters being a part of the primary and secondary domains of this StorageRouter
vdisk = self.vdisk
vdisk_storagerouter = StorageRouter(vdisk.storagerouter_guid)
primary_domains = [
junction.domain for junction in vdisk_storagerouter.domains
if junction.backup is False
]
secondary_domains = [
junction.domain for junction in vdisk_storagerouter.domains
if junction.backup is True
]
primary_storagerouters = set()
secondary_storagerouters = set()
for domain in primary_domains:
primary_storagerouters.update(
StorageRouterList.get_primary_storagerouters_for_domain(
domain))
for domain in secondary_domains:
secondary_storagerouters.update(
StorageRouterList.get_primary_storagerouters_for_domain(
domain))
# In case no domains have been configured
if len(primary_storagerouters) == 0:
primary_storagerouters = set(
StorageRouterList.get_storagerouters())
# Remove all excluded StorageRouters from primary StorageRouters
primary_storagerouters = primary_storagerouters.difference(
self.excluded_storagerouters)
# Remove all StorageRouters from secondary which are present in primary, all excluded
secondary_storagerouters = secondary_storagerouters.difference(
primary_storagerouters)
secondary_storagerouters = secondary_storagerouters.difference(
self.excluded_storagerouters)
# Make sure to only use the StorageRouters related to the current vDisk's vPool
related_storagerouters = [
sd.storagerouter for sd in vdisk.vpool.storagedrivers
if sd.storagerouter is not None
]
primary_storagerouters = list(
primary_storagerouters.intersection(related_storagerouters))
secondary_storagerouters = list(
secondary_storagerouters.intersection(related_storagerouters))
if vdisk_storagerouter not in primary_storagerouters:
raise RuntimeError(
'Host of vDisk {0} ({1}) should be part of the primary domains'
.format(vdisk.name, vdisk_storagerouter.name))
primary_storagerouters.sort(
key=lambda sr: ExtensionsToolbox.advanced_sort(element=sr.ip,
separator='.'))
secondary_storagerouters.sort(
key=lambda sr: ExtensionsToolbox.advanced_sort(element=sr.ip,
separator='.'))
for primary_storagerouter in primary_storagerouters:
self._logger.debug(
'vDisk {0} - Primary StorageRouter {1} with IP {2}'.format(
vdisk.guid, primary_storagerouter.name,
primary_storagerouter.ip))
for secondary_storagerouter in secondary_storagerouters:
self._logger.debug(
'vDisk {0} - Secondary StorageRouter {1} with IP {2}'.format(
vdisk.guid, secondary_storagerouter.name,
secondary_storagerouter.ip))
for excluded_storagerouter in self.excluded_storagerouters:
self._logger.debug(
'vDisk {0} - Excluded StorageRouter {1} with IP {2}'.format(
vdisk.guid, excluded_storagerouter.name,
excluded_storagerouter.ip))
return primary_storagerouters, secondary_storagerouters
def get_mds_config(self):
# type: () -> Tuple[int, int, int]
"""
Get the MDS Config parameters
:return: tlogs, safety and maxload
:rtype: int, int, int
"""
mds_config = Configuration.get('/ovs/vpools/{0}/mds_config'.format(
self.vdisk.vpool_guid))
return mds_config['mds_tlogs'], mds_config['mds_safety'], mds_config[
'mds_maxload']
def get_reconfiguration_reasons(self):
# type: () -> List[str]
"""
Check if reconfiguration is required
Fill in the state of all MDSes while checking the reasons
:return: All reconfiguration reasons
:rtype: List[str]
"""
services_by_socket = self.map_mds_services_by_socket()
primary_storagerouters, secondary_storagerouters = self.get_primary_and_secondary_storagerouters(
)
vdisk_storagerouter = StorageRouter(self.vdisk.storagerouter_guid)
current_service_ips = []
reconfigure_reasons = set()
for index, config in enumerate(self.metadata_backend_config_start
): # Ordered MASTER, SLAVE(S)
config_key = '{0}:{1}'.format(config['ip'], config['port'])
service = services_by_socket.get(config_key)
if service is None:
self._logger.critical(
'vDisk {0} - Storage leak detected. Namespace {1} for service {2} will never be deleted automatically because service does no longer exist in model'
.format(self.vdisk.guid, self.vdisk.volume_id, config_key))
reconfigure_reasons.add(
'{0} {1} cannot be used anymore'.format(
'Master' if index == 0 else 'Slave', config_key))
else:
if service.storagerouter.ip in current_service_ips:
reconfigure_reasons.add(
'Multiple MDS services on the same node with IP {0}'.
format(service.storagerouter.ip))
else:
current_service_ips.append(service.storagerouter.ip)
if index == 0:
self.master_service = service
else:
self.slave_services.append(service)
nodes = set()
for service in services_by_socket.itervalues():
importance = None
if service.storagerouter in primary_storagerouters:
importance = 'primary'
elif service.storagerouter in secondary_storagerouters:
importance = 'secondary'
# If MDS already in use, take current load, else take next load
loads = self.get_mds_load(mds_service=service.mds_service)
if service == self.master_service or service in self.slave_services: # Service is still in use
load = loads[0]
if importance is not None:
self.mds_layout[importance]['used'].append(service)
else:
reconfigure_reasons.add(
'Service {0} cannot be used anymore because StorageRouter with IP {1} is not part of the domains'
.format(service.name, service.storagerouter.ip))
else: # Service is not in use, but available
load = loads[1]
self.services_load[service] = load
if importance is not None:
nodes.add(service.storagerouter.ip)
self.mds_layout[importance]['available'].append(service)
if load <= self.max_load:
self._logger.debug(
'vDisk {0} - Service {1}:{2} has capacity - Load: {3}%'
.format(self.vdisk.guid, service.storagerouter.ip,
service.ports[0], load))
if load not in self.mds_layout[importance]['loads']:
self.mds_layout[importance]['loads'][load] = []
self.mds_layout[importance]['loads'][load].append(service)
else:
self._logger.debug(
'vDisk {0} - Service {1}:{2} is overloaded - Load: {3}%'
.format(self.vdisk.guid, service.storagerouter.ip,
service.ports[0], load))
if len(current_service_ips) > self.safety:
reconfigure_reasons.add(
'Too much safety - Current: {0} - Expected: {1}'.format(
len(current_service_ips), self.safety))
if len(current_service_ips) < self.safety and len(
current_service_ips) < len(nodes):
reconfigure_reasons.add(
'Not enough safety - Current: {0} - Expected: {1}'.format(
len(current_service_ips), self.safety))
if self.master_service is not None:
if self.services_load[self.master_service] > self.max_load:
reconfigure_reasons.add(
'Master overloaded - Current load: {0}% - Max load: {1}%'.
format(self.services_load[self.master_service],
self.max_load))
if self.master_service.storagerouter_guid != self.vdisk.storagerouter_guid:
reconfigure_reasons.add(
'Master {0}:{1} is not local - Current location: {0} - Expected location: {2}'
.format(self.master_service.storagerouter.ip,
self.master_service.ports[0],
vdisk_storagerouter.ip))
if self.master_service not in self.mds_layout['primary']['used']:
reconfigure_reasons.add(
'Master service {0}:{1} not in primary domain'.format(
self.master_service.storagerouter.ip,
self.master_service.ports[0]))
for slave_service in self.slave_services:
if self.services_load[slave_service] > self.max_load:
reconfigure_reasons.add(
'Slave {0}:{1} overloaded - Current load: {2}% - Max load: {3}%'
.format(slave_service.storagerouter.ip,
slave_service.ports[0],
self.services_load[slave_service], self.max_load))
# Check reconfigure required based upon domains
self.recommended_primary = int(math.ceil(
self.safety /
2.0)) if len(secondary_storagerouters) > 0 else self.safety
self.recommended_secondary = self.safety - self.recommended_primary
primary_services_used = len(self.mds_layout['primary']['used'])
primary_services_available = len(
self.mds_layout['primary']['available'])
if primary_services_used < self.recommended_primary and primary_services_used < primary_services_available:
reconfigure_reasons.add(
'Not enough services in use in primary domain - Current: {0} - Expected: {1}'
.format(primary_services_used, self.recommended_primary))
if primary_services_used > self.recommended_primary:
reconfigure_reasons.add(
'Too many services in use in primary domain - Current: {0} - Expected: {1}'
.format(primary_services_used, self.recommended_primary))
# More services can be used in secondary domain
secondary_services_used = len(self.mds_layout['secondary']['used'])
secondary_services_available = len(
self.mds_layout['secondary']['available'])
if secondary_services_used < self.recommended_secondary and secondary_services_used < secondary_services_available:
reconfigure_reasons.add(
'Not enough services in use in secondary domain - Current: {0} - Expected: {1}'
.format(secondary_services_used, self.recommended_secondary))
if secondary_services_used > self.recommended_secondary:
# Too many services in secondary domain
reconfigure_reasons.add(
'Too many services in use in secondary domain - Current: {0} - Expected: {1}'
.format(secondary_services_used, self.recommended_secondary))
# If secondary domain present, check order in which the slave services are configured
secondary = False
for slave_service in self.slave_services:
if secondary is True and slave_service in self.mds_layout[
'primary']['used']:
reconfigure_reasons.add(
'A slave in secondary domain has priority over a slave in primary domain'
)
break
if slave_service in self.mds_layout['secondary']['used']:
secondary = True
self._logger.info('vDisk {0} - Current configuration: {1}'.format(
self.vdisk.guid, self.metadata_backend_config_start))
return reconfigure_reasons
def create_new_master(self):
# type: () -> Tuple[List[Service], Service]
"""
Check and create a new MDS master if necessary
Master configured according to StorageDriver must be modelled
Master must be local
Master cannot be overloaded
Master must be in primary domain (if no domains available, this check is irrelevant because all StorageRouters will match)
:return: The newly created services and the previous master (if a master switch happened)
:rtype: Tuple[List[Service], Service]
"""
new_services = []
previous_master = None
log_start = 'vDisk {0}'.format(self.vdisk.guid)
if self.master_service is not None and self.master_service.storagerouter_guid == self.vdisk.storagerouter_guid and self.services_load[
self.
master_service] <= self.max_load and self.master_service in self.mds_layout[
'primary']['used']:
new_services.append(
self.master_service
) # Master is OK, so add as 1st element to new configuration. Reconfiguration is now based purely on slave misconfiguration
self._logger.debug(
'{0} - Master is still OK, re-calculating slaves'.format(
log_start))
else:
# Master is not OK --> try to find the best non-overloaded LOCAL MDS slave in the primary domain to make master
self._logger.debug(
'{0} - Master is not OK, re-calculating master'.format(
log_start))
current_load = 0
new_local_master_service = None
re_used_local_slave_service = None
for service in self.mds_layout['primary']['available']:
if service == self.master_service:
# Make sure the current master_service is not re-used as master for whatever reason
continue
next_load = self.services_load[
service] # This load indicates the load it would become if a vDisk would be moved to this Service
if next_load <= self.max_load and service.storagerouter_guid == self.vdisk.storagerouter_guid:
if current_load > next_load or (
re_used_local_slave_service is None
and new_local_master_service is None):
current_load = next_load # Load for least loaded service
new_local_master_service = service # If no local slave is found to re-use, this new_local_master_service is used
if service in self.slave_services:
self._logger.debug(
'{0} - Slave service {1}:{2} will be recycled'.
format(log_start, service.storagerouter.ip,
service.ports[0]))
re_used_local_slave_service = service # A slave service is found to re-use as new master
self.slave_services.remove(service)
if re_used_local_slave_service is None:
# There's no non-overloaded local slave found. Keep the current master (if available) and add a local MDS (if available) as slave.
# Next iteration, the newly added slave will be checked if it has caught up already
# If amount of tlogs to catchup is < configured amount of tlogs --> we wait for catchup, so master can be removed and slave can be promoted
if self.master_service is not None:
self._logger.debug(
'{0} - Keeping current master service'.format(
log_start))
new_services.append(self.master_service)
if new_local_master_service is not None:
self._logger.debug(
'{0} - Adding new slave service {1}:{2} to catch up'.
format(log_start,
new_local_master_service.storagerouter.ip,
new_local_master_service.ports[0]))
new_services.append(new_local_master_service)
else:
# A non-overloaded local slave was found
# We verify how many tlogs the slave is behind and do 1 of the following:
# 1. tlogs_behind_master < tlogs configured --> Invoke the catchup action and wait for it
# 2. tlogs_behind_master >= tlogs configured --> Add current master service as 1st in list, append non-overloaded local slave as 2nd in list and let StorageDriver do the catchup (next iteration we check again)
# noinspection PyTypeChecker
client = MetadataServerClient.load(
service=re_used_local_slave_service,
timeout=self.mds_client_timeout)
if client is None:
raise RuntimeError(
'Cannot establish a MDS client connection for service {0}:{1}'
.format(re_used_local_slave_service.storagerouter.ip,
re_used_local_slave_service.ports[0]))
self.mds_client_cache[re_used_local_slave_service] = client
try:
tlogs_behind_master = client.catch_up(
str(self.vdisk.volume_id), dry_run=True
) # Verify how much tlogs local slave Service is behind (No catchup action is invoked)
except RuntimeError as ex:
if 'Namespace does not exist' in ex.message:
client.create_namespace(str(self.vdisk.volume_id))
tlogs_behind_master = client.catch_up(str(
self.vdisk.volume_id),
dry_run=True)
else:
raise
self._logger.debug(
'{0} - Recycled slave is {1} tlogs behind'.format(
log_start, tlogs_behind_master))
if tlogs_behind_master < self.tlogs:
start = time.time()
try:
client.catch_up(str(self.vdisk.volume_id),
dry_run=False)
self._logger.debug('{0} - Catchup took {1}s'.format(
log_start, round(time.time() - start, 2)))
except Exception:
self._logger.exception(
'{0} - Catching up failed'.format(log_start))
raise # Catchup failed, so we don't know whether the new slave can be promoted to master yet
# It's up to date, so add it as a new master
new_services.append(re_used_local_slave_service)
if self.master_service is not None:
# The current master (if available) is now candidate to become one of the slaves (Determined below during slave calculation)
# The current master can potentially be on a different node, thus might become slave
self.slave_services.insert(0, self.master_service)
previous_master = self.master_service
else:
# It's not up to date, keep the previous master (if available) and give the local slave some more time to catch up
if self.master_service is not None:
new_services.append(self.master_service)
new_services.append(re_used_local_slave_service)
service_string = ', '.join([
"{{'ip': '{0}', 'port': {1}}}".format(service.storagerouter.ip,
service.ports[0])
for service in new_services
])
self._logger.debug(
'vDisk {0} - Configuration after MASTER calculation: [{1}]'.format(
self.vdisk.guid, service_string))
return new_services, previous_master
def create_new_slaves(self, new_services):
# type: (List[str]) -> Tuple[List[Service], List[Service]]
"""
Check and create a new MDS slaves if necessary
:param new_services: Services used for MDS master
:type new_services: List[str]
:return: New slave services for the primary domain, New slave services for the secondary domain
:rtype: Tuple[List[Service], List[Service]]
"""
def _add_suitable_nodes(local_importance,
local_safety,
services_to_recycle=None):
if services_to_recycle is None:
services_to_recycle = []
if local_importance == 'primary':
local_services = new_primary_services
else:
local_services = new_secondary_services
if len(new_node_ips) < local_safety:
for local_load in sorted(
self.mds_layout[local_importance]['loads']):
possible_services = self.mds_layout[local_importance][
'loads'][local_load]
if len(services_to_recycle) > 0:
possible_services = [
serv for serv in services_to_recycle
if serv in possible_services
] # Maintain order of services_to_recycle
for local_service in possible_services:
if len(new_node_ips) >= local_safety:
return
if local_service.storagerouter.ip not in new_node_ips:
if local_service.storagerouter not in storagerouter_cache:
try:
SSHClient(local_service.storagerouter)
storagerouter_cache[
local_service.storagerouter] = True
except UnableToConnectException:
storagerouter_cache[
local_service.storagerouter] = False
if storagerouter_cache[
local_service.storagerouter] is True:
local_services.append(local_service)
new_node_ips.add(
local_service.storagerouter.ip)
else:
self._logger.debug(
'vDisk {0} - Skipping StorageRouter with IP {1} as it is unreachable'
.format(self.vdisk.guid,
local_service.storagerouter.ip))
new_node_ips = {
new_services[0].storagerouter.ip
} if len(new_services) > 0 else set(
) # Currently we can only have the local IP in the list of new_services
storagerouter_cache = {}
new_primary_services = []
new_secondary_services = []
# Try to re-use slaves from primary domain until recommended_primary safety reached
_add_suitable_nodes(local_importance='primary',
local_safety=self.recommended_primary,
services_to_recycle=self.slave_services)
# Add new slaves until primary safety reached
_add_suitable_nodes(local_importance='primary',
local_safety=self.recommended_primary)
# Try to re-use slaves from secondary domain until safety reached
_add_suitable_nodes(local_importance='secondary',
local_safety=self.safety,
services_to_recycle=self.slave_services)
# Add new slaves until safety reached
_add_suitable_nodes(local_importance='secondary',
local_safety=self.safety)
# In case safety has not been reached yet, we try to add nodes from primary domain until safety has been reached
_add_suitable_nodes(local_importance='primary',
local_safety=self.safety)
# Extend the new services with the newly added primary and secondary services
return new_primary_services, new_secondary_services
def apply_reconfigurations(self, new_services, previous_master_service):
# type: (List[Service], Service) -> None
"""
Applies all calculated reconfigurations
- Deploys the services
- Notifies the Storagerouter
:param new_services: List of new services to be used in the reconfiguration (Master and slaves)
Note the order matters here! First the master, then slaves in primary domain, then slaves in secondary domain
:type new_services: List[Service]
:param previous_master_service: Previous master service incase the master should be switched around (None if no previous master)
:type previous_master_service: Service
:return: None
:rtype: NoneType
"""
# Verify an MDSClient can be created for all relevant services
services_to_check = new_services + self.slave_services
if self.master_service is not None:
services_to_check.append(self.master_service)
for service in services_to_check:
if service not in self.mds_client_cache:
client = MetadataServerClient.load(
service=service, timeout=self.mds_client_timeout)
if client is None:
raise RuntimeError(
'Cannot establish a MDS client connection for service {0}:{1}'
.format(service.storagerouter.ip, service.ports[0]))
self.mds_client_cache[service] = client
configs_all = []
new_namespace_services = []
configs_without_replaced_master = []
log_start = 'vDisk {0}'.format(self.vdisk.guid)
for service in new_services:
client = self.mds_client_cache[service]
try:
if str(self.vdisk.volume_id) not in client.list_namespaces():
client.create_namespace(
str(self.vdisk.volume_id)
) # StorageDriver does not throw error if already existing or does not create a duplicate namespace
new_namespace_services.append(service)
except Exception:
self._logger.exception(
'{0} - Creating new namespace {1} failed for Service {2}:{3}'
.format(log_start, self.vdisk.volume_id,
service.storagerouter.ip, service.ports[0]))
# Clean up newly created namespaces
for new_namespace_service in new_namespace_services:
client = self.mds_client_cache[new_namespace_service]
try:
self._logger.warning(
'{0}: Deleting newly created namespace {1} for service {2}:{3}'
.format(log_start, self.vdisk.volume_id,
new_namespace_service.storagerouter.ip,
new_namespace_service.ports[0]))
client.remove_namespace(str(self.vdisk.volume_id))
except RuntimeError:
pass # If somehow the namespace would not exist, we don't care.
raise # Currently nothing has been changed on StorageDriver level, so we can completely abort
# noinspection PyArgumentList
config = MDSNodeConfig(address=str(service.storagerouter.ip),
port=service.ports[0])
if previous_master_service != service: # This only occurs when a slave has caught up with master and old master gets replaced with new master
configs_without_replaced_master.append(config)
configs_all.append(config)
start = time.time()
update_failure = False
try:
self._logger.debug(
'{0} - Updating MDS configuration'.format(log_start))
if len(configs_without_replaced_master) != len(
configs_all
): # First update without previous master to avoid race conditions (required by voldrv)
self._logger.debug(
'{0} - Without previous master: {1}:{2}'.format(
log_start, previous_master_service.storagerouter.ip,
previous_master_service.ports[0]))
self.vdisk.storagedriver_client.update_metadata_backend_config(
volume_id=str(self.vdisk.volume_id),
metadata_backend_config=MDSMetaDataBackendConfig(
configs_without_replaced_master),
req_timeout_secs=self.sr_client_timeout)
self._logger.debug(
'{0} - Updating MDS configuration without previous master took {1}s'
.format(log_start,
time.time() - start))
self.vdisk.storagedriver_client.update_metadata_backend_config(
volume_id=str(self.vdisk.volume_id),
metadata_backend_config=MDSMetaDataBackendConfig(configs_all),
req_timeout_secs=self.sr_client_timeout)
# Verify the configuration - chosen by the framework - passed to the StorageDriver is effectively the correct configuration
self.vdisk.invalidate_dynamics('info')
self._logger.debug('{0} - Configuration after update: {1}'.format(
self.vdisk.guid, self.vdisk.info['metadata_backend_config']))
duration = time.time() - start
if duration > 5:
self._logger.critical(
'{0} - Updating MDS configuration took {1}s'.format(
log_start, duration))
except RuntimeError:
# @TODO: Timeout throws RuntimeError for now. Replace this once https://github.com/openvstorage/volumedriver/issues/349 is fixed
if time.time(
) - start >= self.sr_client_timeout: # Timeout reached, clean up must be done manually once server side finished
self._logger.critical(
'{0} - Updating MDS configuration timed out'.format(
log_start))
for service in [
svc for svc in services_to_check
if svc not in new_services
]:
self._logger.critical(
'{0} - Manual remove namespace action required for MDS {1}:{2} and namespace {3}'
.format(log_start, service.storagerouter.ip,
service.ports[0], self.vdisk.volume_id))
for service in new_services[1:]:
self._logger.critical(
'{0} - Manual set SLAVE role action required for MDS {1}:{2} and namespace {3}'
.format(log_start, service.storagerouter.ip,
service.ports[0], self.vdisk.volume_id))
self._logger.critical(
'{0} - Sync vDisk to reality action required'.format(
log_start))
else:
self._logger.exception(
'{0}: Failed to update the metadata backend configuration'.
format(log_start))
update_failure = True # No need to clean new namespaces if time out would have occurred
# Always raise
# * In case of a timeout, the manual actions are logged and user knows the ensure_safety has failed
# * In any other case, the newly created namespaces are deleted
raise
except Exception:
self._logger.exception(
'{0}: Failed to update the metadata backend configuration'.
format(log_start))
update_failure = True
raise
finally:
if update_failure is True:
# Remove newly created namespaces when updating would go wrong to avoid storage leaks
for new_namespace_service in new_namespace_services:
client = self.mds_client_cache[new_namespace_service]
try:
self._logger.warning(
'{0}: Deleting newly created namespace {1} for service {2}:{3}'
.format(log_start, self.vdisk.volume_id,
new_namespace_service.storagerouter.ip,
new_namespace_service.ports[0]))
client.remove_namespace(str(self.vdisk.volume_id))
except RuntimeError:
pass # If somehow the namespace would not exist, we don't care.
self._sync_vdisk_to_reality(self.vdisk)
for service in services_to_check:
if service not in new_services:
self._logger.debug(
'{0} - Deleting namespace for vDisk on service {1}:{2}'.
format(log_start, service.storagerouter.ip,
service.ports[0]))
client = self.mds_client_cache[service]
try:
client.remove_namespace(str(self.vdisk.volume_id))
except RuntimeError:
pass # If somehow the namespace would not exist, we don't care.
for service in new_services[1:]:
client = self.mds_client_cache[service]
try:
if client.get_role(nspace=str(self.vdisk.volume_id)
) != MetadataServerClient.MDS_ROLE.SLAVE:
self._logger.debug(
'{0} - Demoting service {1}:{2} to SLAVE'.format(
log_start, service.storagerouter.ip,
service.ports[0]))
start = time.time()
client.set_role(nspace=str(self.vdisk.volume_id),
role=MetadataServerClient.MDS_ROLE.SLAVE)
duration = time.time() - start
if duration > 5:
self._logger.critical(
'{0} - Demoting service {1}:{2} to SLAVE took {3}s'
.format(log_start, service.storagerouter.ip,
service.ports[0], duration))
except Exception:
self._logger.critical(
'{0} - Failed to demote service {1}:{2} to SLAVE'.format(
log_start, service.storagerouter.ip, service.ports[0]))
raise
def catchup_mds_slaves(self):
# type: () -> None
"""
Performs a catchup for MDS slaves if their tlogs behind reach a certain threshold
"""
def ensure_safety(self):
# type: () -> None
"""
Ensures (or tries to ensure) the safety of a given vDisk.
Assumptions:
* A local overloaded master is better than a non-local non-overloaded master
* Prefer master/slaves to be on different hosts, a subsequent slave on the same node doesn't add safety
* Don't actively overload services (e.g. configure an MDS as slave causing it to get overloaded)
* Too much safety is not wanted (it adds loads to nodes while not required)
* Order of slaves is:
* All slaves on StorageRouters in primary Domain of vDisk host
* All slaves on StorageRouters in secondary Domain of vDisk host
* Eg: Safety of 2 (1 master + 1 slave)
mds config = [local master in primary, slave in secondary]
* Eg: Safety of 3 (1 master + 2 slaves)
mds config = [local master in primary, slave in primary, slave in secondary]
* Eg: Safety of 4 (1 master + 3 slaves)
mds config = [local master in primary, slave in primary, slave in secondary, slave in secondary]
:raises RuntimeError: If host of vDisk is part of the excluded StorageRouters
If host of vDisk is not part of the StorageRouters in the primary domain
If catchup command fails for a slave
If MDS client cannot be created for any of the current or new MDS services
If updateMetadataBackendConfig would fail for whatever reason
:raises SRCObjectNotFoundException: If vDisk does not have a StorageRouter GUID
:return: None
:rtype: NoneType
"""
self._logger.info('vDisk {0} - Start checkup for vDisk {1}'.format(
self.vdisk.guid, self.vdisk.name))
self.validate_vdisk()
self._logger.debug(
'vDisk {0} - Safety: {1}, Max load: {2}%, Tlogs: {3}'.format(
self.vdisk.guid, self.safety, self.max_load, self.tlogs))
self.vdisk.reload_client('storagedriver')
self.vdisk.reload_client('objectregistry')
reconfigure_reasons = self.get_reconfiguration_reasons()
if not reconfigure_reasons:
self._logger.info('vDisk {0} - No reconfiguration required'.format(
self.vdisk.guid))
self._sync_vdisk_to_reality(self.vdisk)
return
self._logger.info(
'vDisk {0} - Reconfiguration required. Reasons:'.format(
self.vdisk.guid))
for reason in reconfigure_reasons:
self._logger.info('vDisk {0} - * {1}'.format(
self.vdisk.guid, reason))
new_services = []
new_master_services, previous_master = self.create_new_master()
new_services.extend(new_master_services)
# At this point we can have:
# Local master which is OK
# Local master + catching up new local master (because 1st is overloaded)
# Local master + catching up slave (because 1st was overloaded)
# Local slave which has caught up and been added as 1st in list of new_services
# Nothing at all --> Can only occur when the current master service (according to StorageDriver) has been deleted in the model and no other local MDS is available (Very unlikely scenario to occur, if possible at all)
# Now the slaves will be added according to the rules described in the docstring
# When local master + catching up service is present, this counts as safety of 1, because eventually the current master will be removed
new_primary_services, new_secondary_services = self.create_new_slaves(
new_services)
new_services.extend(new_primary_services)
new_services.extend(new_secondary_services)
service_string = ', '.join([
"{{'ip': '{0}', 'port': {1}}}".format(service.storagerouter.ip,
service.ports[0])
for service in new_services
])
self._logger.debug(
'vDisk {0} - Configuration after SLAVE calculation: [{1}]'.format(
self.vdisk.guid, service_string))
if new_services == [self.master_service] + self.slave_services and len(
new_services) == len(self.metadata_backend_config_start):
self._logger.info(
'vDisk {0} - Could not calculate a better MDS layout. Nothing to update'
.format(self.vdisk.guid))
self._sync_vdisk_to_reality(self.vdisk)
return
self.apply_reconfigurations(new_services, previous_master)
self._logger.info('vDisk {0}: Completed'.format(self.vdisk.guid))
def dtl_checkup(vpool_guid=None, vdisk_guid=None, storagerouters_to_exclude=None):
"""
Check DTL for all volumes
:param vpool_guid: vPool to check the DTL configuration of all its disks
:type vpool_guid: String
:param vdisk_guid: Virtual Disk to check its DTL configuration
:type vdisk_guid: String
:param storagerouters_to_exclude: Storage Routers to exclude from possible targets
:type storagerouters_to_exclude: List
:return: None
"""
if vpool_guid is not None and vdisk_guid is not None:
raise ValueError('vpool and vdisk are mutually exclusive')
if storagerouters_to_exclude is None:
storagerouters_to_exclude = []
from ovs.lib.vpool import VPoolController
logger.info('DTL checkup started')
required_params = {'dtl_mode': (str, StorageDriverClient.VPOOL_DTL_MODE_MAP.keys()),
'dtl_enabled': (bool, None)}
vdisk = VDisk(vdisk_guid) if vdisk_guid else None
vpool = VPool(vpool_guid) if vpool_guid else None
errors_found = False
root_client_map = {}
vpool_dtl_config_cache = {}
vdisks = VDiskList.get_vdisks() if vdisk is None and vpool is None else vpool.vdisks if vpool is not None else [vdisk]
for vdisk in vdisks:
logger.info(' Verifying vDisk {0} with guid {1}'.format(vdisk.name, vdisk.guid))
vdisk.invalidate_dynamics(['storagedriver_client', 'storagerouter_guid'])
if vdisk.storagedriver_client is None:
continue
vpool = vdisk.vpool
if vpool.guid not in vpool_dtl_config_cache:
vpool_config = VPoolController.get_configuration(vpool.guid) # Config on vPool is permanent for DTL settings
vpool_dtl_config_cache[vpool.guid] = vpool_config
Toolbox.verify_required_params(required_params, vpool_config)
volume_id = str(vdisk.volume_id)
vpool_config = vpool_dtl_config_cache[vpool.guid]
dtl_vpool_enabled = vpool_config['dtl_enabled']
try:
current_dtl_config = vdisk.storagedriver_client.get_dtl_config(volume_id)
current_dtl_config_mode = vdisk.storagedriver_client.get_dtl_config_mode(volume_id)
except RuntimeError as rte:
# Can occur when a volume has not been stolen yet from a dead node
logger.error('Retrieving DTL configuration from storage driver failed with error: {0}'.format(rte))
errors_found = True
continue
if dtl_vpool_enabled is False and (current_dtl_config is None or current_dtl_config.host == 'null'):
logger.info(' DTL is globally disabled for vPool {0} with guid {1}'.format(vpool.name, vpool.guid))
vdisk.storagedriver_client.set_manual_dtl_config(volume_id, None)
continue
elif current_dtl_config_mode == DTLConfigMode.MANUAL and (current_dtl_config is None or current_dtl_config.host == 'null'):
logger.info(' DTL is disabled for virtual disk {0} with guid {1}'.format(vdisk.name, vdisk.guid))
continue
storage_router = StorageRouter(vdisk.storagerouter_guid)
available_storagerouters = []
# 1. Check available storage routers in the backup failure domain
if storage_router.secondary_failure_domain is not None:
for storagerouter in storage_router.secondary_failure_domain.primary_storagerouters:
if vpool.guid not in storagerouter.vpools_guids:
continue
if storagerouter not in root_client_map:
try:
root_client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
logger.warning(' Storage Router with IP {0} of vDisk {1} is not reachable'.format(storagerouter.ip, vdisk.name))
continue
root_client_map[storagerouter] = root_client
else:
root_client = root_client_map[storagerouter]
if ServiceManager.get_service_status('dtl_{0}'.format(vpool.name), client=root_client) is True:
available_storagerouters.append(storagerouter)
# 2. Check available storage routers in the same failure domain as current storage router
if len(available_storagerouters) == 0:
for storagerouter in storage_router.primary_failure_domain.primary_storagerouters:
if vpool.guid not in storagerouter.vpools_guids or storagerouter == storage_router:
continue
if storagerouter not in root_client_map:
try:
root_client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
logger.warning(' Storage Router with IP {0} of vDisk {1} is not reachable'.format(storagerouter.ip, vdisk.name))
continue
root_client_map[storagerouter] = root_client
else:
root_client = root_client_map[storagerouter]
if ServiceManager.get_service_status('dtl_{0}'.format(vpool.name), client=root_client) is True:
available_storagerouters.append(storagerouter)
# Remove storage routers to exclude
for sr_guid in storagerouters_to_exclude:
sr_to_exclude = StorageRouter(sr_guid)
if sr_to_exclude in available_storagerouters:
available_storagerouters.remove(sr_to_exclude)
if len(available_storagerouters) == 0:
logger.info(' No Storage Routers could be found as valid DTL target')
vdisk.storagedriver_client.set_manual_dtl_config(volume_id, None)
continue
# Check whether reconfiguration is required
reconfigure_required = False
if current_dtl_config is None:
logger.info(' No DTL configuration found, but there are Storage Routers available')
reconfigure_required = True
elif current_dtl_config_mode == DTLConfigMode.AUTOMATIC:
logger.info(' DTL configuration set to AUTOMATIC, switching to manual')
reconfigure_required = True
else:
dtl_host = current_dtl_config.host
dtl_port = current_dtl_config.port
storage_drivers = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == dtl_host]
logger.info(' DTL host: {0}'.format(dtl_host or '-'))
logger.info(' DTL port: {0}'.format(dtl_port or '-'))
if dtl_host not in [sr.ip for sr in available_storagerouters]:
logger.info(' Host not in available Storage Routers')
reconfigure_required = True
elif dtl_port != storage_drivers[0].ports[2]:
logger.info(' Configured port does not match expected port ({0} vs {1})'.format(dtl_port, storage_drivers[0].ports[2]))
reconfigure_required = True
# Perform the reconfiguration
if reconfigure_required is True:
logger.info(' Reconfigure required')
index = random.randint(0, len(available_storagerouters) - 1)
dtl_target = available_storagerouters[index]
storage_drivers = [sd for sd in vpool.storagedrivers if sd.storagerouter == dtl_target]
if len(storage_drivers) == 0:
raise ValueError('Could not retrieve related storagedriver')
port = storage_drivers[0].ports[2]
vpool_dtl_mode = vpool_config.get('dtl_mode', StorageDriverClient.FRAMEWORK_DTL_ASYNC)
logger.info(' DTL config that will be set --> Host: {0}, Port: {1}, Mode: {2}'.format(dtl_target.ip, port, vpool_dtl_mode))
dtl_config = DTLConfig(str(dtl_target.ip), port, StorageDriverClient.VDISK_DTL_MODE_MAP[vpool_dtl_mode])
vdisk.storagedriver_client.set_manual_dtl_config(volume_id, dtl_config)
if errors_found is True:
logger.error('DTL checkup ended with errors')
raise Exception('DTL checkup failed with errors. Please check /var/log/ovs/lib.log for more information')
logger.info('DTL checkup ended')
def delete_snapshots_storagedriver(storagedriver_guid, timestamp=None, group_id=None):
"""
Delete snapshots per storagedriver & scrubbing policy
Implemented delete snapshot policy:
< 1d | 1d bucket | 1 | best of bucket | 1d
< 1w | 1d bucket | 6 | oldest of bucket | 7d = 1w
< 1m | 1w bucket | 3 | oldest of bucket | 4w = 1m
> 1m | delete
:param storagedriver_guid: Guid of the StorageDriver to remove snapshots on
:type storagedriver_guid: str
:param timestamp: Timestamp to determine whether snapshots should be kept or not, if none provided, current time will be used
:type timestamp: float
:param group_id: ID of the group task. Used to identify which snapshot deletes were called during the scheduled task
:type group_id: str
:return: None
"""
if group_id:
log_id = 'Group job {} - '.format(group_id)
else:
log_id = ''
def format_log(message):
return '{}{}'.format(log_id, message)
GenericController._logger.info(format_log('Delete snapshots started for StorageDriver {0}'.format(storagedriver_guid)))
storagedriver = StorageDriver(storagedriver_guid)
exceptions = []
day = timedelta(1)
week = day * 7
def make_timestamp(offset):
"""
Create an integer based timestamp
:param offset: Offset in days
:return: Timestamp
"""
return int(mktime((base - offset).timetuple()))
# Calculate bucket structure
if timestamp is None:
timestamp = time.time()
base = datetime.fromtimestamp(timestamp).date() - day
buckets = []
# Buckets first 7 days: [0-1[, [1-2[, [2-3[, [3-4[, [4-5[, [5-6[, [6-7[
for i in xrange(0, 7):
buckets.append({'start': make_timestamp(day * i),
'end': make_timestamp(day * (i + 1)),
'type': '1d',
'snapshots': []})
# Week buckets next 3 weeks: [7-14[, [14-21[, [21-28[
for i in xrange(1, 4):
buckets.append({'start': make_timestamp(week * i),
'end': make_timestamp(week * (i + 1)),
'type': '1w',
'snapshots': []})
buckets.append({'start': make_timestamp(week * 4),
'end': 0,
'type': 'rest',
'snapshots': []})
# Get a list of all snapshots that are used as parents for clones
parent_snapshots = set([vd.parentsnapshot for vd in VDiskList.get_with_parent_snaphots()])
# Place all snapshots in bucket_chains
bucket_chains = []
for vdisk_guid in storagedriver.vdisks_guids:
try:
vdisk = VDisk(vdisk_guid)
vdisk.invalidate_dynamics('being_scrubbed')
if vdisk.being_scrubbed:
continue
if vdisk.info['object_type'] in ['BASE']:
bucket_chain = copy.deepcopy(buckets)
for snapshot in vdisk.snapshots:
if snapshot.get('is_sticky') is True:
continue
if snapshot['guid'] in parent_snapshots:
GenericController._logger.info(format_log('Not deleting snapshot {0} because it has clones'.format(snapshot['guid'])))
continue
timestamp = int(snapshot['timestamp'])
for bucket in bucket_chain:
if bucket['start'] >= timestamp > bucket['end']:
bucket['snapshots'].append({'timestamp': timestamp,
'snapshot_id': snapshot['guid'],
'vdisk_guid': vdisk.guid,
'is_consistent': snapshot['is_consistent']})
bucket_chains.append(bucket_chain)
except Exception as ex:
exceptions.append(ex)
# Clean out the snapshot bucket_chains, we delete the snapshots we want to keep
# And we'll remove all snapshots that remain in the buckets
for bucket_chain in bucket_chains:
first = True
for bucket in bucket_chain:
if first is True:
best = None
for snapshot in bucket['snapshots']:
if best is None:
best = snapshot
# Consistent is better than inconsistent
elif snapshot['is_consistent'] and not best['is_consistent']:
best = snapshot
# Newer (larger timestamp) is better than older snapshots
elif snapshot['is_consistent'] == best['is_consistent'] and \
snapshot['timestamp'] > best['timestamp']:
best = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != best['timestamp']]
first = False
elif bucket['end'] > 0:
oldest = None
for snapshot in bucket['snapshots']:
if oldest is None:
oldest = snapshot
# Older (smaller timestamp) is the one we want to keep
elif snapshot['timestamp'] < oldest['timestamp']:
oldest = snapshot
bucket['snapshots'] = [s for s in bucket['snapshots'] if
s['timestamp'] != oldest['timestamp']]
# Delete obsolete snapshots
for bucket_chain in bucket_chains:
# Each bucket chain represents one vdisk's snapshots
try:
for bucket in bucket_chain:
for snapshot in bucket['snapshots']:
VDiskController.delete_snapshot(vdisk_guid=snapshot['vdisk_guid'],
snapshot_id=snapshot['snapshot_id'])
except RuntimeError as ex:
vdisk_guid = next((snapshot['vdisk_guid'] for bucket in bucket_chain for snapshot in bucket['snapshots']), '')
vdisk_id_log = ''
if vdisk_guid:
vdisk_id_log = ' for VDisk with guid {}'.format(vdisk_guid)
if SCRUB_VDISK_EXCEPTION_MESSAGE in ex.message:
GenericController._logger.warning(format_log('Being scrubbed exception occurred while deleting snapshots{}'.format(vdisk_id_log)))
else:
GenericController._logger.exception(format_log('Exception occurred while deleting snapshots{}'.format(vdisk_id_log)))
exceptions.append(ex)
if exceptions:
raise RuntimeError('Exceptions occurred while deleting snapshots: \n- {}'.format('\n- '.join((str(ex) for ex in exceptions))))
GenericController._logger.info(format_log('Delete snapshots finished for StorageDriver {0}'))
def test_clone(self):
"""
Test the clone functionality
- Create a vDisk with name 'clone1'
- Clone the vDisk and make some assertions
- Attempt to clone again using same name and same devicename
- Attempt to clone on Storage Router which is not linked to the vPool on which the original vDisk is hosted
- Attempt to clone on Storage Driver without MDS service
- Attempt to clone from snapshot which is not yet completely synced to backend
- Attempt to delete the snapshot from which a clone was made
- Clone the vDisk on another Storage Router
- Clone another vDisk with name 'clone1' linked to another vPool
"""
structure = Helper.build_service_structure(
{'vpools': [1, 2],
'storagerouters': [1, 2, 3],
'storagedrivers': [(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]} # (<id>, <storagedriver_id>)
)
vpools = structure['vpools']
mds_services = structure['mds_services']
service_type = structure['service_type']
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpools[1], storagerouters=storagerouters)
self._roll_out_dtl_services(vpool=vpools[2], storagerouters=storagerouters)
# Basic clone scenario
vdisk1 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
clone1_info = VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks')
clones = VDiskList.get_by_parentsnapshot(vdisk1.snapshots[0]['guid'])
self.assertTrue(expr=len(clones) == 1, msg='Expected to find 1 vDisk with parent snapshot')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 1, msg='Expected to find 1 child vDisk')
for expected_key in ['vdisk_guid', 'name', 'backingdevice']:
self.assertTrue(expr=expected_key in clone1_info, msg='Expected to find key "{0}" in clone_info'.format(expected_key))
self.assertTrue(expr=clones[0].guid == clone1_info['vdisk_guid'], msg='Guids do not match')
self.assertTrue(expr=clones[0].name == clone1_info['name'], msg='Names do not match')
self.assertTrue(expr=clones[0].devicename == clone1_info['backingdevice'], msg='Device names do not match')
# Attempt to clone again with same name
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 1')
# Attempt to clone again with a name which will have identical devicename
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1%')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 2')
# Attempt to clone on Storage Router on which vPool is not extended
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
storagerouter_guid=storagerouters[2].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 3')
# Attempt to clone on non-existing Storage Driver
storagedrivers[1].storagedriver_id = 'non-existing'
storagedrivers[1].save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 4')
storagedrivers[1].storagedriver_id = '1'
storagedrivers[1].save()
# Attempt to clone on Storage Driver without MDS service
mds_services[1].service.storagerouter = storagerouters[3]
mds_services[1].service.save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 5')
mds_services[1].service.storagerouter = storagerouters[1]
mds_services[1].service.save()
# Attempt to clone by providing snapshot_id not synced to backend
self.assertTrue(expr=len(vdisk1.snapshots) == 1, msg='Expected to find only 1 snapshot before cloning')
metadata = {'label': 'label1',
'timestamp': int(time.time()),
'is_sticky': False,
'in_backend': False,
'is_automatic': True,
'is_consistent': True}
snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid, metadata=metadata)
self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots')
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks after failed clone attempt 6')
# Update backend synced flag and retry
vdisk1.storagedriver_client._set_snapshot_in_backend(vdisk1.volume_id, snapshot_id, True)
vdisk1.invalidate_dynamics('snapshots')
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
vdisk1.invalidate_dynamics()
self.assertTrue(expr=len(vdisks) == 3, msg='Expected to find 3 vDisks')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 2, msg='Expected to find 2 child vDisks')
self.assertTrue(expr=len(vdisk1.snapshots) == 2, msg='Expected to find 2 snapshots after cloning from a specified snapshot')
# Attempt to delete the snapshot that has clones
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=snapshot_id)
# Clone on specific Storage Router
storagedriver = StorageDriver()
storagedriver.vpool = vpools[1]
storagedriver.storagerouter = storagerouters[2]
storagedriver.name = '3'
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[2].ip
storagedriver.storage_ip = '127.0.0.1'
storagedriver.storagedriver_id = '3'
storagedriver.ports = {'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4}
storagedriver.save()
s_id = '{0}-1'.format(storagedriver.storagerouter.name)
service = Service()
service.name = s_id
service.storagerouter = storagedriver.storagerouter
service.ports = [3]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storagedriver.vpool
mds_service.save()
clone3 = VDisk(VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone3',
storagerouter_guid=storagerouters[2].guid)['vdisk_guid'])
self.assertTrue(expr=clone3.storagerouter_guid == storagerouters[2].guid, msg='Incorrect Storage Router on which the clone is attached')
# Clone vDisk with existing name on another vPool
vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[2].guid))
clone_vdisk2 = VDisk(VDiskController.clone(vdisk_guid=vdisk2.guid,
name='clone1')['vdisk_guid'])
self.assertTrue(expr=clone_vdisk2.vpool == vpools[2], msg='Cloned vDisk with name "clone1" was created on incorrect vPool')
self.assertTrue(expr=len([vdisk for vdisk in VDiskList.get_vdisks() if vdisk.name == 'clone1']) == 2, msg='Expected to find 2 vDisks with name "clone1"')
# Attempt to clone without specifying snapshot and snapshot fails to sync to backend
StorageRouterClient.synced = False
vdisk2 = VDisk(VDiskController.create_new(volume_name='vdisk_2', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk2.guid,
name='clone4')
vdisk2.invalidate_dynamics()
self.assertTrue(expr=len(vdisk2.snapshots) == 0, msg='Expected to find 0 snapshots after clone failure')
self.assertTrue(expr=len(vdisk2.child_vdisks) == 0, msg='Expected to find 0 children after clone failure')
StorageRouterClient.synced = True
def _execute_scrub_work(scrub_location, vdisk_guids):
def verify_mds_config(current_vdisk):
"""
Retrieve the metadata backend configuration for vDisk
:param current_vdisk: vDisk to retrieve configuration for
:type current_vdisk: vDisk
:return: MDS configuration for vDisk
"""
current_vdisk.invalidate_dynamics(['info'])
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
logger.info('Execute Scrub - Started')
logger.info(
'Execute Scrub - Scrub location - {0}'.format(scrub_location))
total = len(vdisk_guids)
skipped = 0
storagedrivers = {}
failures = []
for vdisk_guid in vdisk_guids:
vdisk = VDisk(vdisk_guid)
try:
# Load the vDisk's StorageDriver
logger.info(
'Execute Scrub - Virtual disk {0} - {1} - Started'.format(
vdisk.guid, vdisk.name))
vdisk.invalidate_dynamics(['storagedriver_id'])
if vdisk.storagedriver_id not in storagedrivers:
storagedrivers[
vdisk.
storagedriver_id] = StorageDriverList.get_by_storagedriver_id(
vdisk.storagedriver_id)
storagedriver = storagedrivers[vdisk.storagedriver_id]
# Load the vDisk's MDS configuration
configs = verify_mds_config(current_vdisk=vdisk)
# Check MDS master is local. Trigger MDS handover if necessary
if configs[0].get('ip') != storagedriver.storagerouter.ip:
logger.debug(
'Execute Scrub - Virtual disk {0} - {1} - MDS master is not local, trigger handover'
.format(vdisk.guid, vdisk.name))
MDSServiceController.ensure_safety(vdisk)
configs = verify_mds_config(current_vdisk=vdisk)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
skipped += 1
logger.info(
'Execute Scrub - Virtual disk {0} - {1} - Skipping because master MDS still not local'
.format(vdisk.guid, vdisk.name))
continue
with vdisk.storagedriver_client.make_locked_client(
str(vdisk.volume_id)) as locked_client:
logger.info(
'Execute Scrub - Virtual disk {0} - {1} - Retrieve and apply scrub work'
.format(vdisk.guid, vdisk.name))
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
scrubbing_result = locked_client.scrub(
work_unit, scrub_location)
locked_client.apply_scrubbing_result(scrubbing_result)
if work_units:
logger.info(
'Execute Scrub - Virtual disk {0} - {1} - Scrub successfully applied'
.format(vdisk.guid, vdisk.name))
else:
logger.info(
'Execute Scrub - Virtual disk {0} - {1} - No scrubbing required'
.format(vdisk.guid, vdisk.name))
except Exception as ex:
failures.append(
'Failed scrubbing work unit for volume {0} with guid {1}: {2}'
.format(vdisk.name, vdisk.guid, ex))
failed = len(failures)
logger.info(
'Execute Scrub - Finished - Success: {0} - Failed: {1} - Skipped: {2}'
.format((total - failed - skipped), failed, skipped))
if failed > 0:
raise Exception('\n - '.join(failures))
return vdisk_guids
def _execute_move(self,
vdisk_guid,
destination_std,
force,
interactive,
minimum_potential=1):
"""
Perform a move
:param vdisk_guid: VDisk to move
:param destination_std: Destination to move to
:param force: Use force when moving
:param interactive: Prompt for user input before moving
:return: None
"""
_ = force
try:
vd = VDisk(vdisk_guid)
current_sr = StorageRouter(vd.storagerouter_guid).name
next_sr = destination_std.storagerouter.name
if vd.storagerouter_guid == destination_std.storagerouter_guid:
# Ownership changed in meantime
self.logger.info(
'No longer need to move VDisk {0} to {1}'.format(
vdisk_guid, destination_std.storagerouter.name))
return
rebalance_message = 'Rebalancing vPool by moving vDisk {0} from {1} to {2}'.format(
vdisk_guid, current_sr, next_sr)
if interactive:
retry = True
while retry:
proceed = raw_input('{0}. Continue? (press Enter)'.format(
rebalance_message))
if proceed == '': # Mock 'Enter' key
retry = False
try:
volume_potential = destination_std.vpool.storagedriver_client.volume_potential(
str(destination_std.storagedriver_id))
except:
self.logger.exception(
'Unable to retrieve volume potential. Aborting')
raise
if volume_potential > minimum_potential:
self.logger.info(rebalance_message)
try:
vd.storagedriver_client.migrate(str(vd.volume_id),
str(destination_std.name),
False)
except RuntimeError:
# When a RunTimeError occurs. Try restarting the volume locally for safety measures.
self.logger.warning(
'Encountered RunTimeError. Checking if vdisk({0}) is not running and restarting it.'
.format(vd.guid))
vd.invalidate_dynamics('info')
if vd.info['live_status'] != vd.STATUSES.RUNNING:
vd.storagedriver_client.restart_object(
str(vd.volume_id), False)
# Now check if the migration succeeded and if the volume is running on the correct storagedriver.
if vd.storagedriver_id == destination_std.name:
self.logger.info(
'Vdisk({0}) got restarted and runs on destination storagedriver. Previous error can be ignored.'
.format(vd.guid))
else:
self.logger.warning(
'Vdisk({0}) got restarted but doesn\'t run on destination storagedriver.'
.format(vd.guid))
else:
raise ValueError(
'Volume potential is lower than {0}. Not moving anymore!'.
format(minimum_potential))
except ObjectNotFoundException as ex:
self.logger.warning(
'Could not retrieve an object. Assuming it\'s a vDisk: {}'.
format(ex))
def test_create_from_template(self):
"""
Test the create from template functionality
- Create a vDisk and convert to vTemplate
- Attempt to create from template from a vDisk which is not a vTemplate
- Create from template basic scenario
- Attempt to create from template using same name
- Attempt to create from template using same devicename
- Attempt to create from template using Storage Router on which vPool is not extended
- Attempt to create from template using non-existing Storage Driver
- Attempt to create from template using Storage Driver which does not have an MDS service
- Create from template on another Storage Router
- Create from template without specifying a Storage Router
"""
structure = DalHelper.build_dal_structure({
'vpools': [1],
'storagerouters': [1, 2, 3],
'storagedrivers':
[(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]
} # (<id>, <storagedriver_id>)
)
vpool = structure['vpools'][1]
mds_services = structure['mds_services']
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpool, storagerouters=storagerouters)
template = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[1].guid))
vdisk_name = 'from_template_1'
VDiskController.set_as_template(vdisk_guid=template.guid)
self.assertTrue(expr=template.is_vtemplate,
msg='Dynamic property "is_vtemplate" should be True')
# Create from vDisk which is not a vTemplate
template.storagedriver_client._set_object_type(template.volume_id,
'BASE')
template.invalidate_dynamics(['info', 'is_vtemplate'])
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(
vdisk_guid=template.guid,
name=vdisk_name,
storagerouter_guid=storagerouters[1].guid)
# Create from template
template.storagedriver_client._set_object_type(template.volume_id,
'TEMPLATE')
template.invalidate_dynamics(['info', 'is_vtemplate'])
info = VDiskController.create_from_template(
vdisk_guid=template.guid,
name=vdisk_name,
storagerouter_guid=storagerouters[1].guid)
expected_keys = ['vdisk_guid', 'name', 'backingdevice']
self.assertEqual(
first=set(info.keys()),
second=set(expected_keys),
msg='Create from template returned not the expected keys')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks')
vdisk = [vdisk for vdisk in vdisks if vdisk.is_vtemplate is False][0]
self.assertTrue(
expr=vdisk.name == vdisk_name,
msg='vDisk name is incorrect. Expected: {0} - Actual: {1}'.
format(vdisk_name, vdisk.name))
self.assertTrue(expr=vdisk.parent_vdisk == template,
msg='The parent of the vDisk is incorrect')
# Attempt to create from template using same name
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(
vdisk_guid=template.guid,
name=vdisk_name,
storagerouter_guid=storagerouters[1].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2,
msg='Expected 2 vDisks after failed attempt 1')
# Attempt to create from template using same devicename
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(
vdisk_guid=template.guid,
name='^{0}$*'.format(vdisk_name),
storagerouter_guid=storagerouters[1].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2,
msg='Expected 2 vDisks after failed attempt 2')
# Attempt to create from template on Storage Router on which vPool is not extended
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(
vdisk_guid=template.guid,
name='from_template_2',
storagerouter_guid=storagerouters[3].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2,
msg='Expected 2 vDisks after failed attempt 3')
# Attempt to create on non-existing Storage Driver
storagedrivers[1].storagedriver_id = 'non-existing'
storagedrivers[1].save()
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid,
name='from_template_2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2,
msg='Expected 2 vDisks after failed attempt 4')
storagedrivers[1].storagedriver_id = '1'
storagedrivers[1].save()
# Attempt to create on Storage Driver without MDS service
mds_services[1].service.storagerouter = storagerouters[3]
mds_services[1].service.save()
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(
vdisk_guid=template.guid,
name='from_template_2',
storagerouter_guid=storagerouters[1].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2,
msg='Expected 2 vDisks after failed attempt 5')
mds_services[1].service.storagerouter = storagerouters[1]
mds_services[1].service.save()
# Create from template on another Storage Router
vdisk2 = VDisk(
VDiskController.create_from_template(
vdisk_guid=template.guid,
name='from_template_2',
storagerouter_guid=storagerouters[2].guid)['vdisk_guid'])
self.assertTrue(
expr=vdisk2.storagerouter_guid == storagerouters[2].guid,
msg='Expected vdisk2 to be hosted by Storage Router 2')
# Create from template without specifying Storage Router
vdisk3 = VDisk(
VDiskController.create_from_template(
vdisk_guid=template.guid,
name='from_template_3')['vdisk_guid'])
self.assertTrue(
expr=vdisk3.storagerouter_guid == template.storagerouter_guid,
msg='Expected vdisk3 to be hosted by Storage Router 1')
def test_clone(self):
"""
Test the clone functionality
- Create a vDisk with name 'clone1'
- Clone the vDisk and make some assertions
- Attempt to clone again using same name and same devicename
- Attempt to clone on Storage Router which is not linked to the vPool on which the original vDisk is hosted
- Attempt to clone on Storage Driver without MDS service
- Attempt to clone from snapshot which is not yet completely synced to backend
- Attempt to delete the snapshot from which a clone was made
- Clone the vDisk on another Storage Router
- Clone another vDisk with name 'clone1' linked to another vPool
"""
structure = DalHelper.build_dal_structure({
'vpools': [1, 2],
'storagerouters': [1, 2, 3],
'storagedrivers':
[(1, 1, 1), (2, 2, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]
} # (<id>, <storagedriver_id>)
)
vpools = structure['vpools']
mds_services = structure['mds_services']
service_type = structure['service_types']['MetadataServer']
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpools[1],
storagerouters=storagerouters)
self._roll_out_dtl_services(vpool=vpools[2],
storagerouters=storagerouters)
# Basic clone scenario
vdisk1 = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[1].guid))
clone1_info = VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected to find 2 vDisks')
clones = VDiskList.get_by_parentsnapshot(vdisk1.snapshot_ids[0])
self.assertTrue(expr=len(clones) == 1,
msg='Expected to find 1 vDisk with parent snapshot')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 1,
msg='Expected to find 1 child vDisk')
for expected_key in ['vdisk_guid', 'name', 'backingdevice']:
self.assertTrue(
expr=expected_key in clone1_info,
msg='Expected to find key "{0}" in clone_info'.format(
expected_key))
self.assertTrue(expr=clones[0].guid == clone1_info['vdisk_guid'],
msg='Guids do not match')
self.assertTrue(expr=clones[0].name == clone1_info['name'],
msg='Names do not match')
self.assertTrue(
expr=clones[0].devicename == clone1_info['backingdevice'],
msg='Device names do not match')
# Attempt to clone again with same name
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 1')
# Attempt to clone again with a name which will have identical devicename
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone1%')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 2')
# Attempt to clone on Storage Router on which vPool is not extended
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
storagerouter_guid=storagerouters[2].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 3')
# Attempt to clone on non-existing Storage Driver
storagedrivers[1].storagedriver_id = 'non-existing'
storagedrivers[1].save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 4')
storagedrivers[1].storagedriver_id = '1'
storagedrivers[1].save()
# Attempt to clone on Storage Driver without MDS service
mds_services[1].service.storagerouter = storagerouters[3]
mds_services[1].service.save()
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid, name='clone2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 5')
mds_services[1].service.storagerouter = storagerouters[1]
mds_services[1].service.save()
# Attempt to clone by providing snapshot_id not synced to backend
self.assertTrue(expr=len(vdisk1.snapshots) == 1,
msg='Expected to find only 1 snapshot before cloning')
self.assertTrue(
expr=len(vdisk1.snapshot_ids) == 1,
msg='Expected to find only 1 snapshot ID before cloning')
metadata = {
'label': 'label1',
'timestamp': int(time.time()),
'is_sticky': False,
'in_backend': False,
'is_automatic': True,
'is_consistent': True
}
snapshot_id = VDiskController.create_snapshot(vdisk_guid=vdisk1.guid,
metadata=metadata)
self.assertTrue(expr=len(vdisk1.snapshots) == 2,
msg='Expected to find 2 snapshots')
self.assertTrue(expr=len(vdisk1.snapshot_ids) == 2,
msg='Expected to find 2 snapshot IDs')
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
self.assertTrue(
expr=len(vdisks) == 2,
msg='Expected to find 2 vDisks after failed clone attempt 6')
# Update backend synced flag and retry
vdisk1.storagedriver_client._set_snapshot_in_backend(
vdisk1.volume_id, snapshot_id, True)
vdisk1.invalidate_dynamics(['snapshots', 'snapshot_ids'])
VDiskController.clone(vdisk_guid=vdisk1.guid,
name='clone2',
snapshot_id=snapshot_id)
vdisks = VDiskList.get_vdisks()
vdisk1.invalidate_dynamics()
self.assertTrue(expr=len(vdisks) == 3, msg='Expected to find 3 vDisks')
self.assertTrue(expr=len(vdisk1.child_vdisks) == 2,
msg='Expected to find 2 child vDisks')
self.assertTrue(
expr=len(vdisk1.snapshots) == 2,
msg=
'Expected to find 2 snapshots after cloning from a specified snapshot'
)
self.assertTrue(
expr=len(vdisk1.snapshot_ids) == 2,
msg=
'Expected to find 2 snapshot IDs after cloning from a specified snapshot'
)
# Attempt to delete the snapshot that has clones
with self.assertRaises(RuntimeError):
VDiskController.delete_snapshot(vdisk_guid=vdisk1.guid,
snapshot_id=snapshot_id)
# Clone on specific Storage Router
storagedriver = StorageDriver()
storagedriver.vpool = vpools[1]
storagedriver.storagerouter = storagerouters[2]
storagedriver.name = '3'
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[2].ip
storagedriver.storage_ip = '127.0.0.1'
storagedriver.storagedriver_id = '3'
storagedriver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storagedriver.save()
s_id = '{0}-1'.format(storagedriver.storagerouter.name)
service = Service()
service.name = s_id
service.storagerouter = storagedriver.storagerouter
service.ports = [3]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storagedriver.vpool
mds_service.save()
clone3 = VDisk(
VDiskController.clone(
vdisk_guid=vdisk1.guid,
name='clone3',
storagerouter_guid=storagerouters[2].guid)['vdisk_guid'])
self.assertTrue(
expr=clone3.storagerouter_guid == storagerouters[2].guid,
msg='Incorrect Storage Router on which the clone is attached')
# Clone vDisk with existing name on another vPool
vdisk2 = VDisk(
VDiskController.create_new(
volume_name='vdisk_1',
volume_size=1024**3,
storagedriver_guid=storagedrivers[2].guid))
clone_vdisk2 = VDisk(
VDiskController.clone(vdisk_guid=vdisk2.guid,
name='clone1')['vdisk_guid'])
self.assertTrue(
expr=clone_vdisk2.vpool == vpools[2],
msg='Cloned vDisk with name "clone1" was created on incorrect vPool'
)
self.assertTrue(expr=len([
vdisk for vdisk in VDiskList.get_vdisks() if vdisk.name == 'clone1'
]) == 2,
msg='Expected to find 2 vDisks with name "clone1"')
# Attempt to clone without specifying snapshot and snapshot fails to sync to backend
StorageRouterClient.synced = False
vdisk2 = VDisk(
VDiskController.create_new(
volume_name='vdisk_2',
volume_size=1024**3,
storagedriver_guid=storagedrivers[1].guid))
with self.assertRaises(RuntimeError):
VDiskController.clone(vdisk_guid=vdisk2.guid, name='clone4')
vdisk2.invalidate_dynamics()
self.assertTrue(expr=len(vdisk2.snapshots) == 0,
msg='Expected to find 0 snapshots after clone failure')
self.assertTrue(
expr=len(vdisk2.snapshot_ids) == 0,
msg='Expected to find 0 snapshot IDs after clone failure')
self.assertTrue(expr=len(vdisk2.child_vdisks) == 0,
msg='Expected to find 0 children after clone failure')
StorageRouterClient.synced = True
def clone(diskguid, snapshotid, devicename, pmachineguid, machinename=None, machineguid=None, detached=False):
"""
Clone a disk
"""
pmachine = PMachine(pmachineguid)
hypervisor = Factory.get(pmachine)
if machinename is None:
description = devicename
else:
description = '{0} {1}'.format(machinename, devicename)
properties_to_clone = ['description', 'size', 'type', 'retentionpolicyguid',
'snapshotpolicyguid', 'autobackup']
vdisk = VDisk(diskguid)
location = hypervisor.get_backing_disk_path(machinename, devicename)
if machineguid is not None and detached is True:
raise ValueError('A vMachine GUID was specified while detached is True')
if snapshotid is None:
# Create a new snapshot
timestamp = str(int(time.time()))
metadata = {'label': '',
'is_consistent': False,
'timestamp': timestamp,
'machineguid': machineguid,
'is_automatic': True}
VDiskController.create_snapshot(diskguid, metadata)
tries = 25 # About 5 minutes
while snapshotid is None and tries > 0:
tries -= 1
time.sleep(25 - tries)
vdisk.invalidate_dynamics(['snapshots'])
snapshots = [snapshot for snapshot in vdisk.snapshots
if snapshot['in_backend'] is True and snapshot['timestamp'] == timestamp]
if len(snapshots) == 1:
snapshotid = snapshots[0]['guid']
if snapshotid is None:
raise RuntimeError('Could not find created snapshot in time')
new_vdisk = VDisk()
new_vdisk.copy(vdisk, include=properties_to_clone)
new_vdisk.parent_vdisk = vdisk
new_vdisk.name = '{0}-clone'.format(vdisk.name)
new_vdisk.description = description
new_vdisk.devicename = hypervisor.clean_backing_disk_filename(location)
new_vdisk.parentsnapshot = snapshotid
if detached is False:
new_vdisk.vmachine = VMachine(machineguid) if machineguid else vdisk.vmachine
new_vdisk.vpool = vdisk.vpool
new_vdisk.save()
try:
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if storagedriver is None:
raise RuntimeError('Could not find StorageDriver with id {0}'.format(vdisk.storagedriver_id))
mds_service = MDSServiceController.get_preferred_mds(storagedriver.storagerouter, vdisk.vpool)
if mds_service is None:
raise RuntimeError('Could not find a MDS service')
logger.info('Clone snapshot {0} of disk {1} to location {2}'.format(snapshotid, vdisk.name, location))
volume_id = vdisk.storagedriver_client.create_clone(
target_path=location,
metadata_backend_config=MDSMetaDataBackendConfig([MDSNodeConfig(address=str(mds_service.service.storagerouter.ip),
port=mds_service.service.ports[0])]),
parent_volume_id=str(vdisk.volume_id),
parent_snapshot_id=str(snapshotid),
node_id=str(vdisk.storagedriver_id)
)
except Exception as ex:
logger.error('Caught exception during clone, trying to delete the volume. {0}'.format(ex))
new_vdisk.delete()
VDiskController.delete_volume(location)
raise
new_vdisk.volume_id = volume_id
new_vdisk.save()
try:
MDSServiceController.ensure_safety(new_vdisk)
except Exception as ex:
logger.error('Caught exception during "ensure_safety" {0}'.format(ex))
return {'diskguid': new_vdisk.guid,
'name': new_vdisk.name,
'backingdevice': location}
def test_create_from_template(self):
"""
Test the create from template functionality
- Create a vDisk and convert to vTemplate
- Attempt to create from template from a vDisk which is not a vTemplate
- Create from template basic scenario
- Attempt to create from template using same name
- Attempt to create from template using same devicename
- Attempt to create from template using Storage Router on which vPool is not extended
- Attempt to create from template using non-existing Storage Driver
- Attempt to create from template using Storage Driver which does not have an MDS service
- Create from template on another Storage Router
- Create from template without specifying a Storage Router
"""
structure = Helper.build_service_structure(
{'vpools': [1],
'storagerouters': [1, 2, 3],
'storagedrivers': [(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1), (2, 2)]} # (<id>, <storagedriver_id>)
)
vpool = structure['vpools'][1]
mds_services = structure['mds_services']
storagedrivers = structure['storagedrivers']
storagerouters = structure['storagerouters']
self._roll_out_dtl_services(vpool=vpool, storagerouters=storagerouters)
template = VDisk(VDiskController.create_new(volume_name='vdisk_1', volume_size=1024 ** 3, storagedriver_guid=storagedrivers[1].guid))
vdisk_name = 'from_template_1'
VDiskController.set_as_template(vdisk_guid=template.guid)
self.assertTrue(expr=template.is_vtemplate, msg='Dynamic property "is_vtemplate" should be True')
# Create from vDisk which is not a vTemplate
template.storagedriver_client._set_object_type(template.volume_id, 'BASE')
template.invalidate_dynamics(['info', 'is_vtemplate'])
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid, name=vdisk_name, storagerouter_guid=storagerouters[1].guid)
# Create from template
template.storagedriver_client._set_object_type(template.volume_id, 'TEMPLATE')
template.invalidate_dynamics(['info', 'is_vtemplate'])
info = VDiskController.create_from_template(vdisk_guid=template.guid, name=vdisk_name, storagerouter_guid=storagerouters[1].guid)
expected_keys = ['vdisk_guid', 'name', 'backingdevice']
self.assertEqual(first=set(info.keys()),
second=set(expected_keys),
msg='Create from template returned not the expected keys')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks')
vdisk = [vdisk for vdisk in vdisks if vdisk.is_vtemplate is False][0]
self.assertTrue(expr=vdisk.name == vdisk_name, msg='vDisk name is incorrect. Expected: {0} - Actual: {1}'.format(vdisk_name, vdisk.name))
self.assertTrue(expr=vdisk.parent_vdisk == template, msg='The parent of the vDisk is incorrect')
# Attempt to create from template using same name
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid, name=vdisk_name, storagerouter_guid=storagerouters[1].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks after failed attempt 1')
# Attempt to create from template using same devicename
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid, name='^{0}$*'.format(vdisk_name), storagerouter_guid=storagerouters[1].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks after failed attempt 2')
# Attempt to create from template on Storage Router on which vPool is not extended
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid, name='from_template_2', storagerouter_guid=storagerouters[3].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks after failed attempt 3')
# Attempt to create on non-existing Storage Driver
storagedrivers[1].storagedriver_id = 'non-existing'
storagedrivers[1].save()
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid, name='from_template_2')
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks after failed attempt 4')
storagedrivers[1].storagedriver_id = '1'
storagedrivers[1].save()
# Attempt to create on Storage Driver without MDS service
mds_services[1].service.storagerouter = storagerouters[3]
mds_services[1].service.save()
with self.assertRaises(RuntimeError):
VDiskController.create_from_template(vdisk_guid=template.guid, name='from_template_2', storagerouter_guid=storagerouters[1].guid)
vdisks = VDiskList.get_vdisks()
self.assertTrue(expr=len(vdisks) == 2, msg='Expected 2 vDisks after failed attempt 5')
mds_services[1].service.storagerouter = storagerouters[1]
mds_services[1].service.save()
# Create from template on another Storage Router
vdisk2 = VDisk(VDiskController.create_from_template(vdisk_guid=template.guid, name='from_template_2', storagerouter_guid=storagerouters[2].guid)['vdisk_guid'])
self.assertTrue(expr=vdisk2.storagerouter_guid == storagerouters[2].guid, msg='Expected vdisk2 to be hosted by Storage Router 2')
# Create from template without specifying Storage Router
vdisk3 = VDisk(VDiskController.create_from_template(vdisk_guid=template.guid, name='from_template_3')['vdisk_guid'])
self.assertTrue(expr=vdisk3.storagerouter_guid == template.storagerouter_guid, msg='Expected vdisk3 to be hosted by Storage Router 1')
def clone(diskguid, snapshotid, devicename, pmachineguid, machinename=None, machineguid=None, detached=False):
"""
Clone a disk
:param diskguid: Guid of the disk to clone
:param snapshotid: ID of the snapshot to clone from
:param devicename: Name of the device to use in clone's description
:param pmachineguid: Guid of the physical machine
:param machinename: Name of the machine the disk is attached to
:param machineguid: Guid of the machine
:param detached: Boolean indicating the disk is attached to a machine or not
"""
# 1. Validations
name_regex = "^[0-9a-zA-Z][-_a-zA-Z0-9]{1,48}[a-zA-Z0-9]$"
if not re.match(name_regex, devicename):
raise RuntimeError("Invalid name for virtual disk clone")
if VDiskList.get_vdisk_by_name(vdiskname=devicename) is not None:
raise RuntimeError("A virtual disk with this name already exists")
vdisk = VDisk(diskguid)
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if storagedriver is None:
raise RuntimeError('Could not find StorageDriver with ID {0}'.format(vdisk.storagedriver_id))
if machineguid is not None and detached is True:
raise ValueError('A vMachine GUID was specified while detached is True')
# 2. Create new snapshot if required
if snapshotid is None:
timestamp = str(int(time.time()))
metadata = {'label': '',
'is_consistent': False,
'timestamp': timestamp,
'machineguid': machineguid,
'is_automatic': True}
sd_snapshot_id = VDiskController.create_snapshot(diskguid, metadata)
tries = 25 # 5 minutes
while snapshotid is None and tries > 0:
time.sleep(25 - tries)
tries -= 1
vdisk.invalidate_dynamics(['snapshots'])
for snapshot in vdisk.snapshots:
if snapshot['guid'] != sd_snapshot_id:
continue
if snapshot['in_backend'] is True:
snapshotid = snapshot['guid']
if snapshotid is None:
try:
VDiskController.delete_snapshot(diskguid=diskguid,
snapshotid=sd_snapshot_id)
except:
pass
raise RuntimeError('Could not find created snapshot in time')
# 3. Model new cloned virtual disk
hypervisor = Factory.get(PMachine(pmachineguid))
location = hypervisor.get_disk_path(machinename, devicename)
new_vdisk = VDisk()
new_vdisk.copy(vdisk, include=['description', 'size', 'type', 'retentionpolicyguid', 'snapshotpolicyguid', 'autobackup'])
new_vdisk.parent_vdisk = vdisk
new_vdisk.name = devicename
new_vdisk.description = devicename if machinename is None else '{0} {1}'.format(machinename, devicename)
new_vdisk.devicename = hypervisor.clean_backing_disk_filename(location)
new_vdisk.parentsnapshot = snapshotid
if detached is False:
new_vdisk.vmachine = VMachine(machineguid) if machineguid else vdisk.vmachine
new_vdisk.vpool = vdisk.vpool
new_vdisk.save()
# 4. Configure Storage Driver
try:
mds_service = MDSServiceController.get_preferred_mds(storagedriver.storagerouter, vdisk.vpool)
if mds_service is None:
raise RuntimeError('Could not find a MDS service')
logger.info('Clone snapshot {0} of disk {1} to location {2}'.format(snapshotid, vdisk.name, location))
backend_config = MDSMetaDataBackendConfig([MDSNodeConfig(address=str(mds_service.service.storagerouter.ip),
port=mds_service.service.ports[0])])
volume_id = vdisk.storagedriver_client.create_clone(target_path=location,
metadata_backend_config=backend_config,
parent_volume_id=str(vdisk.volume_id),
parent_snapshot_id=str(snapshotid),
node_id=str(vdisk.storagedriver_id))
except Exception as ex:
logger.error('Caught exception during clone, trying to delete the volume. {0}'.format(ex))
try:
VDiskController.clean_bad_disk(new_vdisk.guid)
except Exception as ex2:
logger.exception('Exception during exception handling of "create_clone_from_template" : {0}'.format(str(ex2)))
raise
new_vdisk.volume_id = volume_id
new_vdisk.save()
# 5. Check MDS & DTL for new clone
try:
MDSServiceController.ensure_safety(new_vdisk)
except Exception as ex:
logger.error('Caught exception during "ensure_safety" {0}'.format(ex))
VDiskController.dtl_checkup.delay(vdisk_guid=new_vdisk.guid)
return {'diskguid': new_vdisk.guid,
'name': new_vdisk.name,
'backingdevice': location}
