class MetadataServerClient(object):
"""
Builds a MDSClient
"""
_logger = LogHandler.get('extensions', name='storagedriver')
storagerouterclient.Logger.setupLogging(
LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
MDS_ROLE = type('MDSRole', (), {
'MASTER': Role.Master,
'SLAVE': Role.Slave
})
def __init__(self):
"""
Dummy init method
"""
pass
@staticmethod
def load(service):
"""
Loads a MDSClient
:param service: Service for which the MDSClient needs to be loaded
"""
if service.storagerouter is None:
raise ValueError(
'MDS service {0} does not have a Storage Router linked to it'.
format(service.name))
key = service.guid
if key not in mdsclient_service_cache:
try:
# noinspection PyArgumentList
client = MDSClient(
MDSNodeConfig(address=str(service.storagerouter.ip),
port=service.ports[0]))
mdsclient_service_cache[key] = client
except RuntimeError as ex:
MetadataServerClient._logger.error(
'Error loading MDSClient on {0}: {1}'.format(
service.storagerouter.ip, ex))
return None
return mdsclient_service_cache[key]
def __init__(self, config_type, vpool_guid, storagedriver_id):
"""
Initializes the class
"""
if config_type != 'storagedriver':
raise RuntimeError('Invalid configuration type. Allowed: storagedriver')
storagerouterclient.Logger.setupLogging(LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
self._logger = LogHandler.get('extensions', name='storagedriver')
self.config_type = config_type
self.configuration = {}
self.key = '/ovs/vpools/{0}/hosts/{1}/config'.format(vpool_guid, storagedriver_id)
self.remote_path = Configuration.get_configuration_path(self.key).strip('/')
self.is_new = True
self.dirty_entries = []
def __init__(self, config_type, vpool_guid, storagedriver_id):
"""
Initializes the class
"""
if config_type != 'storagedriver':
raise RuntimeError(
'Invalid configuration type. Allowed: storagedriver')
storagerouterclient.Logger.setupLogging(
LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
self._logger = LogHandler.get('extensions', name='storagedriver')
self.config_type = config_type
self.configuration = {}
self.key = '/ovs/vpools/{0}/hosts/{1}/config'.format(
vpool_guid, storagedriver_id)
self.remote_path = Configuration.get_configuration_path(
self.key).strip('/')
self.is_new = True
self.dirty_entries = []
def __init__(self, config_type, vpool_guid, storagedriver_id):
"""
Initializes the class
"""
def make_configure(sct):
"""
section closure
:param sct: Section to create configure function for
"""
return lambda **kwargs: self._add(sct, **kwargs)
if config_type != 'storagedriver':
raise RuntimeError(
'Invalid configuration type. Allowed: storagedriver')
storagerouterclient.Logger.setupLogging(
LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
self._logger = LogHandler.get('extensions', name='storagedriver')
self.config_type = config_type
self.configuration = {}
self.path = '/ovs/vpools/{0}/hosts/{1}/config/{{0}}'.format(
vpool_guid, storagedriver_id)
self.remote_path = 'etcd://127.0.0.1:2379{0}'.format(
self.path.format('')).strip('/')
self.is_new = True
self.dirty_entries = []
self.params = copy.deepcopy(StorageDriverConfiguration.parameters
) # Never use parameters directly
# Fix some manual "I know what I'm doing" overrides
backend_connection_manager = 'backend_connection_manager'
self.params[self.config_type][backend_connection_manager][
'optional'].append('s3_connection_strict_consistency')
# Generate configure_* methods
for section in self.params[self.config_type]:
setattr(self, 'configure_{0}'.format(section),
make_configure(section))
class StorageDriverClient(object):
"""
Client to access storagedriver client
"""
storagerouterclient.Logger.setupLogging(
LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
VOLDRV_DTL_SYNC = 'Synchronous'
VOLDRV_DTL_ASYNC = 'Asynchronous'
VOLDRV_DTL_MANUAL_MODE = 'Manual'
VOLDRV_DTL_AUTOMATIC_MODE = 'Automatic'
VOLDRV_DTL_TRANSPORT_TCP = 'TCP'
VOLDRV_DTL_TRANSPORT_RSOCKET = 'RSocket'
FRAMEWORK_DTL_SYNC = 'sync'
FRAMEWORK_DTL_ASYNC = 'a_sync'
FRAMEWORK_DTL_NO_SYNC = 'no_sync'
FRAMEWORK_DTL_TRANSPORT_TCP = 'tcp'
FRAMEWORK_DTL_TRANSPORT_RSOCKET = 'rdma'
VDISK_DTL_MODE_MAP = {
FRAMEWORK_DTL_SYNC: DTLMode.SYNCHRONOUS,
FRAMEWORK_DTL_ASYNC: DTLMode.ASYNCHRONOUS,
FRAMEWORK_DTL_NO_SYNC: None
}
VPOOL_DTL_MODE_MAP = {
FRAMEWORK_DTL_SYNC: VOLDRV_DTL_SYNC,
FRAMEWORK_DTL_ASYNC: VOLDRV_DTL_ASYNC,
FRAMEWORK_DTL_NO_SYNC: None
}
VPOOL_DTL_TRANSPORT_MAP = {
FRAMEWORK_DTL_TRANSPORT_TCP: VOLDRV_DTL_TRANSPORT_TCP,
FRAMEWORK_DTL_TRANSPORT_RSOCKET: VOLDRV_DTL_TRANSPORT_RSOCKET
}
REVERSE_DTL_MODE_MAP = {
VOLDRV_DTL_SYNC: FRAMEWORK_DTL_SYNC,
VOLDRV_DTL_ASYNC: FRAMEWORK_DTL_ASYNC,
DTLMode.SYNCHRONOUS: FRAMEWORK_DTL_SYNC,
DTLMode.ASYNCHRONOUS: FRAMEWORK_DTL_ASYNC
}
REVERSE_DTL_TRANSPORT_MAP = {
VOLDRV_DTL_TRANSPORT_TCP: FRAMEWORK_DTL_TRANSPORT_TCP,
VOLDRV_DTL_TRANSPORT_RSOCKET: FRAMEWORK_DTL_TRANSPORT_RSOCKET
}
CLUSTER_SIZES = [4, 8, 16, 32, 64]
TLOG_MULTIPLIER_MAP = {4: 16, 8: 8, 16: 4, 32: 2, 64: 1, 128: 1}
DTL_STATUS = {
'': 0,
'ok_sync': 10,
'ok_standalone': 20,
'catch_up': 30,
'checkup_required': 30,
'degraded': 40
}
EMPTY_STATISTICS = staticmethod(lambda: Statistics())
EMPTY_INFO = staticmethod(lambda: VolumeInfo())
STAT_SUMS = {
'operations': ['write_operations', 'read_operations'],
'cache_hits': ['sco_cache_hits', 'cluster_cache_hits'],
'cache_misses': ['sco_cache_misses'],
'4k_operations': ['4k_read_operations', '4k_write_operations'],
'4k_unaligned_operations':
['4k_unaligned_read_operations', '4k_unaligned_write_operations'],
'data_transferred': ['data_written', 'data_read']
}
def __init__(self):
"""
Dummy init method
"""
pass
@staticmethod
def load(vpool, excluded_storagedrivers=None):
"""
Initializes the wrapper for a given vpool
:param vpool: vPool for which the StorageRouterClient needs to be loaded
:type vpool: vPool
:param excluded_storagedrivers: A list of storagedrivers that cannot be used as a client
:type excluded_storagedrivers: list or None
"""
if excluded_storagedrivers is None:
excluded_storagedrivers = []
key = vpool.identifier
if key not in client_vpool_cache:
cluster_contacts = []
for storagedriver in vpool.storagedrivers[:3]:
if storagedriver not in excluded_storagedrivers:
cluster_contacts.append(
ClusterContact(str(storagedriver.cluster_ip),
storagedriver.ports['xmlrpc']))
client = StorageRouterClient(str(vpool.guid), cluster_contacts)
client_vpool_cache[key] = client
return client_vpool_cache[key]
class MDSServiceController(object):
"""
Contains all BLL related to MDSServices
"""
_logger = LogHandler.get('lib', name='mds')
storagerouterclient.Logger.setupLogging(
LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
@staticmethod
def prepare_mds_service(storagerouter, vpool, fresh_only, reload_config):
"""
Prepares an MDS service:
* Creates the required configuration
* Sets up the service files
Assumes the StorageRouter and VPool are already configured with a StorageDriver and that all model-wise
configuration regarding both is completed.
:param storagerouter: Storagerouter on which MDS service will be created
:type storagerouter: StorageRouter
:param vpool: The vPool for which the MDS service will be created
:type vpool: VPool
:param fresh_only: If True and no current mds services exist for this vpool on this storagerouter, a new 1 will be created
:type fresh_only: bool
:param reload_config: If True, the volumedriver's updated configuration will be reloaded
:type reload_config: bool
:return: Newly created service
:rtype: MDSService
"""
# Fetch service sequence number based on MDS services for current vPool and current storage router
service_number = -1
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
service_number = max(mds_service.number, service_number)
if fresh_only is True and service_number >= 0:
return # There is already 1 or more MDS services running, aborting
# VALIDATIONS
# 1. Find free port based on MDS services for all vPools on current storage router
client = SSHClient(storagerouter)
mdsservice_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.MD_SERVER)
occupied_ports = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
occupied_ports.extend(service.ports)
mds_port_range = Configuration.get(
'/ovs/framework/hosts/{0}/ports|mds'.format(
System.get_my_machine_id(client)))
free_ports = System.get_free_ports(selected_range=mds_port_range,
exclude=occupied_ports,
nr=1,
client=client)
if not free_ports:
raise RuntimeError(
'Failed to find an available port on storage router {0} within range {1}'
.format(storagerouter.name, mds_port_range))
# 2. Partition check
db_partition = None
for disk in storagerouter.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.DB in partition.roles:
db_partition = partition
break
if db_partition is None:
raise RuntimeError(
'Could not find DB partition on storage router {0}'.format(
storagerouter.name))
# 3. Verify storage driver configured
storagedrivers = [
sd for sd in vpool.storagedrivers
if sd.storagerouter_guid == storagerouter.guid
]
if not storagedrivers:
raise RuntimeError(
'Expected to find a configured storagedriver for vpool {0} on storage router {1}'
.format(vpool.name, storagerouter.name))
storagedriver = storagedrivers[0]
# MODEL UPDATES
# 1. Service
service_number += 1
service = Service()
service.name = 'metadataserver_{0}_{1}'.format(vpool.name,
service_number)
service.type = mdsservice_type
service.ports = [free_ports[0]]
service.storagerouter = storagerouter
service.save()
mds_service = MDSService()
mds_service.vpool = vpool
mds_service.number = service_number
mds_service.service = service
mds_service.save()
# 2. Storage driver partitions
from ovs.lib.storagedriver import StorageDriverController
StorageDriverController.add_storagedriverpartition(
storagedriver, {
'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': db_partition,
'mds_service': mds_service
})
# CONFIGURATIONS
# 1. Volumedriver
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service is not None:
if mds_service.vpool_guid == vpool.guid:
sdp = [
sd_partition for sd_partition in
mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB
and sd_partition.sub_role ==
StorageDriverPartition.SUBROLE.MDS
][0]
mds_nodes.append({
'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp.path,
'scratch_directory': sdp.path
})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool.guid, storagedriver.storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
return mds_service
@staticmethod
def remove_mds_service(mds_service,
vpool,
reconfigure,
allow_offline=False):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:type mds_service: MDSService
:param vpool: The vPool for which the MDS service will be removed
:type vpool: VPool
:param reconfigure: Indicates whether reconfiguration is required
:type reconfigure: bool
:param allow_offline: Indicates whether it's OK that the node for which mds services are cleaned is offline
:type allow_offline: bool
"""
if len(mds_service.vdisks_guids) > 0 and allow_offline is False:
raise RuntimeError(
'Cannot remove MDSService that is still serving disks')
mdsservice_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.MD_SERVER)
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
if allow_offline is True: # Certain vdisks might still be attached to this offline MDS service --> Delete relations
for junction in mds_service.vdisks:
junction.delete()
mds_service.delete()
mds_service.service.delete()
storagerouter = mds_service.service.storagerouter
try:
client = SSHClient(storagerouter)
if reconfigure is True:
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
sdp = [
sd_partition for sd_partition in
mds_service.storagedriver_partitions
if sd_partition.role == DiskPartition.ROLES.DB
and sd_partition.sub_role ==
StorageDriverPartition.SUBROLE.MDS
][0]
mds_nodes.append({
'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp.path,
'scratch_directory': sdp.path
})
# Generate the correct section in the Storage Driver's configuration
storagedriver = [
sd for sd in storagerouter.storagedrivers
if sd.vpool_guid == vpool.guid
][0]
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool.guid,
storagedriver.storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_metadata_server(
mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reconfigure)
tries = 5
while tries > 0:
try:
root_client = SSHClient(storagerouter, username='******')
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
MDSServiceController._logger.debug(
'Recursively removed {0}'.format(dir_name))
break
except Exception:
MDSServiceController._logger.debug(
'Waiting for the MDS service to go down...')
time.sleep(5)
tries -= 1
if tries == 0:
raise
except UnableToConnectException:
if allow_offline is True:
MDSServiceController._logger.info(
'Allowed offline node during mds service removal')
else:
raise
@staticmethod
def sync_vdisk_to_reality(vdisk):
"""
Syncs a vdisk to reality (except hypervisor)
:param vdisk: vDisk to synchronize
:type vdisk: VDisk
:return: None
"""
vdisk.reload_client('storagedriver')
vdisk.invalidate_dynamics(['info'])
config = vdisk.info['metadata_backend_config']
config_dict = {}
for item in config:
if item['ip'] not in config_dict:
config_dict[item['ip']] = []
config_dict[item['ip']].append(item['port'])
mds_dict = {}
for junction in vdisk.mds_services:
service = junction.mds_service.service
storagerouter = service.storagerouter
if config[0]['ip'] == storagerouter.ip and config[0][
'port'] == service.ports[0]:
junction.is_master = True
junction.save()
if storagerouter.ip not in mds_dict:
mds_dict[storagerouter.ip] = []
mds_dict[storagerouter.ip].append(service.ports[0])
elif storagerouter.ip in config_dict and service.ports[
0] in config_dict[storagerouter.ip]:
junction.is_master = False
junction.save()
if storagerouter.ip not in mds_dict:
mds_dict[storagerouter.ip] = []
mds_dict[storagerouter.ip].append(service.ports[0])
else:
junction.delete()
for ip, ports in config_dict.iteritems():
for port in ports:
if ip not in mds_dict or port not in mds_dict[ip]:
service = ServiceList.get_by_ip_ports(ip, [port])
if service is not None:
mds_service_vdisk = MDSServiceVDisk()
mds_service_vdisk.vdisk = vdisk
mds_service_vdisk.mds_service = service.mds_service
mds_service_vdisk.is_master = config[0][
'ip'] == service.storagerouter.ip and config[0][
'port'] == service.ports[0]
mds_service_vdisk.save()
@staticmethod
def ensure_safety(vdisk, excluded_storagerouters=None):
"""
Ensures (or tries to ensure) the safety of a given vdisk (except hypervisor).
Assumptions:
* A local overloaded master is better than a non-local non-overloaded master
* Prefer master/services to be on different hosts, a subsequent slave on the same node doesn't add safety
* Don't actively overload services (e.g. configure an MDS as slave causing it to get overloaded)
* Too much safety is not wanted (it adds loads to nodes while not required)
:param vdisk: vDisk to calculate a new safety for
:type vdisk: VDisk
:param excluded_storagerouters: Storagerouters to leave out of calculation (Eg: When 1 is down or unavailable)
:type excluded_storagerouters: list
:return: None
"""
def _add_suitable_nodes(_importance, _safety):
if len(nodes) < _safety:
for local_load in sorted(all_info_dict[_importance]['loads']):
for local_service in all_info_dict[_importance]['loads'][
local_load]:
if len(
nodes
) < _safety and local_service.storagerouter.ip not in nodes:
try:
SSHClient(local_service.storagerouter)
new_services.append(local_service)
nodes.add(local_service.storagerouter.ip)
except UnableToConnectException:
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Skipping storagerouter with IP {1} as it is unreachable'
.format(vdisk.guid,
service.storagerouter.ip))
return nodes, new_services
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Start checkup for virtual disk {1}'.format(
vdisk.guid, vdisk.name))
tlogs = Configuration.get('/ovs/framework/storagedriver|mds_tlogs')
safety = Configuration.get('/ovs/framework/storagedriver|mds_safety')
max_load = Configuration.get(
'/ovs/framework/storagedriver|mds_maxload')
######################
# GATHER INFORMATION #
######################
vdisk.reload_client('storagedriver')
vdisk.reload_client('objectregistry')
vdisk.invalidate_dynamics(['storagedriver_id', 'storagerouter_guid'])
if vdisk.storagerouter_guid is None:
raise SRCObjectNotFoundException(
'Cannot ensure MDS safety for vDisk {0} with guid {1} because vDisk is not attached to any Storage Router'
.format(vdisk.name, vdisk.guid))
if excluded_storagerouters is None:
excluded_storagerouters = []
# Sorted was added merely for unittests, because they rely on specific order of services and their ports
# Default sorting behavior for relations used to be based on order in which relations were added
# Now sorting is based on guid (DAL speedup changes)
nodes = set()
services = sorted([
mds_service.service for mds_service in vdisk.vpool.mds_services
if mds_service.service.storagerouter not in excluded_storagerouters
],
key=lambda k: k.ports)
service_per_key = {}
for service in services:
nodes.add(service.storagerouter.ip)
service_per_key['{0}:{1}'.format(service.storagerouter.ip,
service.ports[0])] = service
# Create a pool of StorageRouters being a part of the primary and secondary domains of this Storage Router
vdisk_storagerouter = StorageRouter(vdisk.storagerouter_guid)
primary_domains = [
junction.domain for junction in vdisk_storagerouter.domains
if junction.backup is False
]
secondary_domains = [
junction.domain for junction in vdisk_storagerouter.domains
if junction.backup is True
]
primary_storagerouters = set()
secondary_storagerouters = set()
for domain in primary_domains:
primary_storagerouters.update(
StorageRouterList.get_primary_storagerouters_for_domain(
domain))
for domain in secondary_domains:
secondary_storagerouters.update(
StorageRouterList.get_primary_storagerouters_for_domain(
domain))
# In case no domains have been configured
if len(primary_storagerouters) == 0:
primary_storagerouters = set(
StorageRouterList.get_storagerouters())
if vdisk_storagerouter not in primary_storagerouters or vdisk_storagerouter in secondary_storagerouters:
raise ValueError(
'StorageRouter {0} for vDisk {1} should be part of the primary domains and NOT be part of the secondary domains'
.format(vdisk_storagerouter.name, vdisk.name))
# Remove all storagerouters from secondary which are present in primary
secondary_storagerouters = secondary_storagerouters.difference(
primary_storagerouters)
###################################
# VERIFY RECONFIGURATION REQUIRED #
###################################
vdisk.invalidate_dynamics(['info'])
configs = vdisk.info[
'metadata_backend_config'] # Ordered MASTER, SLAVE (secondary domain of master)
master_service = None
reconfigure_reasons = []
if len(configs) > 0:
config = configs.pop(0)
config_key = '{0}:{1}'.format(config['ip'], config['port'])
master_service = service_per_key.get(config_key)
if master_service is None:
reconfigure_reasons.append(
'Master ({0}:{1}) cannot be used anymore'.format(
config['ip'], config['port']))
slave_services = []
for config in configs:
config_key = '{0}:{1}'.format(config['ip'], config['port'])
if config_key in service_per_key:
slave_services.append(service_per_key[config_key])
else:
reconfigure_reasons.append(
'Slave ({0}:{1}) cannot be used anymore'.format(
config['ip'], config['port']))
# If MDS already in use, take current load, else take next load
all_info_dict = {
'primary': {
'used': [],
'loads': {},
'available': []
},
'secondary': {
'used': [],
'loads': {},
'available': []
}
}
services_load = {}
for service in services:
importance = None
if service.storagerouter in primary_storagerouters:
importance = 'primary'
elif service.storagerouter in secondary_storagerouters:
importance = 'secondary'
loads = MDSServiceController.get_mds_load(service.mds_service)
if service == master_service or service in slave_services: # Service is still in use
load = loads[0]
if importance is not None:
all_info_dict[importance]['used'].append(service)
else:
reconfigure_reasons.append(
'Service {0} cannot be used anymore because storagerouter with IP {1} is not part of the domains'
.format(service.name, service.storagerouter.ip))
else: # Service is not in use, but available
load = loads[1]
services_load[service] = load
if importance is not None:
all_info_dict[importance]['available'].append(service)
if load <= max_load:
if load not in all_info_dict[importance]['loads']:
all_info_dict[importance]['loads'][load] = []
all_info_dict[importance]['loads'][load].append(service)
service_nodes = []
if master_service is not None:
service_nodes.append(master_service.storagerouter.ip)
for service in slave_services:
ip = service.storagerouter.ip
if ip in service_nodes:
reconfigure_reasons.append(
'Multiple MDS services on the same node')
else:
service_nodes.append(ip)
if len(service_nodes) > safety:
reconfigure_reasons.append('Too much safety')
if len(service_nodes) < safety and len(service_nodes) < len(nodes):
reconfigure_reasons.append('Not enough safety')
if master_service is not None and services_load[
master_service] > max_load:
reconfigure_reasons.append('Master overloaded')
if master_service is not None and master_service.storagerouter_guid != vdisk.storagerouter_guid:
reconfigure_reasons.append('Master is not local')
if any(service for service in slave_services
if services_load[service] > max_load):
reconfigure_reasons.append('One or more slaves overloaded')
# Check reconfigure required based upon domains
recommended_primary = math.ceil(
safety / 2.0) if len(secondary_storagerouters) > 0 else safety
recommended_secondary = safety - recommended_primary
if master_service is not None and master_service not in all_info_dict[
'primary']['used']:
# Master service not present in primary domain
reconfigure_reasons.append('Master service not in primary domain')
primary_services_used = len(all_info_dict['primary']['used'])
primary_services_available = len(all_info_dict['primary']['available'])
if primary_services_used < recommended_primary and primary_services_used < primary_services_available:
# More services can be used in primary domain
reconfigure_reasons.append(
'Not enough services in use in primary domain')
if primary_services_used > recommended_primary:
# Too many services in primary domain
reconfigure_reasons.append(
'Too many services in use in primary domain')
# More services can be used in secondary domain
secondary_services_used = len(all_info_dict['secondary']['used'])
secondary_services_available = len(
all_info_dict['secondary']['available'])
if secondary_services_used < recommended_secondary and secondary_services_used < secondary_services_available:
reconfigure_reasons.append(
'Not enough services in use in secondary domain')
if secondary_services_used > recommended_secondary:
# Too many services in secondary domain
reconfigure_reasons.append(
'Too many services in use in secondary domain')
# If secondary domain present, check order in which the slave services are configured
secondary = False
for slave_service in slave_services:
if secondary is True and slave_service in all_info_dict['primary'][
'used']:
reconfigure_reasons.append(
'A slave in secondary domain has priority over a slave in primary domain'
)
break
if slave_service in all_info_dict['secondary']['used']:
secondary = True
if not reconfigure_reasons:
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: No reconfiguration required'.format(
vdisk.guid))
MDSServiceController.sync_vdisk_to_reality(vdisk)
return
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Reconfiguration required. Reasons:'.format(
vdisk.guid))
for reason in reconfigure_reasons:
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: * {1}'.format(vdisk.guid, reason))
############################
# CREATE NEW CONFIGURATION #
############################
# Check whether the master (if available) is non-local to the vdisk and/or is overloaded
new_services = []
master_ok = master_service is not None
if master_ok is True:
master_ok = master_service.storagerouter_guid == vdisk.storagerouter_guid and services_load[
master_service] <= max_load
previous_master = None
if master_ok:
# Add this master to the fresh configuration
new_services.append(master_service)
else:
# Try to find the best non-overloaded LOCAL MDS slave to make master
candidate_master_service = None
candidate_master_load = 0
local_mds = None
local_mds_load = 0
for service in all_info_dict['primary']['available']:
load = services_load[service]
if load <= max_load and service.storagerouter_guid == vdisk.storagerouter_guid:
if local_mds is None or local_mds_load > load:
# This service is a non-overloaded local MDS
local_mds = service
local_mds_load = load
if service in slave_services:
if candidate_master_service is None or candidate_master_load > load:
# This service is a non-overloaded local slave
candidate_master_service = service
candidate_master_load = load
if candidate_master_service is not None:
# A non-overloaded local slave was found.
client = MetadataServerClient.load(candidate_master_service)
try:
amount_of_tlogs = client.catch_up(str(vdisk.volume_id),
True)
except RuntimeError as ex:
if 'Namespace does not exist' in ex.message:
client.create_namespace(str(vdisk.volume_id))
amount_of_tlogs = client.catch_up(
str(vdisk.volume_id), True)
else:
raise
if amount_of_tlogs < tlogs:
# Almost there. Catching up right now, and continue as soon as it's up-to-date
start = time.time()
client.catch_up(str(vdisk.volume_id), False)
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Catchup took {1}s'.format(
vdisk.guid, round(time.time() - start, 2)))
# It's up to date, so add it as a new master
new_services.append(candidate_master_service)
if master_service is not None:
# The current master (if available) is now candidate to become one of the slaves
slave_services.append(master_service)
previous_master = master_service
else:
# It's not up to date, keep the previous master (if available) and give the local slave some more time to catch up
if master_service is not None:
new_services.append(master_service)
new_services.append(candidate_master_service)
if candidate_master_service in slave_services:
slave_services.remove(candidate_master_service)
else:
# There's no non-overloaded local slave found. Keep the current master (if available) and add a local MDS (if available) as slave
if master_service is not None:
new_services.append(master_service)
if local_mds is not None:
new_services.append(local_mds)
if local_mds in slave_services:
slave_services.remove(local_mds)
# At this point, there might (or might not) be a (new) master, and a (catching up) slave. The rest of the non-local
# MDS nodes must now be added to the configuration until the safety is reached. There's always one extra
# slave recycled to make sure there's always an (almost) up-to-date slave ready for failover
nodes = set(service.storagerouter.ip for service in new_services)
# Recycle slave for faster failover
secondary_node_count = 0
service_to_recycle = None
if len(nodes) < safety:
if recommended_primary > 1: # If primary is 1, we only have master in primary
# Try to recycle slave which is in primary domain
for load in sorted(all_info_dict['primary']['loads']):
for service in all_info_dict['primary']['loads'][load]:
if service_to_recycle is None and service in slave_services and service.storagerouter.ip not in nodes:
try:
SSHClient(service.storagerouter)
service_to_recycle = service
except UnableToConnectException:
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Skipping storagerouter with IP {1} as it is unreachable'
.format(vdisk.guid,
service.storagerouter.ip))
# Try to recycle slave which is in secondary domain if none found in primary
if service_to_recycle is None and len(
secondary_storagerouters) > 0:
for load in sorted(all_info_dict['secondary']['loads']):
for service in all_info_dict['secondary']['loads'][load]:
if service_to_recycle is None and service in slave_services and service.storagerouter.ip not in nodes:
try:
SSHClient(service.storagerouter)
service_to_recycle = service
secondary_node_count = 1 # We do not want to configure the secondary slave BEFORE the primary slaves
except UnableToConnectException:
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Skipping storagerouter with IP {1} as it is unreachable'
.format(vdisk.guid,
service.storagerouter.ip))
if service_to_recycle is not None:
slave_services.remove(service_to_recycle)
if secondary_node_count == 0: # Add service to recycle because its in primary domain
new_services.append(service_to_recycle)
nodes.add(service_to_recycle.storagerouter.ip)
# Add extra (new) slaves until primary safety reached
nodes, new_services = _add_suitable_nodes(_importance='primary',
_safety=recommended_primary)
# Add recycled secondary slave after primary slaves have been added
if secondary_node_count == 1:
new_services.append(service_to_recycle)
nodes.add(service_to_recycle.storagerouter.ip)
# Add extra (new) slaves until secondary safety reached
if len(secondary_storagerouters) > 0:
nodes, new_services = _add_suitable_nodes(_importance='secondary',
_safety=safety)
# Add extra slaves from primary domain in case no suitable nodes found in secondary domain
if len(nodes) < safety:
nodes, new_services = _add_suitable_nodes(
_importance='primary', _safety=safety)
# Build the new configuration and update the vdisk
configs_no_ex_master = []
configs_all = []
for service in new_services:
client = MetadataServerClient.load(service)
client.create_namespace(str(vdisk.volume_id))
# noinspection PyArgumentList
config = MDSNodeConfig(address=str(service.storagerouter.ip),
port=service.ports[0])
if previous_master != service:
configs_no_ex_master.append(config)
configs_all.append(config)
try:
if len(configs_no_ex_master) != len(configs_all):
vdisk.storagedriver_client.update_metadata_backend_config(
volume_id=str(vdisk.volume_id),
metadata_backend_config=MDSMetaDataBackendConfig(
configs_no_ex_master))
vdisk.storagedriver_client.update_metadata_backend_config(
volume_id=str(vdisk.volume_id),
metadata_backend_config=MDSMetaDataBackendConfig(configs_all))
except Exception:
MDSServiceController._logger.exception(
'MDS safety: vDisk {0}: Failed to update the metadata backend configuration'
.format(vdisk.guid))
raise Exception(
'MDS configuration for volume {0} with guid {1} could not be changed'
.format(vdisk.name, vdisk.guid))
for service in new_services[1:]:
client = MetadataServerClient.load(service)
client.set_role(str(vdisk.volume_id),
MetadataServerClient.MDS_ROLE.SLAVE)
MDSServiceController.sync_vdisk_to_reality(vdisk)
MDSServiceController._logger.debug(
'MDS safety: vDisk {0}: Completed'.format(vdisk.guid))
@staticmethod
def get_preferred_mds(storagerouter, vpool):
"""
Gets the MDS on this StorageRouter/VPool pair which is preferred to achieve optimal balancing
:param storagerouter: Storagerouter to retrieve the best MDS service for
:type storagerouter: StorageRouter
:param vpool: vPool to retrieve the best MDS service for
:type vpool: VPool
:return: Preferred MDS service (least loaded), current load on that MDS service
:rtype: tuple
"""
mds_service = (None, float('inf'))
for current_mds_service in vpool.mds_services:
if current_mds_service.service.storagerouter_guid == storagerouter.guid:
load = MDSServiceController.get_mds_load(
current_mds_service)[0]
if mds_service is None or load < mds_service[1]:
mds_service = (current_mds_service, load)
return mds_service
@staticmethod
def get_mds_load(mds_service):
"""
Gets a 'load' for an MDS service based on its capacity and the amount of assigned VDisks
:param mds_service: MDS service the get current load for
:type mds_service: MDSService
:return: Load of the MDS service
:rtype: tuple
"""
service_capacity = float(mds_service.capacity)
if service_capacity < 0:
return 50, 50
if service_capacity == 0:
return float('inf'), float('inf')
usage = len(mds_service.vdisks_guids)
return round(usage / service_capacity * 100.0, 5), round(
(usage + 1) / service_capacity * 100.0, 5)
@staticmethod
def get_mds_storagedriver_config_set(vpool, check_online=False):
"""
Builds a configuration for all StorageRouters from a given VPool with following goals:
* Primary MDS is the local one
* All slaves are on different hosts
* Maximum `mds_safety` nodes are returned
The configuration returned is the default configuration used by the volumedriver of which in normal use-cases
only the 1st entry is used, because at volume creation time, the volumedriver needs to create 1 master MDS
During ensure_safety, we actually create/set the MDS slaves for each volume
:param vpool: vPool to get storagedriver configuration for
:type vpool: VPool
:param check_online: Check whether the storage routers are actually responsive
:type check_online: bool
:return: MDS configuration for a vPool
:rtype: dict
"""
mds_per_storagerouter = {}
mds_per_load = {}
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
if check_online is True:
try:
SSHClient(storagerouter)
except UnableToConnectException:
continue
mds_service, load = MDSServiceController.get_preferred_mds(
storagerouter, vpool)
if mds_service is None:
raise RuntimeError('Could not find an MDS service')
mds_per_storagerouter[storagerouter] = {
'host': storagerouter.ip,
'port': mds_service.service.ports[0]
}
if load not in mds_per_load:
mds_per_load[load] = []
mds_per_load[load].append(storagerouter)
safety = Configuration.get('/ovs/framework/storagedriver|mds_safety')
config_set = {}
for storagerouter, ip_info in mds_per_storagerouter.iteritems():
config_set[storagerouter.guid] = [ip_info]
for importance in ['primary', 'secondary']:
domains = [
junction.domain for junction in storagerouter.domains
if junction.backup is (importance == 'secondary')
]
possible_storagerouters = set()
for domain in domains:
possible_storagerouters.update(
StorageRouterList.
get_primary_storagerouters_for_domain(domain))
for load in sorted(mds_per_load):
if len(config_set[storagerouter.guid]) >= safety:
break
other_storagerouters = mds_per_load[load]
random.shuffle(other_storagerouters)
for other_storagerouter in other_storagerouters:
if len(config_set[storagerouter.guid]) >= safety:
break
if other_storagerouter != storagerouter and other_storagerouter in possible_storagerouters:
config_set[storagerouter.guid].append(
mds_per_storagerouter[other_storagerouter])
return config_set
@staticmethod
@celery.task(name='ovs.mds.mds_checkup',
schedule=Schedule(minute='30', hour='0,4,8,12,16,20'))
@ensure_single(task_name='ovs.mds.mds_checkup', mode='CHAINED')
def mds_checkup():
"""
Validates the current MDS setup/configuration and takes actions where required
"""
MDSServiceController._logger.info('MDS checkup - Started')
mds_dict = {}
for vpool in VPoolList.get_vpools():
MDSServiceController._logger.info('MDS checkup - vPool {0}'.format(
vpool.name))
mds_dict[vpool] = {}
for mds_service in vpool.mds_services:
storagerouter = mds_service.service.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': None,
'services': []
}
try:
mds_dict[vpool][storagerouter]['client'] = SSHClient(
storagerouter, username='******')
MDSServiceController._logger.info(
'MDS checkup - vPool {0} - Storage Router {1} - ONLINE'
.format(vpool.name, storagerouter.name))
except UnableToConnectException:
MDSServiceController._logger.info(
'MDS checkup - vPool {0} - Storage Router {1} - OFFLINE'
.format(vpool.name, storagerouter.name))
mds_dict[vpool][storagerouter]['services'].append(mds_service)
failures = []
max_load = Configuration.get(
'/ovs/framework/storagedriver|mds_maxload')
for vpool, storagerouter_info in mds_dict.iteritems():
# 1. First, make sure there's at least one MDS on every StorageRouter that's not overloaded
# If not, create an extra MDS for that StorageRouter
for storagerouter in storagerouter_info:
client = mds_dict[vpool][storagerouter]['client']
mds_services = mds_dict[vpool][storagerouter]['services']
has_room = False
for mds_service in mds_services[:]:
if mds_service.capacity == 0 and len(
mds_service.vdisks_guids) == 0:
MDSServiceController._logger.info(
'MDS checkup - Removing mds_service {0} for vPool {1}'
.format(mds_service.number, vpool.name))
MDSServiceController.remove_mds_service(
mds_service,
vpool,
reconfigure=True,
allow_offline=client is None)
mds_services.remove(mds_service)
for mds_service in mds_services:
_, load = MDSServiceController.get_mds_load(mds_service)
if load < max_load:
has_room = True
break
MDSServiceController._logger.info(
'MDS checkup - vPool {0} - Storage Router {1} - Capacity available: {2}'
.format(vpool.name, storagerouter.name, has_room))
if has_room is False and client is not None:
mds_service = MDSServiceController.prepare_mds_service(
storagerouter=storagerouter,
vpool=vpool,
fresh_only=False,
reload_config=True)
if mds_service is None:
raise RuntimeError('Could not add MDS node')
mds_services.append(mds_service)
mds_config_set = MDSServiceController.get_mds_storagedriver_config_set(
vpool, True)
for storagerouter in storagerouter_info:
client = mds_dict[vpool][storagerouter]['client']
if client is None:
MDSServiceController._logger.info(
'MDS checkup - vPool {0} - Storage Router {1} - Marked as offline, not setting default MDS configuration'
.format(vpool.name, storagerouter.name))
continue
storagedriver = [
sd for sd in storagerouter.storagedrivers
if sd.vpool_guid == vpool.guid
][0]
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool.guid,
storagedriver.storagedriver_id)
storagedriver_config.load()
if storagedriver_config.is_new is False:
MDSServiceController._logger.info(
'MDS checkup - vPool {0} - Storage Router {1} - Storing default MDS configuration: {2}'
.format(vpool.name, storagerouter.name,
mds_config_set[storagerouter.guid]))
storagedriver_config.configure_filesystem(
fs_metadata_backend_mds_nodes=mds_config_set[
storagerouter.guid])
storagedriver_config.save(client)
# 2. Per VPool, execute a safety check, making sure the master/slave configuration is optimal.
MDSServiceController._logger.info(
'MDS checkup - vPool {0} - Ensuring safety for all virtual disks'
.format(vpool.name))
for vdisk in vpool.vdisks:
try:
MDSServiceController.ensure_safety(vdisk)
except Exception:
message = 'Ensure safety for vDisk {0} with guid {1} failed'.format(
vdisk.name, vdisk.guid)
MDSServiceController._logger.exception(message)
failures.append(message)
if len(failures) > 0:
raise Exception('\n - ' + '\n - '.join(failures))
MDSServiceController._logger.info('MDS checkup - Finished')
@staticmethod
def print_current_mds_layout():
"""
Prints the current MDS layout
"""
output = [
'', 'Open vStorage - MDS debug information',
'=====================================',
'timestamp: {0}'.format(time.time()), ''
]
for storagerouter in StorageRouterList.get_storagerouters():
output.append('+ {0} ({1})'.format(storagerouter.name,
storagerouter.ip))
vpools = set(sd.vpool for sd in storagerouter.storagedrivers)
for vpool in vpools:
output.append(' + {0}'.format(vpool.name))
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
masters, slaves = 0, 0
for junction in mds_service.vdisks:
if junction.is_master:
masters += 1
else:
slaves += 1
capacity = mds_service.capacity
if capacity == -1:
capacity = 'infinite'
load, _ = MDSServiceController.get_mds_load(
mds_service)
if load == float('inf'):
load = 'infinite'
else:
load = '{0}%'.format(round(load, 2))
output.append(
' + {0} - port {1} - {2} master(s), {3} slave(s) - capacity: {4}, load: {5}'
.format(mds_service.number,
mds_service.service.ports[0], masters,
slaves, capacity, load))
print '\n'.join(output)
class StorageDriverClient(object):
"""
Client to access storagedriver client
"""
storagerouterclient.Logger.setupLogging(
LogHandler.load_path('storagerouterclient'))
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
VOLDRV_DTL_SYNC = 'Synchronous'
VOLDRV_DTL_ASYNC = 'Asynchronous'
VOLDRV_NO_CACHE = 'NoCache'
VOLDRV_CACHE_ON_READ = 'CacheOnRead'
VOLDRV_CONTENT_BASED = 'ContentBased'
VOLDRV_CACHE_ON_WRITE = 'CacheOnWrite'
VOLDRV_LOCATION_BASED = 'LocationBased'
VOLDRV_DTL_MANUAL_MODE = 'Manual'
VOLDRV_DTL_AUTOMATIC_MODE = 'Automatic'
VOLDRV_DTL_TRANSPORT_TCP = 'TCP'
VOLDRV_DTL_TRANSPORT_RSOCKET = 'RSocket'
FRAMEWORK_DTL_SYNC = 'sync'
FRAMEWORK_DTL_ASYNC = 'a_sync'
FRAMEWORK_DTL_NO_SYNC = 'no_sync'
FRAMEWORK_NO_CACHE = 'none'
FRAMEWORK_CACHE_ON_READ = 'on_read'
FRAMEWORK_CONTENT_BASED = 'dedupe'
FRAMEWORK_CACHE_ON_WRITE = 'on_write'
FRAMEWORK_LOCATION_BASED = 'non_dedupe'
FRAMEWORK_DTL_TRANSPORT_TCP = 'tcp'
FRAMEWORK_DTL_TRANSPORT_RSOCKET = 'rdma'
METADATA_CACHE_PAGE_SIZE = 256 * 24
DEFAULT_METADATA_CACHE_SIZE = 8192 * METADATA_CACHE_PAGE_SIZE
VDISK_CACHE_MAP = {
FRAMEWORK_NO_CACHE: ReadCacheBehaviour.NO_CACHE,
FRAMEWORK_CACHE_ON_READ: ReadCacheBehaviour.CACHE_ON_READ,
FRAMEWORK_CACHE_ON_WRITE: ReadCacheBehaviour.CACHE_ON_WRITE
}
VPOOL_CACHE_MAP = {
FRAMEWORK_NO_CACHE: VOLDRV_NO_CACHE,
FRAMEWORK_CACHE_ON_READ: VOLDRV_CACHE_ON_READ,
FRAMEWORK_CACHE_ON_WRITE: VOLDRV_CACHE_ON_WRITE
}
VDISK_DEDUPE_MAP = {
FRAMEWORK_CONTENT_BASED: ReadCacheMode.CONTENT_BASED,
FRAMEWORK_LOCATION_BASED: ReadCacheMode.LOCATION_BASED
}
VPOOL_DEDUPE_MAP = {
FRAMEWORK_CONTENT_BASED: VOLDRV_CONTENT_BASED,
FRAMEWORK_LOCATION_BASED: VOLDRV_LOCATION_BASED
}
VDISK_DTL_MODE_MAP = {
FRAMEWORK_DTL_SYNC: DTLMode.SYNCHRONOUS,
FRAMEWORK_DTL_ASYNC: DTLMode.ASYNCHRONOUS,
FRAMEWORK_DTL_NO_SYNC: None
}
VPOOL_DTL_MODE_MAP = {
FRAMEWORK_DTL_SYNC: VOLDRV_DTL_SYNC,
FRAMEWORK_DTL_ASYNC: VOLDRV_DTL_ASYNC,
FRAMEWORK_DTL_NO_SYNC: None
}
VPOOL_DTL_TRANSPORT_MAP = {
FRAMEWORK_DTL_TRANSPORT_TCP: VOLDRV_DTL_TRANSPORT_TCP,
FRAMEWORK_DTL_TRANSPORT_RSOCKET: VOLDRV_DTL_TRANSPORT_RSOCKET
}
REVERSE_CACHE_MAP = {
VOLDRV_NO_CACHE: FRAMEWORK_NO_CACHE,
VOLDRV_CACHE_ON_READ: FRAMEWORK_CACHE_ON_READ,
VOLDRV_CACHE_ON_WRITE: FRAMEWORK_CACHE_ON_WRITE,
ReadCacheBehaviour.NO_CACHE: FRAMEWORK_NO_CACHE,
ReadCacheBehaviour.CACHE_ON_READ: FRAMEWORK_CACHE_ON_READ,
ReadCacheBehaviour.CACHE_ON_WRITE: FRAMEWORK_CACHE_ON_WRITE
}
REVERSE_DEDUPE_MAP = {
VOLDRV_CONTENT_BASED: FRAMEWORK_CONTENT_BASED,
VOLDRV_LOCATION_BASED: FRAMEWORK_LOCATION_BASED,
ReadCacheMode.CONTENT_BASED: FRAMEWORK_CONTENT_BASED,
ReadCacheMode.LOCATION_BASED: FRAMEWORK_LOCATION_BASED
}
REVERSE_DTL_MODE_MAP = {
VOLDRV_DTL_SYNC: FRAMEWORK_DTL_SYNC,
VOLDRV_DTL_ASYNC: FRAMEWORK_DTL_ASYNC,
DTLMode.SYNCHRONOUS: FRAMEWORK_DTL_SYNC,
DTLMode.ASYNCHRONOUS: FRAMEWORK_DTL_ASYNC
}
REVERSE_DTL_TRANSPORT_MAP = {
VOLDRV_DTL_TRANSPORT_TCP: FRAMEWORK_DTL_TRANSPORT_TCP,
VOLDRV_DTL_TRANSPORT_RSOCKET: FRAMEWORK_DTL_TRANSPORT_RSOCKET
}
CLUSTER_SIZES = [4, 8, 16, 32, 64]
TLOG_MULTIPLIER_MAP = {4: 16, 8: 8, 16: 4, 32: 2, 64: 1, 128: 1}
DTL_STATUS = {
'': 0,
'ok_standalone': 10,
'ok_sync': 10,
'catch_up': 20,
'degraded': 30
}
EMPTY_STATISTICS = staticmethod(lambda: Statistics())
EMPTY_INFO = staticmethod(lambda: VolumeInfo())
STAT_SUMS = {
'operations': ['write_operations', 'read_operations'],
'cache_hits': ['sco_cache_hits', 'cluster_cache_hits'],
'cache_misses': ['sco_cache_misses'],
'4k_operations': ['4k_read_operations', '4k_write_operations'],
'data_transferred': ['data_written', 'data_read']
}
STAT_KEYS = [
'backend_data_read', 'backend_data_written', 'backend_read_operations',
'backend_write_operations', 'cluster_cache_hits',
'cluster_cache_misses', 'data_read', 'data_written',
'metadata_store_hits', 'metadata_store_misses', 'read_operations',
'sco_cache_hits', 'sco_cache_misses', 'write_operations',
'4k_read_operations', '4k_write_operations', 'stored'
]
STAT_KEYS.extend(STAT_SUMS.keys())
def __init__(self):
"""
Dummy init method
"""
pass
@staticmethod
def load(vpool):
"""
Initializes the wrapper given a vpool name for which it finds the corresponding Storage Driver
Loads and returns the client
:param vpool: vPool for which the StorageRouterClient needs to be loaded
"""
key = vpool.identifier
if key not in client_vpool_cache:
cluster_contacts = []
for storagedriver in vpool.storagedrivers[:3]:
cluster_contacts.append(
ClusterContact(str(storagedriver.cluster_ip),
storagedriver.ports[1]))
client = SRClient(str(vpool.guid), cluster_contacts)
client_vpool_cache[key] = client
return client_vpool_cache[key]
