def __init__(self, vpool_guid, storagedriver_id):
"""
Initializes the class
"""
_log_level = LOG_LEVEL_MAPPING[OVSLogger(
'extensions').getEffectiveLevel()]
# noinspection PyCallByClass,PyTypeChecker
storagerouterclient.Logger.setupLogging(
OVSLogger.load_path('storagerouterclient'), _log_level)
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
self._key = '/ovs/vpools/{0}/hosts/{1}/config'.format(
vpool_guid, storagedriver_id)
self._logger = OVSLogger('extensions')
self._dirty_entries = []
self.remote_path = Configuration.get_configuration_path(
self._key).strip('/')
# Load configuration
if Configuration.exists(self._key):
self.configuration = Configuration.get(self._key)
self.config_missing = False
else:
self.configuration = {}
self.config_missing = True
self._logger.debug(
'Could not find config {0}, a new one will be created'.format(
self._key))
class MetadataServerClient(object):
"""
Builds a MDSClient
"""
_logger = OVSLogger('extensions')
_log_level = LOG_LEVEL_MAPPING[_logger.getEffectiveLevel()]
# noinspection PyCallByClass,PyTypeChecker
storagerouterclient.Logger.setupLogging(
OVSLogger.load_path('storagerouterclient'), _log_level)
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
MDS_ROLE = type('MDSRole', (), {
'MASTER': Role.Master,
'SLAVE': Role.Slave
})
def __init__(self):
"""
Dummy init method
"""
pass
@staticmethod
def load(service, timeout=20):
"""
Loads a MDSClient
:param service: Service for which the MDSClient needs to be loaded
:type service: ovs.dal.hybrids.service.Service
:param timeout: All calls performed by this MDSClient instance will time out after this period (in seconds)
:type timeout: int
:return: An MDSClient instance for the specified Service
:rtype: MDSClient
"""
if service.storagerouter is None:
raise ValueError(
'Service {0} does not have a StorageRouter linked to it'.
format(service.name))
key = service.guid
# Create MDSClient instance if no instance has been cached yet or if another timeout has been specified
if key not in mdsclient_service_cache or timeout != mdsclient_service_cache[
key]['timeout']:
try:
# noinspection PyArgumentList
mdsclient_service_cache[key] = {
'client':
MDSClient(timeout_secs=timeout,
mds_node_config=MDSNodeConfig(
address=str(service.storagerouter.ip),
port=service.ports[0])),
'timeout':
timeout
}
except RuntimeError:
MetadataServerClient._logger.exception(
'Error loading MDSClient on {0}'.format(
service.storagerouter.ip))
return None
return mdsclient_service_cache[key]['client']
class MDSServiceController(MDSShared):
"""
Contains all BLL related to MDSServices
"""
_logger = Logger('lib')
_log_level = LOG_LEVEL_MAPPING[_logger.getEffectiveLevel()]
# noinspection PyCallByClass,PyTypeChecker
storagerouterclient.Logger.setupLogging(
Logger.load_path('storagerouterclient'), _log_level)
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
@staticmethod
def remove_mds_service(mds_service, reconfigure, allow_offline=False):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:type mds_service: ovs.dal.hybrids.j_mdsservice.MDSService
:param reconfigure: Indicates whether reconfiguration is required
:type reconfigure: bool
:param allow_offline: Indicates whether it's OK that the node for which mds services are cleaned is offline
:type allow_offline: bool
:raises RuntimeError: When vDisks present on the MDSService to be removed
No StorageDriver is linked to the MDSService to be removed
:raises UnableToConnectException: When StorageRouter on which the MDSService resides is unreachable and allow_offline flag is False
:return: None
:rtype: NoneType
"""
if len(mds_service.vdisks_guids) > 0 and allow_offline is False:
raise RuntimeError(
'Cannot remove MDSService that is still serving disks')
if len(
mds_service.storagedriver_partitions
) == 0 or mds_service.storagedriver_partitions[0].storagedriver is None:
raise RuntimeError(
'Failed to retrieve the linked StorageDriver to this MDS Service {0}'
.format(mds_service.service.name))
vpool = mds_service.vpool
root_client = None
storagerouter = mds_service.service.storagerouter
storagedriver = mds_service.storagedriver_partitions[0].storagedriver
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Removing MDS junction service for port {2}'
.format(storagerouter.name, vpool.name,
mds_service.service.ports[0]))
try:
root_client = SSHClient(endpoint=storagerouter, username='******')
MDSServiceController._logger.debug(
'StorageRouter {0} - vPool {1}: Established SSH connection'.
format(storagerouter.name, vpool.name))
except UnableToConnectException:
if allow_offline is True:
MDSServiceController._logger.warning(
'StorageRouter {0} - vPool {1}: Allowed offline node during MDS service removal'
.format(storagerouter.name, vpool.name))
else:
MDSServiceController._logger.exception(
'StorageRouter {0} - vPool {1}: Failed to connect to StorageRouter'
.format(storagerouter.name, vpool.name))
raise
# Reconfigure StorageDriver
if reconfigure is True and root_client is not None:
mds_nodes = []
for sd_partition in storagedriver.partitions:
if sd_partition.role == DiskPartition.ROLES.DB and sd_partition.sub_role == StorageDriverPartition.SUBROLE.MDS and sd_partition.mds_service != mds_service:
service = sd_partition.mds_service.service
mds_nodes.append({
'host':
service.storagerouter.ip,
'port':
service.ports[0],
'db_directory':
'{0}/db'.format(sd_partition.path),
'scratch_directory':
'{0}/scratch'.format(sd_partition.path)
})
# Generate the correct section in the StorageDriver's configuration
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Configuring StorageDriver with MDS nodes: {2}'
.format(storagerouter.name, vpool.name, mds_nodes))
storagedriver_config = StorageDriverConfiguration(
vpool.guid, storagedriver.storagedriver_id)
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(root_client)
# Clean up model
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Cleaning model'.format(
storagerouter.name, vpool.name))
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
if allow_offline is True: # Certain vDisks might still be attached to this offline MDS service --> Delete relations
for junction in mds_service.vdisks:
junction.delete()
mds_service.delete()
mds_service.service.delete()
# Clean up file system
if root_client is not None:
MDSServiceController._logger.info(
'StorageRouter {0} - vPool {1}: Deleting directories from file system: {2}'
.format(storagerouter.name, vpool.name, directories_to_clean))
tries = 5
while tries > 0:
try:
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
MDSServiceController._logger.debug(
'StorageRouter {0} - vPool {1}: Recursively removed directory: {2}'
.format(storagerouter.name, vpool.name, dir_name))
break
except Exception:
MDSServiceController._logger.warning(
'StorageRouter {0} - vPool {1}: Waiting for the MDS service to go down...'
.format(storagerouter.name, vpool.name))
time.sleep(5)
tries -= 1
if tries == 0:
MDSServiceController._logger.exception(
'StorageRouter {0} - vPool {1}: Deleting directories failed'
.format(storagerouter.name, vpool.name))
raise
@staticmethod
@ovs_task(name='ovs.mds.mds_checkup',
schedule=Schedule(minute='30', hour='0,4,8,12,16,20'),
ensure_single_info={'mode': 'CHAINED'})
def mds_checkup():
"""
Validates the current MDS setup/configuration and takes actions where required
Actions:
* Verify which StorageRouters are available
* Make mapping between vPools and its StorageRouters
* For each vPool make sure every StorageRouter has at least 1 MDS service with capacity available
* For each vPool retrieve the optimal configuration and store it for each StorageDriver
* For each vPool run an ensure safety for all vDisks
:raises RuntimeError: When ensure safety fails for any vDisk
:return: None
:rtype: NoneType
"""
MDSServiceController._logger.info('Started')
# Verify StorageRouter availability
root_client_cache = {}
storagerouters = StorageRouterList.get_storagerouters()
storagerouters.sort(key=lambda _sr: ExtensionsToolbox.advanced_sort(
element=_sr.ip, separator='.'))
offline_nodes = []
for storagerouter in storagerouters:
try:
root_client = SSHClient(endpoint=storagerouter,
username='******')
MDSServiceController._logger.debug(
'StorageRouter {0} - ONLINE'.format(storagerouter.name))
except UnableToConnectException:
root_client = None
offline_nodes.append(storagerouter)
MDSServiceController._logger.error(
'StorageRouter {0} - OFFLINE'.format(storagerouter.name))
root_client_cache[storagerouter] = root_client
# Create mapping per vPool and its StorageRouters
mds_dict = collections.OrderedDict()
for vpool in sorted(VPoolList.get_vpools(), key=lambda k: k.name):
MDSServiceController._logger.info('vPool {0}'.format(vpool.name))
mds_dict[vpool] = {}
# Loop all StorageDrivers and add StorageDriver to mapping
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': root_client_cache.get(storagerouter),
'services': [],
'storagedriver': storagedriver
}
# Loop all MDS Services and append services to appropriate vPool / StorageRouter combo
mds_services = vpool.mds_services
mds_services.sort(
key=lambda _mds_service: ExtensionsToolbox.advanced_sort(
element=_mds_service.service.storagerouter.ip,
separator='.'))
for mds_service in mds_services:
service = mds_service.service
storagerouter = service.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': root_client_cache.get(storagerouter),
'services': [],
'storagedriver': None
}
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Service on port {2}'.
format(vpool.name, storagerouter.name, service.ports[0]))
mds_dict[vpool][storagerouter]['services'].append(mds_service)
failures = []
for vpool, storagerouter_info in mds_dict.iteritems():
# Make sure there's at least 1 MDS on every StorageRouter that's not overloaded
# Remove all MDS Services which have been manually marked for removal (by setting its capacity to 0)
max_load = Configuration.get(
'/ovs/vpools/{0}/mds_config|mds_maxload'.format(vpool.guid))
for storagerouter in sorted(storagerouter_info,
key=lambda k: k.ip):
total_load = 0.0
root_client = mds_dict[vpool][storagerouter]['client']
mds_services = mds_dict[vpool][storagerouter]['services']
for mds_service in list(
sorted(mds_services, key=lambda k: k.number)):
port = mds_service.service.ports[0]
number = mds_service.number
# Manual intervention required here in order for the MDS to be cleaned up
# @TODO: Remove this and make a dynamic calculation to check which MDSes to remove
if mds_service.capacity == 0 and len(
mds_service.vdisks_guids) == 0:
MDSServiceController._logger.warning(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: Removing'
.format(vpool.name, storagerouter.name, number,
port))
try:
MDSServiceController.remove_mds_service(
mds_service=mds_service,
reconfigure=True,
allow_offline=root_client is None)
except Exception:
MDSServiceController._logger.exception(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: Failed to remove'
.format(vpool.name, storagerouter.name, number,
port))
mds_services.remove(mds_service)
else:
_, next_load = MDSServiceController.get_mds_load(
mds_service=mds_service)
if next_load == float('inf'):
total_load = sys.maxint * -1 # Cast to lowest possible value if any MDS service capacity is set to infinity
else:
total_load += next_load
if next_load < max_load:
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: Capacity available - Load at {4}%'
.format(vpool.name, storagerouter.name, number,
port, next_load))
else:
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: No capacity available - Load at {4}%'
.format(vpool.name, storagerouter.name, number,
port, next_load))
if total_load >= max_load * len(mds_services):
mds_services_to_add = int(
math.ceil((total_load - max_load * len(mds_services)) /
max_load))
MDSServiceController._logger.info(
'vPool {0} - StorageRouter {1} - Average load per service {2:.2f}% - Max load per service {3:.2f}% - {4} MDS service{5} will be added'
.format(vpool.name, storagerouter.name,
total_load / len(mds_services), max_load,
mds_services_to_add,
'' if mds_services_to_add == 1 else 's'))
for _ in range(mds_services_to_add):
MDSServiceController._logger.info(
'vPool {0} - StorageRouter {1} - Adding new MDS Service'
.format(vpool.name, storagerouter.name))
try:
mds_services.append(
MDSServiceController.prepare_mds_service(
storagerouter=storagerouter, vpool=vpool))
except Exception:
MDSServiceController._logger.exception(
'vPool {0} - StorageRouter {1} - Failed to create new MDS Service'
.format(vpool.name, storagerouter.name))
# After potentially having added new MDSes, retrieve the optimal configuration
mds_config_set = {}
try:
mds_config_set = MDSServiceController.get_mds_storagedriver_config_set(
vpool=vpool, offline_nodes=offline_nodes)
MDSServiceController._logger.debug(
'vPool {0} - Optimal configuration {1}'.format(
vpool.name, mds_config_set))
except (NotFoundException, RuntimeError):
MDSServiceController._logger.exception(
'vPool {0} - Failed to retrieve the optimal configuration'.
format(vpool.name))
# Apply the optimal MDS configuration per StorageDriver
for storagerouter in sorted(storagerouter_info,
key=lambda k: k.ip):
root_client = mds_dict[vpool][storagerouter]['client']
storagedriver = mds_dict[vpool][storagerouter]['storagedriver']
if storagedriver is None:
MDSServiceController._logger.critical(
'vPool {0} - StorageRouter {1} - No matching StorageDriver found'
.format(vpool.name, storagerouter.name))
continue
if storagerouter.guid not in mds_config_set:
MDSServiceController._logger.critical(
'vPool {0} - StorageRouter {1} - Not marked as offline, but could not retrieve an optimal MDS config'
.format(vpool.name, storagerouter.name))
continue
if root_client is None:
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Marked as offline, not setting optimal MDS configuration'
.format(vpool.name, storagerouter.name))
continue
storagedriver_config = StorageDriverConfiguration(
vpool_guid=vpool.guid,
storagedriver_id=storagedriver.storagedriver_id)
if storagedriver_config.config_missing is False:
optimal_mds_config = mds_config_set[storagerouter.guid]
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Storing optimal MDS configuration: {2}'
.format(vpool.name, storagerouter.name,
optimal_mds_config))
# Filesystem section in StorageDriver configuration are all parameters used for vDisks created directly on the filesystem
# So when a vDisk gets created on the filesystem, these MDSes will be assigned to them
storagedriver_config.configure_filesystem(
fs_metadata_backend_mds_nodes=optimal_mds_config)
storagedriver_config.save(root_client)
# Execute a safety check, making sure the master/slave configuration is optimal.
MDSServiceController._logger.info(
'vPool {0} - Ensuring safety for all vDisks'.format(
vpool.name))
for vdisk in vpool.vdisks:
try:
MDSServiceController.ensure_safety(vdisk_guid=vdisk.guid)
except Exception:
message = 'Ensure safety for vDisk {0} with guid {1} failed'.format(
vdisk.name, vdisk.guid)
MDSServiceController._logger.exception(message)
failures.append(message)
if len(failures) > 0:
raise RuntimeError('\n - ' + '\n - '.join(failures))
MDSServiceController._logger.info('Finished')
# noinspection PyUnresolvedReferences
@staticmethod
@ovs_task(name='ovs.mds.ensure_safety',
ensure_single_info={'mode': 'CHAINED'})
def ensure_safety(vdisk_guid, excluded_storagerouter_guids=None):
"""
Ensures (or tries to ensure) the safety of a given vDisk.
Assumptions:
* A local overloaded master is better than a non-local non-overloaded master
* Prefer master/slaves to be on different hosts, a subsequent slave on the same node doesn't add safety
* Don't actively overload services (e.g. configure an MDS as slave causing it to get overloaded)
* Too much safety is not wanted (it adds loads to nodes while not required)
* Order of slaves is:
* All slaves on StorageRouters in primary Domain of vDisk host
* All slaves on StorageRouters in secondary Domain of vDisk host
* Eg: Safety of 2 (1 master + 1 slave)
mds config = [local master in primary, slave in secondary]
* Eg: Safety of 3 (1 master + 2 slaves)
mds config = [local master in primary, slave in primary, slave in secondary]
* Eg: Safety of 4 (1 master + 3 slaves)
mds config = [local master in primary, slave in primary, slave in secondary, slave in secondary]
:param vdisk_guid: vDisk GUID to calculate a new safety for
:type vdisk_guid: str
:param excluded_storagerouter_guids: GUIDs of StorageRouters to leave out of calculation (Eg: When 1 is down or unavailable)
:type excluded_storagerouter_guids: list[str]
:raises RuntimeError: If host of vDisk is part of the excluded StorageRouters
If host of vDisk is not part of the StorageRouters in the primary domain
If catchup command fails for a slave
If MDS client cannot be created for any of the current or new MDS services
If updateMetadataBackendConfig would fail for whatever reason
:raises SRCObjectNotFoundException: If vDisk does not have a StorageRouter GUID
:return: None
:rtype: NoneType
"""
if excluded_storagerouter_guids is None:
excluded_storagerouter_guids = []
safety_ensurer = SafetyEnsurer(vdisk_guid,
excluded_storagerouter_guids)
safety_ensurer.ensure_safety()
@staticmethod
def get_preferred_mds(storagerouter, vpool):
"""
Gets the MDS on this StorageRouter/vPool pair which is preferred to achieve optimal balancing
:param storagerouter: StorageRouter to retrieve the best MDS service for
:type storagerouter: ovs.dal.hybrids.storagerouter.StorageRouter
:param vpool: vPool to retrieve the best MDS service for
:type vpool: ovs.dal.hybrids.vpool.VPool
:return: Preferred MDS service (least loaded), current load on that MDS service
:rtype: tuple(ovs.dal.hybrids.j_mdsservice.MDSService, float)
"""
mds_info = (None, float('inf'))
for mds_service in vpool.mds_services:
if mds_service.service.storagerouter_guid == storagerouter.guid:
load = MDSServiceController.get_mds_load(
mds_service=mds_service)[0]
if mds_info[0] is None or load < mds_info[1]:
mds_info = (mds_service, load)
return mds_info
@staticmethod
def get_mds_storagedriver_config_set(vpool, offline_nodes=None):
"""
Builds a configuration for all StorageRouters from a given vPool with following goals:
* Primary MDS is the local one
* All slaves are on different hosts
* Maximum `mds_safety` nodes are returned
The configuration returned is the default configuration used by the volumedriver of which in normal use-cases
only the 1st entry is used, because at volume creation time, the volumedriver needs to create 1 master MDS
During ensure_safety, we actually create/set the MDS slaves for each volume
:param vpool: vPool to get StorageDriver configuration for
:type vpool: ovs.dal.hybrids.vpool.VPool
:param offline_nodes: Nodes which are currently unreachable via the SSHClient functionality
:type offline_nodes: list
:raises RuntimeError: When no MDS Service can be found for a specific vPool/StorageRouter combo
:raises NotFoundException: When configuration management is unavailable
:return: MDS configuration for a vPool
:rtype: dict[list]
"""
if offline_nodes is None:
offline_nodes = []
mds_per_storagerouter = {}
mds_per_load = {}
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
if storagerouter in offline_nodes:
continue
mds_service, load = MDSServiceController.get_preferred_mds(
storagerouter, vpool)
if mds_service is None:
raise RuntimeError('Could not find an MDS service')
mds_per_storagerouter[storagerouter] = {
'host': storagerouter.ip,
'port': mds_service.service.ports[0]
}
if load not in mds_per_load:
mds_per_load[load] = []
mds_per_load[load].append(storagerouter)
safety = Configuration.get(
'/ovs/vpools/{0}/mds_config|mds_safety'.format(vpool.guid))
config_set = {}
for storagerouter, ip_info in mds_per_storagerouter.iteritems():
config_set[storagerouter.guid] = [ip_info]
for importance in ['primary', 'secondary']:
domains = [
junction.domain for junction in storagerouter.domains
if junction.backup is (importance == 'secondary')
]
possible_storagerouters = set()
for domain in domains:
possible_storagerouters.update(
StorageRouterList.
get_primary_storagerouters_for_domain(domain))
for load in sorted(mds_per_load):
if len(config_set[storagerouter.guid]) >= safety:
break
other_storagerouters = mds_per_load[load]
random.shuffle(other_storagerouters)
for other_storagerouter in other_storagerouters:
if len(config_set[storagerouter.guid]) >= safety:
break
if other_storagerouter != storagerouter and other_storagerouter in possible_storagerouters:
config_set[storagerouter.guid].append(
mds_per_storagerouter[other_storagerouter])
return config_set
@staticmethod
def monitor_mds_layout():
"""
Prints the current MDS layout
:return: None
:rtype: NoneType
"""
try:
while True:
output = [
'', 'Open vStorage - MDS debug information',
'=====================================',
'timestamp: {0}'.format(datetime.datetime.now()), ''
]
vpools_deployed = False
for storagerouter in sorted(
StorageRouterList.get_storagerouters(),
key=lambda k: k.name):
vpools = set(sd.vpool
for sd in storagerouter.storagedrivers)
if len(vpools) > 0:
vpools_deployed = True
output.append('+ {0} ({1})'.format(
storagerouter.name, storagerouter.ip))
for vpool in sorted(vpools, key=lambda k: k.name):
output.append(' + {0}'.format(vpool.name))
for mds_service in sorted(vpool.mds_services,
key=lambda k: k.number):
if mds_service.service.storagerouter_guid == storagerouter.guid:
masters, slaves = 0, 0
for junction in mds_service.vdisks:
if junction.is_master:
masters += 1
else:
slaves += 1
capacity = mds_service.capacity
if capacity == -1:
capacity = 'infinite'
load, _ = MDSServiceController.get_mds_load(
mds_service)
if load == float('inf'):
load = 'infinite'
else:
load = '{0}%'.format(round(load, 2))
output.append(
' + {0} - port {1} - {2} master(s), {3} slave(s) - capacity: {4}, load: {5}'
.format(mds_service.number,
mds_service.service.ports[0],
masters, slaves, capacity, load))
if vpools_deployed is False:
output.append('No vPools deployed')
print '\x1b[2J\x1b[H' + '\n'.join(output)
time.sleep(1)
except KeyboardInterrupt:
pass
@staticmethod
@ovs_task(name='ovs.mds.mds_catchup',
schedule=Schedule(minute='30', hour='*/2'),
ensure_single_info={'mode': 'DEFAULT'})
def mds_catchup():
"""
Looks to catch up all MDS slaves which are too far behind
Only one catch for every storagedriver is invoked
"""
# Only for caching purposes
def storagedriver_worker(queue, error_list):
# type: (Queue.Queue, List[str]) -> None
while not queue.empty():
mds_catch_up = queue.get() # type: MDSCatchUp
try:
mds_catch_up.catch_up(async=False)
except Exception as ex:
MDSServiceController._logger.exception(
'Exceptions while catching for vDisk {0}'.format(
mds_catch_up.vdisk.guid))
error_list.append(str(ex))
finally:
queue.task_done()
storagedriver_queues = {}
for vdisk in VDiskList.get_vdisks():
if vdisk.storagedriver_id not in storagedriver_queues:
storagedriver_queues[vdisk.storagedriver_id] = Queue.Queue()
# Putting it in the Queue ensures that the reference is still there so the caching is used optimally
catch_up = MDSCatchUp(vdisk.guid)
storagedriver_queues[vdisk.storagedriver_id].put(catch_up)
errors = []
threads = []
for storadriver_id, storagedriver_queue in storagedriver_queues.iteritems(
):
thread = Thread(target=storagedriver_worker,
args=(
storagedriver_queue,
errors,
))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
if len(errors) > 0:
raise RuntimeError(
'Exception occurred while catching up: \n - {0}'.format(
'\n - '.join(errors)))
class StorageRouterController(object):
"""
Contains all BLL related to StorageRouter
"""
_logger = Logger('lib')
_log_level = LOG_LEVEL_MAPPING[_logger.getEffectiveLevel()]
_os_manager = OSFactory.get_manager()
_service_manager = ServiceFactory.get_manager()
# noinspection PyCallByClass,PyTypeChecker
storagerouterclient.Logger.setupLogging(
Logger.load_path('storagerouterclient'), _log_level)
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
@staticmethod
@ovs_task(name='ovs.storagerouter.ping')
def ping(storagerouter_guid, timestamp):
"""
Update a StorageRouter's celery heartbeat
:param storagerouter_guid: Guid of the StorageRouter to update
:type storagerouter_guid: str
:param timestamp: Timestamp to compare to
:type timestamp: float
:return: None
:rtype: NoneType
"""
with volatile_mutex(
'storagerouter_heartbeat_{0}'.format(storagerouter_guid)):
storagerouter = StorageRouter(storagerouter_guid)
if timestamp > storagerouter.heartbeats.get('celery', 0):
storagerouter.heartbeats['celery'] = timestamp
storagerouter.save()
@staticmethod
@ovs_task(name='ovs.storagerouter.get_metadata')
def get_metadata(storagerouter_guid):
"""
Gets physical information about the specified StorageRouter
:param storagerouter_guid: StorageRouter guid to retrieve the metadata for
:type storagerouter_guid: str
:return: Metadata information about the StorageRouter
:rtype: dict
"""
return {
'partitions':
StorageRouterController.get_partition_info(storagerouter_guid),
'ipaddresses':
StorageRouterController.get_ip_addresses(storagerouter_guid),
'scrub_available':
StorageRouterController.check_scrub_partition_present()
}
@staticmethod
def get_ip_addresses(storagerouter_guid):
"""
Retrieves the IP addresses of a StorageRouter
:param storagerouter_guid: Guid of the StorageRouter
:return: List of IP addresses
:rtype: list
"""
client = SSHClient(endpoint=StorageRouter(storagerouter_guid))
return StorageRouterController._os_manager.get_ip_addresses(
client=client)
@staticmethod
def get_partition_info(storagerouter_guid):
"""
Retrieves information about the partitions of a Storagerouter
:param storagerouter_guid: Guid of the Storagerouter
:type storagerouter_guid: str
:return: dict with information about the partitions
:rtype: dict
"""
storagerouter = StorageRouter(storagerouter_guid)
client = SSHClient(endpoint=storagerouter)
services_mds = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.MD_SERVER).services
services_arakoon = [
service for service in ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON).services
if service.name != 'arakoon-ovsdb' and service.is_internal is True
]
partitions = dict((role, []) for role in DiskPartition.ROLES)
for disk in storagerouter.disks:
for disk_partition in disk.partitions:
claimed_space_by_fwk = 0
used_space_by_system = 0
available_space_by_system = 0
for storagedriver_partition in disk_partition.storagedrivers:
claimed_space_by_fwk += storagedriver_partition.size if storagedriver_partition.size is not None else 0
if client.dir_exists(storagedriver_partition.path):
try:
used_space_by_system += int(
client.run([
'du', '-B', '1', '-d', '0',
storagedriver_partition.path
],
timeout=5).split('\t')[0])
except Exception as ex:
StorageRouterController._logger.warning(
'Failed to get directory usage for {0}. {1}'.
format(storagedriver_partition.path, ex))
if disk_partition.mountpoint is not None:
for alias in disk_partition.aliases:
StorageRouterController._logger.info(
'Verifying disk partition usage by checking path {0}'
.format(alias))
disk_partition_device = client.file_read_link(
path=alias)
try:
available_space_by_system = int(
client.run([
'df', '-B', '1', '--output=avail',
disk_partition_device
],
timeout=5).splitlines()[-1])
break
except Exception as ex:
StorageRouterController._logger.warning(
'Failed to get partition usage for {0}. {1}'.
format(disk_partition.mountpoint, ex))
for role in disk_partition.roles:
size = 0 if disk_partition.size is None else disk_partition.size
if available_space_by_system > 0:
# Take available space reported by df then add back used by roles so that the only used space reported is space not managed by us
available = available_space_by_system + used_space_by_system - claimed_space_by_fwk
else:
available = size - claimed_space_by_fwk # Subtract size for roles which have already been claimed by other vpools (but not necessarily already been fully used)
in_use = any(junction
for junction in disk_partition.storagedrivers
if junction.role == role)
if role == DiskPartition.ROLES.DB:
for service in services_arakoon:
if service.storagerouter_guid == storagerouter_guid:
in_use = True
break
for service in services_mds:
if service.storagerouter_guid == storagerouter_guid:
in_use = True
break
partitions[role].append({
'ssd':
disk.is_ssd,
'guid':
disk_partition.guid,
'size':
size,
'in_use':
in_use,
'usable':
True, # Sizes smaller than 1GiB and smaller than 5% of largest WRITE partition will be un-usable
'available':
available if available > 0 else 0,
'mountpoint':
disk_partition.
folder, # Equals to mount point unless mount point is root ('/'), then we pre-pend mount point with '/mnt/storage'
'storagerouter_guid':
storagerouter_guid
})
# Strip out WRITE caches which are smaller than 5% of largest write cache size and smaller than 1GiB
writecache_sizes = []
for partition_info in partitions[DiskPartition.ROLES.WRITE]:
writecache_sizes.append(partition_info['available'])
largest_write_cache = max(
writecache_sizes) if len(writecache_sizes) > 0 else 0
for index, size in enumerate(writecache_sizes):
if size < largest_write_cache * 5 / 100 or size < 1024**3:
partitions[DiskPartition.ROLES.WRITE][index]['usable'] = False
return partitions
@staticmethod
@ovs_task(name='ovs.storagerouter.get_version_info')
def get_version_info(storagerouter_guid):
"""
Returns version information regarding a given StorageRouter
:param storagerouter_guid: StorageRouter guid to get version information for
:type storagerouter_guid: str
:return: Version information
:rtype: dict
"""
package_manager = PackageFactory.get_manager()
client = SSHClient(StorageRouter(storagerouter_guid))
return {
'storagerouter_guid':
storagerouter_guid,
'versions':
dict((pkg_name, str(version)) for pkg_name, version in
package_manager.get_installed_versions(client).iteritems())
}
@staticmethod
@ovs_task(name='ovs.storagerouter.get_support_info')
def get_support_info():
"""
Returns support information for the entire cluster
:return: Support information
:rtype: dict
"""
celery_scheduling = Configuration.get(
key='/ovs/framework/scheduling/celery', default={})
stats_monkey_enabled = any(
celery_scheduling.get(key) is not None for key in
['ovs.stats_monkey.run_all', 'alba.stats_monkey.run_all'])
return {
'cluster_id':
Configuration.get(key='/ovs/framework/cluster_id'),
'stats_monkey':
stats_monkey_enabled,
'support_agent':
Configuration.get(key='/ovs/framework/support|support_agent'),
'remote_access':
Configuration.get(key='ovs/framework/support|remote_access'),
'stats_monkey_config':
Configuration.get(key='ovs/framework/monitoring/stats_monkey',
default={})
}
@staticmethod
@ovs_task(name='ovs.storagerouter.get_support_metadata')
def get_support_metadata():
"""
Returns support metadata for a given StorageRouter. This should be a routed task!
:return: Metadata of the StorageRouter
:rtype: dict
"""
return SupportAgent().get_heartbeat_data()
@staticmethod
@ovs_task(name='ovs.storagerouter.get_logfiles')
def get_logfiles(local_storagerouter_guid):
"""
Collects logs, moves them to a web-accessible location and returns log tgz's filename
:param local_storagerouter_guid: StorageRouter guid to retrieve log files on
:type local_storagerouter_guid: str
:return: Name of tgz containing the logs
:rtype: str
"""
this_storagerouter = System.get_my_storagerouter()
this_client = SSHClient(this_storagerouter, username='******')
logfile = this_client.run(['ovs', 'collect', 'logs']).strip()
logfilename = logfile.split('/')[-1]
storagerouter = StorageRouter(local_storagerouter_guid)
webpath = '/opt/OpenvStorage/webapps/frontend/downloads'
client = SSHClient(storagerouter, username='******')
client.dir_create(webpath)
client.file_upload('{0}/{1}'.format(webpath, logfilename), logfile)
client.run(['chmod', '666', '{0}/{1}'.format(webpath, logfilename)])
return logfilename
@staticmethod
@ovs_task(name='ovs.storagerouter.get_proxy_config')
def get_proxy_config(vpool_guid, storagerouter_guid):
"""
Gets the ALBA proxy for a given StorageRouter and vPool
:param storagerouter_guid: Guid of the StorageRouter on which the ALBA proxy is configured
:type storagerouter_guid: str
:param vpool_guid: Guid of the vPool for which the proxy is configured
:type vpool_guid: str
:return: The ALBA proxy configuration
:rtype: dict
"""
vpool = VPool(vpool_guid)
storagerouter = StorageRouter(storagerouter_guid)
for sd in vpool.storagedrivers:
if sd.storagerouter_guid == storagerouter.guid:
if len(sd.alba_proxies) == 0:
raise ValueError(
'No ALBA proxies configured for vPool {0} on StorageRouter {1}'
.format(vpool.name, storagerouter.name))
return Configuration.get(
'/ovs/vpools/{0}/proxies/{1}/config/main'.format(
vpool.guid, sd.alba_proxies[0].guid))
raise ValueError(
'vPool {0} has not been extended to StorageRouter {1}'.format(
vpool.name, storagerouter.name))
@staticmethod
@ovs_task(name='ovs.storagerouter.configure_support')
def configure_support(support_info):
"""
Configures support on all StorageRouters
:param support_info: Information about which components should be configured
{'stats_monkey': True, # Enable/disable the stats monkey scheduled task
'support_agent': True, # Responsible for enabling the ovs-support-agent service, which collects heart beat data
'remote_access': False, # Cannot be True when support agent is False. Is responsible for opening an OpenVPN tunnel to allow for remote access
'stats_monkey_config': {}} # Dict with information on how to configure the stats monkey (Only required when enabling the stats monkey
:type support_info: dict
:return: None
:rtype: NoneType
"""
ExtensionsToolbox.verify_required_params(actual_params=support_info,
required_params={
'stats_monkey':
(bool, None, False),
'remote_access':
(bool, None, False),
'support_agent':
(bool, None, False),
'stats_monkey_config':
(dict, None, False)
})
# All settings are optional, so if nothing is specified, no need to change anything
if len(support_info) == 0:
StorageRouterController._logger.warning(
'Configure support called without any specific settings. Doing nothing'
)
return
# Collect information
support_agent_key = '/ovs/framework/support|support_agent'
support_agent_new = support_info.get('support_agent')
support_agent_old = Configuration.get(key=support_agent_key)
support_agent_change = support_agent_new is not None and support_agent_old != support_agent_new
remote_access_key = '/ovs/framework/support|remote_access'
remote_access_new = support_info.get('remote_access')
remote_access_old = Configuration.get(key=remote_access_key)
remote_access_change = remote_access_new is not None and remote_access_old != remote_access_new
stats_monkey_celery_key = '/ovs/framework/scheduling/celery'
stats_monkey_config_key = '/ovs/framework/monitoring/stats_monkey'
stats_monkey_new_config = support_info.get('stats_monkey_config')
stats_monkey_old_config = Configuration.get(
key=stats_monkey_config_key, default={})
stats_monkey_celery_config = Configuration.get(
key=stats_monkey_celery_key, default={})
stats_monkey_new = support_info.get('stats_monkey')
stats_monkey_old = stats_monkey_celery_config.get(
'ovs.stats_monkey.run_all'
) is not None or stats_monkey_celery_config.get(
'alba.stats_monkey.run_all') is not None
stats_monkey_change = stats_monkey_new is not None and (
stats_monkey_old != stats_monkey_new
or stats_monkey_new_config != stats_monkey_old_config)
# Make sure support agent is enabled when trying to enable remote access
if remote_access_new is True:
if support_agent_new is False or (support_agent_new is None
and support_agent_old is False):
raise RuntimeError(
'Remote access cannot be enabled without the heart beat enabled'
)
# Collect root_client information
root_clients = {}
for storagerouter in StorageRouterList.get_storagerouters():
try:
root_clients[storagerouter] = SSHClient(endpoint=storagerouter,
username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
if stats_monkey_new is True:
ExtensionsToolbox.verify_required_params(
actual_params=stats_monkey_new_config,
required_params={
'host': (str, ExtensionsToolbox.regex_ip),
'port': (int, {
'min': 1,
'max': 65535
}),
'database': (str, None),
'interval': (int, {
'min': 1,
'max': 86400
}),
'transport': (str, ['influxdb', 'redis', 'graphite']),
'environment': (str, None)
})
if stats_monkey_new_config['transport'] in ['influxdb', 'reddis']:
ExtensionsToolbox.verify_required_params(
actual_params=stats_monkey_new_config,
required_params={'password': (str, None)})
if stats_monkey_new_config['transport'] == 'influxdb':
ExtensionsToolbox.verify_required_params(
actual_params=stats_monkey_new_config,
required_params={'username': (str, None)})
# Configure remote access
if remote_access_change is True:
Configuration.set(key=remote_access_key, value=remote_access_new)
cid = Configuration.get('/ovs/framework/cluster_id').replace(
r"'", r"'\''")
for storagerouter, root_client in root_clients.iteritems():
if remote_access_new is False:
StorageRouterController._logger.info(
'Un-configuring remote access on StorageRouter {0}'.
format(root_client.ip))
nid = storagerouter.machine_id.replace(r"'", r"'\''")
service_name = 'openvpn@ovs_{0}-{1}'.format(cid, nid)
if StorageRouterController._service_manager.has_service(
name=service_name, client=root_client):
StorageRouterController._service_manager.stop_service(
name=service_name, client=root_client)
root_client.file_delete(filenames=['/etc/openvpn/ovs_*'])
# Configure support agent
if support_agent_change is True:
service_name = 'support-agent'
Configuration.set(key=support_agent_key, value=support_agent_new)
for root_client in root_clients.itervalues():
if support_agent_new is True:
StorageRouterController._logger.info(
'Configuring support agent on StorageRouter {0}'.
format(root_client.ip))
if StorageRouterController._service_manager.has_service(
name=service_name, client=root_client) is False:
StorageRouterController._service_manager.add_service(
name=service_name, client=root_client)
StorageRouterController._service_manager.restart_service(
name=service_name, client=root_client)
else:
StorageRouterController._logger.info(
'Un-configuring support agent on StorageRouter {0}'.
format(root_client.ip))
if StorageRouterController._service_manager.has_service(
name=service_name, client=root_client):
StorageRouterController._service_manager.stop_service(
name=service_name, client=root_client)
StorageRouterController._service_manager.remove_service(
name=service_name, client=root_client)
# Configure stats monkey
if stats_monkey_change is True:
# 2 keys matter here:
# - /ovs/framework/scheduling/celery --> used to check whether the stats monkey is disabled or not
# - /ovs/framework/monitoring/stats_monkey --> contains the actual configuration parameters when enabling the stats monkey, such as host, port, username, ...
service_name = 'scheduled-tasks'
if stats_monkey_new is True: # Enable the scheduled task by removing the key
StorageRouterController._logger.info(
'Configuring stats monkey')
interval = stats_monkey_new_config['interval']
# The scheduled task cannot be configured to run more than once a minute, so for intervals < 60, the stats monkey task handles this itself
StorageRouterController._logger.debug(
'Requested interval to run at: {0}'.format(interval))
Configuration.set(key=stats_monkey_config_key,
value=stats_monkey_new_config)
if interval > 0:
days, hours, minutes, _ = ExtensionsToolbox.convert_to_days_hours_minutes_seconds(
seconds=interval)
if days == 1: # Max interval is 24 * 60 * 60, so once every day at 3 AM
schedule = {'hour': '3'}
elif hours > 0:
schedule = {'hour': '*/{0}'.format(hours)}
else:
schedule = {'minute': '*/{0}'.format(minutes)}
stats_monkey_celery_config[
'ovs.stats_monkey.run_all'] = schedule
stats_monkey_celery_config[
'alba.stats_monkey.run_all'] = schedule
StorageRouterController._logger.debug(
'Configured schedule is: {0}'.format(schedule))
else:
stats_monkey_celery_config.pop('ovs.stats_monkey.run_all',
None)
stats_monkey_celery_config.pop('alba.stats_monkey.run_all',
None)
else: # Disable the scheduled task by setting the values for the celery tasks to None
StorageRouterController._logger.info(
'Un-configuring stats monkey')
stats_monkey_celery_config['ovs.stats_monkey.run_all'] = None
stats_monkey_celery_config['alba.stats_monkey.run_all'] = None
Configuration.set(key=stats_monkey_celery_key,
value=stats_monkey_celery_config)
for storagerouter in StorageRouterList.get_masters():
root_client = root_clients[storagerouter]
StorageRouterController._logger.debug(
'Restarting ovs-scheduled-tasks service on node with IP {0}'
.format(root_client.ip))
StorageRouterController._service_manager.restart_service(
name=service_name, client=root_client)
@staticmethod
@ovs_task(name='ovs.storagerouter.mountpoint_exists')
def mountpoint_exists(name, storagerouter_guid):
"""
Checks whether a given mount point for a vPool exists
:param name: Name of the mount point to check
:type name: str
:param storagerouter_guid: Guid of the StorageRouter on which to check for mount point existence
:type storagerouter_guid: str
:return: True if mount point not in use else False
:rtype: bool
"""
client = SSHClient(StorageRouter(storagerouter_guid))
return client.dir_exists(directory='/mnt/{0}'.format(name))
@staticmethod
@ovs_task(name='ovs.storagerouter.refresh_hardware')
def refresh_hardware(storagerouter_guid):
"""
Refreshes all hardware related information
:param storagerouter_guid: Guid of the StorageRouter to refresh the hardware on
:type storagerouter_guid: str
:return: None
:rtype: NoneType
"""
StorageRouterController.set_rdma_capability(storagerouter_guid)
DiskController.sync_with_reality(storagerouter_guid)
@staticmethod
def set_rdma_capability(storagerouter_guid):
"""
Check if the StorageRouter has been reconfigured to be able to support RDMA
:param storagerouter_guid: Guid of the StorageRouter to check and set
:type storagerouter_guid: str
:return: None
:rtype: NoneType
"""
storagerouter = StorageRouter(storagerouter_guid)
client = SSHClient(storagerouter, username='******')
rdma_capable = False
with remote(client.ip, [os], username='******') as rem:
for root, dirs, files in rem.os.walk('/sys/class/infiniband'):
for directory in dirs:
ports_dir = '/'.join([root, directory, 'ports'])
if not rem.os.path.exists(ports_dir):
continue
for sub_root, sub_dirs, _ in rem.os.walk(ports_dir):
if sub_root != ports_dir:
continue
for sub_directory in sub_dirs:
state_file = '/'.join(
[sub_root, sub_directory, 'state'])
if rem.os.path.exists(state_file):
if 'ACTIVE' in client.run(['cat', state_file]):
rdma_capable = True
storagerouter.rdma_capable = rdma_capable
storagerouter.save()
@staticmethod
@ovs_task(name='ovs.storagerouter.configure_disk',
ensure_single_info={
'mode': 'CHAINED',
'global_timeout': 1800
})
def configure_disk(storagerouter_guid, disk_guid, partition_guid, offset,
size, roles):
"""
Configures a partition
:param storagerouter_guid: Guid of the StorageRouter to configure a disk on
:type storagerouter_guid: str
:param disk_guid: Guid of the disk to configure
:type disk_guid: str
:param partition_guid: Guid of the partition on the disk
:type partition_guid: str
:param offset: Offset for the partition
:type offset: int
:param size: Size of the partition
:type size: int
:param roles: Roles assigned to the partition
:type roles: list
:return: None
:rtype: NoneType
"""
# Validations
storagerouter = StorageRouter(storagerouter_guid)
for role in roles:
if role not in DiskPartition.ROLES or role == DiskPartition.ROLES.BACKEND:
raise RuntimeError('Invalid role specified: {0}'.format(role))
disk = Disk(disk_guid)
if disk.storagerouter_guid != storagerouter_guid:
raise RuntimeError(
'The given Disk is not on the given StorageRouter')
for partition in disk.partitions:
if DiskPartition.ROLES.BACKEND in partition.roles:
raise RuntimeError('The given Disk is in use by a Backend')
if len({DiskPartition.ROLES.DB, DiskPartition.ROLES.DTL}.intersection(
set(roles))) > 0:
roles_on_sr = StorageRouterController._get_roles_on_storagerouter(
storagerouter.ip)
for role in [DiskPartition.ROLES.DB, DiskPartition.ROLES.DTL]:
if role in roles_on_sr and role in roles and roles_on_sr[role][
0] != disk.name: # DB and DTL roles still have to be unassignable
raise RoleDuplicationException(
'Disk {0} cannot have the {1} role due to presence on disk {2}'
.format(disk.name, role, roles_on_sr[role][0]))
# Create partition
if partition_guid is None:
StorageRouterController._logger.debug(
'Creating new partition - Offset: {0} bytes - Size: {1} bytes - Roles: {2}'
.format(offset, size, roles))
with remote(storagerouter.ip, [DiskTools], username='******') as rem:
if len(disk.aliases) == 0:
raise ValueError(
'Disk {0} does not have any aliases'.format(disk.name))
rem.DiskTools.create_partition(disk_alias=disk.aliases[0],
disk_size=disk.size,
partition_start=offset,
partition_size=size)
DiskController.sync_with_reality(storagerouter_guid)
disk = Disk(disk_guid)
end_point = offset + size
partition = None
for part in disk.partitions:
if offset < part.offset + part.size and end_point > part.offset:
partition = part
break
if partition is None:
raise RuntimeError(
'No new partition detected on disk {0} after having created 1'
.format(disk.name))
StorageRouterController._logger.debug('Partition created')
else:
StorageRouterController._logger.debug('Using existing partition')
partition = DiskPartition(partition_guid)
if partition.disk_guid != disk_guid:
raise RuntimeError(
'The given DiskPartition is not on the given Disk')
if partition.filesystem in [
'swap', 'linux_raid_member', 'LVM2_member'
]:
raise RuntimeError(
"It is not allowed to assign roles on partitions of type: ['swap', 'linux_raid_member', 'LVM2_member']"
)
metadata = StorageRouterController.get_metadata(storagerouter_guid)
partition_info = metadata['partitions']
removed_roles = set(partition.roles) - set(roles)
used_roles = []
for role in removed_roles:
for info in partition_info[role]:
if info['in_use'] and info['guid'] == partition.guid:
used_roles.append(role)
if len(used_roles) > 0:
raise RuntimeError(
'Roles in use cannot be removed. Used roles: {0}'.format(
', '.join(used_roles)))
# Add filesystem
if partition.filesystem is None or partition_guid is None:
StorageRouterController._logger.debug('Creating filesystem')
if len(partition.aliases) == 0:
raise ValueError(
'Partition with offset {0} does not have any aliases'.
format(partition.offset))
with remote(storagerouter.ip, [DiskTools], username='******') as rem:
rem.DiskTools.make_fs(partition_alias=partition.aliases[0])
DiskController.sync_with_reality(storagerouter_guid)
partition = DiskPartition(partition.guid)
if partition.filesystem not in ['ext4', 'xfs']:
raise RuntimeError('Unexpected filesystem')
StorageRouterController._logger.debug('Filesystem created')
# Mount the partition and add to FSTab
if partition.mountpoint is None:
StorageRouterController._logger.debug('Configuring mount point')
with remote(storagerouter.ip, [DiskTools], username='******') as rem:
counter = 1
mountpoint = '/mnt/{0}{1}'.format(
'ssd' if disk.is_ssd else 'hdd', counter)
while True:
if not rem.DiskTools.mountpoint_exists(mountpoint):
break
counter += 1
mountpoint = '/mnt/{0}{1}'.format(
'ssd' if disk.is_ssd else 'hdd', counter)
StorageRouterController._logger.debug(
'Found mount point: {0}'.format(mountpoint))
rem.DiskTools.add_fstab(partition_aliases=partition.aliases,
mountpoint=mountpoint,
filesystem=partition.filesystem)
rem.DiskTools.mount(mountpoint)
DiskController.sync_with_reality(storagerouter_guid)
partition = DiskPartition(partition.guid)
if partition.mountpoint != mountpoint:
raise RuntimeError('Unexpected mount point')
StorageRouterController._logger.debug('Mount point configured')
partition.roles = roles
partition.save()
StorageRouterController._logger.debug('Partition configured')
@staticmethod
def check_scrub_partition_present():
"""
Checks whether at least 1 scrub partition is present on any StorageRouter
:return: True if at least 1 SCRUB role present in the cluster else False
:rtype: bool
"""
for storage_router in StorageRouterList.get_storagerouters():
for disk in storage_router.disks:
for partition in disk.partitions:
if DiskPartition.ROLES.SCRUB in partition.roles:
return True
return False
@staticmethod
def get_mountpoints(client):
"""
Retrieve the mount points
:param client: SSHClient to retrieve the mount points on
:return: List of mount points
:rtype: list[str]
"""
mountpoints = []
for mountpoint in client.run(['mount', '-v']).strip().splitlines():
mp = mountpoint.split(' ')[2] if len(
mountpoint.split(' ')) > 2 else None
if mp and not mp.startswith('/dev') and not mp.startswith(
'/proc') and not mp.startswith(
'/sys') and not mp.startswith(
'/run') and not mp.startswith(
'/mnt/alba-asd') and mp != '/':
mountpoints.append(mp)
return mountpoints
@staticmethod
def _retrieve_alba_arakoon_config(alba_backend_guid, ovs_client):
"""
Retrieve the ALBA Arakoon configuration
:param alba_backend_guid: Guid of the ALBA Backend
:type alba_backend_guid: str
:param ovs_client: OVS client object
:type ovs_client: OVSClient
:return: Arakoon configuration information
:rtype: dict
"""
task_id = ovs_client.get(
'/alba/backends/{0}/get_config_metadata'.format(alba_backend_guid))
successful, arakoon_config = ovs_client.wait_for_task(task_id,
timeout=300)
if successful is False:
raise RuntimeError(
'Could not load metadata from environment {0}'.format(
ovs_client.ip))
return arakoon_config
@staticmethod
def _revert_vpool_status(vpool,
status=VPool.STATUSES.RUNNING,
storagedriver=None,
client=None,
dirs_created=None):
"""
Remove the vPool being created or revert the vPool being extended
:return: None
:rtype: NoneType
"""
vpool.status = status
vpool.save()
if status == VPool.STATUSES.RUNNING:
if len(dirs_created) > 0:
try:
client.dir_delete(directories=dirs_created)
except Exception:
StorageRouterController._logger.warning(
'Failed to clean up following directories: {0}'.format(
', '.join(dirs_created)))
if storagedriver is not None:
for sdp in storagedriver.partitions:
sdp.delete()
for proxy in storagedriver.alba_proxies:
proxy.delete()
storagedriver.delete()
if len(vpool.storagedrivers) == 0:
vpool.delete()
if Configuration.dir_exists(
key='/ovs/vpools/{0}'.format(vpool.guid)):
Configuration.delete(
key='/ovs/vpools/{0}'.format(vpool.guid))
@staticmethod
def _get_roles_on_storagerouter(ip):
"""
returns a set with the roles present on the storagerouter
:param ip: string with ip of the storagerouter
:return: Dict
"""
sr = StorageRouterList.get_by_ip(ip)
roles_on_sr = {}
for sr_disk in sr.disks:
for partition in sr_disk.partitions:
for part_role in partition.roles:
if part_role not in roles_on_sr:
roles_on_sr[part_role] = [sr_disk.name]
else:
roles_on_sr[part_role].append(sr_disk.name)
return roles_on_sr
class StorageDriverClient(object):
"""
Client to access storagedriver client
"""
_log_level = LOG_LEVEL_MAPPING[OVSLogger('extensions').getEffectiveLevel()]
# noinspection PyCallByClass,PyTypeChecker
storagerouterclient.Logger.setupLogging(
OVSLogger.load_path('storagerouterclient'), _log_level)
# noinspection PyArgumentList
storagerouterclient.Logger.enableLogging()
VOLDRV_DTL_SYNC = 'Synchronous'
VOLDRV_DTL_ASYNC = 'Asynchronous'
VOLDRV_DTL_MANUAL_MODE = 'Manual'
VOLDRV_DTL_AUTOMATIC_MODE = 'Automatic'
VOLDRV_DTL_TRANSPORT_TCP = 'TCP'
VOLDRV_DTL_TRANSPORT_RSOCKET = 'RSocket'
FRAMEWORK_DTL_SYNC = 'sync'
FRAMEWORK_DTL_ASYNC = 'a_sync'
FRAMEWORK_DTL_NO_SYNC = 'no_sync'
FRAMEWORK_DTL_TRANSPORT_TCP = 'tcp'
FRAMEWORK_DTL_TRANSPORT_RSOCKET = 'rdma'
VDISK_DTL_MODE_MAP = {
FRAMEWORK_DTL_SYNC: DTLMode.SYNCHRONOUS,
FRAMEWORK_DTL_ASYNC: DTLMode.ASYNCHRONOUS,
FRAMEWORK_DTL_NO_SYNC: None
}
VPOOL_DTL_MODE_MAP = {
FRAMEWORK_DTL_SYNC: VOLDRV_DTL_SYNC,
FRAMEWORK_DTL_ASYNC: VOLDRV_DTL_ASYNC,
FRAMEWORK_DTL_NO_SYNC: None
}
VPOOL_DTL_TRANSPORT_MAP = {
FRAMEWORK_DTL_TRANSPORT_TCP: VOLDRV_DTL_TRANSPORT_TCP,
FRAMEWORK_DTL_TRANSPORT_RSOCKET: VOLDRV_DTL_TRANSPORT_RSOCKET
}
REVERSE_DTL_MODE_MAP = {
VOLDRV_DTL_SYNC: FRAMEWORK_DTL_SYNC,
VOLDRV_DTL_ASYNC: FRAMEWORK_DTL_ASYNC,
DTLMode.SYNCHRONOUS: FRAMEWORK_DTL_SYNC,
DTLMode.ASYNCHRONOUS: FRAMEWORK_DTL_ASYNC
}
REVERSE_DTL_TRANSPORT_MAP = {
VOLDRV_DTL_TRANSPORT_TCP: FRAMEWORK_DTL_TRANSPORT_TCP,
VOLDRV_DTL_TRANSPORT_RSOCKET: FRAMEWORK_DTL_TRANSPORT_RSOCKET
}
CLUSTER_SIZES = [4, 8, 16, 32, 64]
TLOG_MULTIPLIER_MAP = {4: 16, 8: 8, 16: 4, 32: 2, 64: 1, 128: 1}
DTL_STATUS = {
'': 0,
'ok_sync': 10,
'ok_standalone': 20,
'catch_up': 30,
'checkup_required': 30,
'degraded': 40
}
EMPTY_STATISTICS = staticmethod(lambda: Statistics())
EMPTY_INFO = staticmethod(lambda: VolumeInfo())
STAT_SUMS = {
'operations': ['write_operations', 'read_operations'],
'cache_hits': ['sco_cache_hits', 'cluster_cache_hits'],
'cache_misses': ['sco_cache_misses'],
'4k_operations': ['4k_read_operations', '4k_write_operations'],
'4k_unaligned_operations':
['4k_unaligned_read_operations', '4k_unaligned_write_operations'],
'data_transferred': ['data_written', 'data_read']
}
def __init__(self):
"""
Dummy init method
"""
pass
@staticmethod
def load(vpool, excluded_storagedrivers=None):
"""
Initializes the wrapper for a given vpool
:param vpool: vPool for which the StorageRouterClient needs to be loaded
:type vpool: vPool
:param excluded_storagedrivers: A list of storagedrivers that cannot be used as a client
:type excluded_storagedrivers: list or None
"""
if excluded_storagedrivers is None:
excluded_storagedrivers = []
key = vpool.identifier
if key not in client_vpool_cache:
cluster_contacts = []
for storagedriver in vpool.storagedrivers[:3]:
if storagedriver not in excluded_storagedrivers:
cluster_contacts.append(
ClusterContact(str(storagedriver.cluster_ip),
storagedriver.ports['xmlrpc']))
client = StorageRouterClient(str(vpool.guid), cluster_contacts)
client_vpool_cache[key] = client
return client_vpool_cache[key]
