def _voldrv_arakoon_checkup(create_cluster):
def _add_service(service_storagerouter, arakoon_ports, service_name):
""" Add a service to the storage router """
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = arakoon_ports
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
current_ips = []
current_services = []
service_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
cluster_name = Configuration.get(
'/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is not None:
arakoon_service_name = ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)
for service in service_type.services:
if service.name == arakoon_service_name:
current_services.append(service)
if service.is_internal is True:
current_ips.append(service.storagerouter.ip)
all_sr_ips = [
storagerouter.ip
for storagerouter in StorageRouterList.get_slaves()
]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(
storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(
current_services) == 0: # Create new cluster
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
if metadata is None: # No externally managed cluster found, we create 1 ourselves
if not available_storagerouters:
raise RuntimeError(
'Could not find any Storage Router with a DB role')
storagerouter, partition = available_storagerouters.items()[0]
arakoon_voldrv_cluster = 'voldrv'
arakoon_installer = ArakoonInstaller(
cluster_name=arakoon_voldrv_cluster)
arakoon_installer.create_cluster(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouter.ip,
base_dir=partition.folder,
log_sinks=LogHandler.get_sink_path(
'arakoon-server_{0}'.format(arakoon_voldrv_cluster)),
crash_log_sinks=LogHandler.get_sink_path(
'arakoon-server-crash_{0}'.format(
arakoon_voldrv_cluster)))
arakoon_installer.start_cluster()
ports = arakoon_installer.ports[storagerouter.ip]
metadata = arakoon_installer.metadata
current_ips.append(storagerouter.ip)
else:
ports = []
storagerouter = None
cluster_name = metadata['cluster_name']
Configuration.set('/ovs/framework/arakoon_clusters|voldrv',
cluster_name)
StorageDriverController._logger.info(
'Claiming {0} managed arakoon cluster: {1}'.format(
'externally' if storagerouter is None else 'internally',
cluster_name))
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
current_services.append(
_add_service(
service_storagerouter=storagerouter,
arakoon_ports=ports,
service_name=ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)))
cluster_name = Configuration.get(
'/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is None:
return
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=cluster_name)
if 0 < len(current_services) < len(
available_storagerouters) and metadata['internal'] is True:
for storagerouter, partition in available_storagerouters.iteritems(
):
if storagerouter.ip in current_ips:
continue
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
arakoon_installer.load()
arakoon_installer.extend_cluster(
new_ip=storagerouter.ip,
base_dir=partition.folder,
log_sinks=LogHandler.get_sink_path(
'arakoon-server_{0}'.format(cluster_name)),
crash_log_sinks=LogHandler.get_sink_path(
'arakoon-server-crash_{0}'.format(cluster_name)))
_add_service(
service_storagerouter=storagerouter,
arakoon_ports=arakoon_installer.ports[storagerouter.ip],
service_name=ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name))
current_ips.append(storagerouter.ip)
arakoon_installer.restart_cluster_after_extending(
new_ip=storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
def ensure_nsm_clusters_load(cls,
alba_backend,
nsms_per_storagerouter=None,
min_internal_nsms=1,
external_nsm_cluster_names=None,
version_str=None,
ssh_clients=None):
# type: (AlbaBackend, Optional[Dict[StorageRouter, int]], Optional[int], Optional[List[str], Optional[str]], Optional[StorageRouter, SSHClient]) -> None
"""
Ensure that all NSM clusters are not overloaded
:param alba_backend: Alba Backend to ensure NSM Cluster load for
:type alba_backend: AlbaBackend
:param nsms_per_storagerouter: Amount of NSMs mapped by StorageRouter
:type nsms_per_storagerouter: Dict[StorageRouter, int]
:param min_internal_nsms: Minimum amount of NSM hosts that need to be provided
:type min_internal_nsms: int
:param external_nsm_cluster_names: Information about the additional clusters to claim (only for externally managed Arakoon clusters)
:type external_nsm_cluster_names: list
:param version_str: Alba version string
:type version_str: str
:param ssh_clients: SSHClients to use
:type ssh_clients: Dict[Storagerouter, SSHClient]
:return: None
:rtype: NoneType
"""
if ssh_clients is None:
ssh_clients = {}
if external_nsm_cluster_names is None:
external_nsm_cluster_names = []
nsms_per_storagerouter = nsms_per_storagerouter if nsms_per_storagerouter is not None else cls.get_nsms_per_storagerouter(
alba_backend)
version_str = version_str or AlbaArakoonInstaller.get_alba_version_string(
)
nsm_loads = cls.get_nsm_loads(alba_backend)
internal = AlbaArakoonInstaller.is_internally_managed(alba_backend)
abm_cluster_name = alba_backend.abm_cluster.name
safety = Configuration.get(
'/ovs/framework/plugins/alba/config|nsm.safety')
maxload = Configuration.get(
'/ovs/framework/plugins/alba/config|nsm.maxload')
overloaded = min(nsm_loads.values()) >= maxload
if not overloaded:
# At least 1 NSM is not overloaded yet
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - NSM load OK'.format(alba_backend.name))
if internal:
# When load is not OK, deploy at least 1 additional NSM
nsms_to_add = max(0, min_internal_nsms - len(nsm_loads))
else:
nsms_to_add = len(external_nsm_cluster_names)
if nsms_to_add == 0:
return
else:
AlbaArakoonController._logger.warning(
'ALBA Backend {0} - NSM load is NOT OK'.format(
alba_backend.name))
if internal:
# When load is not OK, deploy at least 1 additional NSM
nsms_to_add = max(1, min_internal_nsms - len(nsm_loads))
else:
# For externally managed clusters we only claim the specified clusters, if none provided, we just log it
nsms_to_add = len(external_nsm_cluster_names)
if nsms_to_add == 0:
cls._logger.critical(
'ALBA Backend {0} - All NSM clusters are overloaded'.
format(alba_backend.name))
return
# Deploy new (internal) or claim existing (external) NSM clusters
cls._logger.debug(
'ALBA Backend {0} - Currently {1} NSM cluster{2}'.format(
alba_backend.name, len(nsm_loads),
'' if len(nsm_loads) == 1 else 's'))
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Trying to add {1} NSM cluster{2}'.format(
alba_backend.name, nsms_to_add,
'' if nsms_to_add == 1 else 's'))
base_number = max(nsm_loads.keys()) + 1
for index, number in enumerate(
xrange(base_number, base_number + nsms_to_add)):
if not internal:
# External clusters
master_client = None
if not ssh_clients:
for storagerouter in StorageRouterList.get_masters():
try:
master_client = SSHClient(storagerouter)
except UnableToConnectException:
cls._logger.warning(
'StorageRouter {0} with IP {1} is not reachable'
.format(storagerouter.name, storagerouter.ip))
else:
for storagerouter, ssh_client in ssh_clients.iteritems():
if storagerouter.node_type == 'MASTER':
master_client = ssh_client
if not master_client:
raise ValueError('Could not find an online master node')
# @todo this might raise an indexerror?
nsm_cluster_name = external_nsm_cluster_names[index]
cls._logger.debug(
'ALBA Backend {0} - Claiming NSM cluster {1}'.format(
alba_backend.name, nsm_cluster_name))
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM,
cluster_name=nsm_cluster_name)
if metadata is None:
cls._logger.critical(
'ALBA Backend {0} - NSM cluster with name {1} could not be found'
.format(alba_backend.name, nsm_cluster_name))
continue
cls._logger.debug(
'ALBA Backend {0} - Modeling services'.format(
alba_backend.name))
AlbaArakoonInstaller.model_arakoon_service(
alba_backend=alba_backend,
cluster_name=nsm_cluster_name,
number=number)
cls._logger.debug('ALBA Backend {0} - Registering NSM'.format(
alba_backend.name))
NSMInstaller.register_nsm(abm_name=abm_cluster_name,
nsm_name=nsm_cluster_name,
ip=master_client.ip)
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Extended cluster'.format(
alba_backend.name))
else:
# Internal clusters
nsm_cluster_name = '{0}-nsm_{1}'.format(
alba_backend.name, number)
cls._logger.debug(
'ALBA Backend {0} - Adding NSM cluster {1}'.format(
alba_backend.name, nsm_cluster_name))
# One of the NSM nodes is overloaded. This means the complete NSM is considered overloaded
# Figure out which StorageRouters are the least occupied
loads = sorted(nsms_per_storagerouter.values())[:safety]
storagerouters = []
for storagerouter, load in nsms_per_storagerouter.iteritems():
if load in loads:
storagerouters.append(storagerouter)
if len(storagerouters) == safety:
break
# Creating a new NSM cluster
for sub_index, storagerouter in enumerate(storagerouters):
nsms_per_storagerouter[storagerouter] += 1
partition = AlbaArakoonInstaller.get_db_partition(
storagerouter)
arakoon_installer = ArakoonInstaller(
cluster_name=nsm_cluster_name)
# @todo Use deploy and extend code. (Disable register nsm in those parts)
if sub_index == 0:
arakoon_installer.create_cluster(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM,
ip=storagerouter.ip,
base_dir=partition.folder,
plugins={NSM_PLUGIN: version_str})
else:
cls._logger.debug(
'ALBA Backend {0} - Extending NSM cluster {1}'.
format(alba_backend.name, nsm_cluster_name))
arakoon_installer.load()
arakoon_installer.extend_cluster(
new_ip=storagerouter.ip,
base_dir=partition.folder,
plugins={NSM_PLUGIN: version_str})
cls._logger.debug(
'ALBA Backend {0} - Linking plugins'.format(
alba_backend.name))
ssh_client = ssh_clients.get(storagerouter) or SSHClient(
StorageRouter)
AlbaArakoonInstaller.link_plugins(
client=ssh_client,
data_dir=partition.folder,
plugins=[NSM_PLUGIN],
cluster_name=nsm_cluster_name)
cls._logger.debug(
'ALBA Backend {0} - Modeling services'.format(
alba_backend.name))
AlbaArakoonInstaller.model_arakoon_service(
alba_backend=alba_backend,
cluster_name=nsm_cluster_name,
ports=arakoon_installer.ports[storagerouter.ip],
storagerouter=storagerouter,
number=number)
if sub_index == 0:
cls._logger.debug(
'ALBA Backend {0} - Starting cluster'.format(
alba_backend.name))
arakoon_installer.start_cluster()
else:
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Restarting cluster'.format(
alba_backend.name))
arakoon_installer.restart_cluster_after_extending(
new_ip=storagerouter.ip)
cls._logger.debug('ALBA Backend {0} - Registering NSM'.format(
alba_backend.name))
NSMInstaller.register_nsm(abm_name=abm_cluster_name,
nsm_name=nsm_cluster_name,
ip=storagerouters[0].ip)
cls._logger.debug(
'ALBA Backend {0} - Added NSM cluster {1}'.format(
alba_backend.name, nsm_cluster_name))