def _get_arakoon_clusters(cls, result_handler):
"""
Retrieves all Arakoon clusters registered in this OVSCluster
:param result_handler: Logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:return: Dict with the Arakoon cluster types as key and list with dicts which contain cluster names and pyrakoon clients
:rtype: dict(str, list[dict])
"""
result_handler.info('Fetching available arakoon clusters.', add_to_result=False)
arakoon_clusters = {}
for cluster_name in list(Configuration.list('/ovs/arakoon')) + ['cacc']:
# Determine Arakoon type
is_cacc = cluster_name == 'cacc'
arakoon_config = ArakoonClusterConfig(cluster_id=cluster_name, load_config=not is_cacc)
if is_cacc is True:
with open(Configuration.CACC_LOCATION) as config_file:
contents = config_file.read()
arakoon_config.read_config(contents=contents)
try:
arakoon_client = ArakoonInstaller.build_client(arakoon_config)
except (ArakoonNoMaster, ArakoonNoMasterResult) as ex:
result_handler.failure('Unable to find a master for Arakoon cluster {0}. (Message: {1})'.format(cluster_name, str(ex)),
code=ErrorCodes.master_none)
except Exception as ex:
msg = 'Unable to connect to Arakoon cluster {0}. (Message: {1})'.format(cluster_name, str(ex))
result_handler.exception(msg, code=ErrorCodes.unhandled_exception)
cls.logger.exception(msg)
continue
metadata = json.loads(arakoon_client.get(ArakoonInstaller.METADATA_KEY))
cluster_type = metadata['cluster_type']
if cluster_type not in arakoon_clusters:
arakoon_clusters[cluster_type] = []
arakoon_clusters[cluster_type].append({'cluster_name': cluster_name, 'client': arakoon_client, 'config': arakoon_config})
return arakoon_clusters
def _get_clusters_residing_on_local_node(result_handler):
"""
Fetches the available local arakoon clusters of a cluster
:param result_handler: logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:return: dict with the arakoon info
:rtype: dict
"""
result_handler.info('Fetching available arakoon clusters.', add_to_result=False)
arakoon_clusters = list(Configuration.list('/ovs/arakoon'))
present_nodes = {}
missing_nodes = {}
missing_tlog = {}
cluster_results = {'present': present_nodes, 'missing': missing_nodes, 'tlog_missing': missing_tlog}
if len(arakoon_clusters) == 0:
# no arakoon clusters on node
result_handler.warning('No installed arakoon clusters detected on this system.')
return cluster_results
# add arakoon clusters
for cluster in arakoon_clusters:
arakoon_config = ArakoonClusterConfig(str(cluster))
master_node_ids = [node.name for node in arakoon_config.nodes]
if ArakoonHealthCheck.LOCAL_SR.machine_id not in master_node_ids:
continue
# add node that is available for arakoon cluster
nodes_per_cluster_result = {}
missing_nodes_per_cluster = {}
missing_tlog_per_cluster = {}
tlog_dir = arakoon_config.export_dict()[ArakoonHealthCheck.LOCAL_SR.machine_id]['tlog_dir']
for node_id in master_node_ids:
machine = StoragerouterHelper.get_by_machine_id(node_id)
if machine is None:
# No information found about the storagerouter - old value in arakoon
missing_nodes_per_cluster.update({node_id: tlog_dir})
result_handler.warning('Could not fetch storagerouter information about node {0} that was stored in Arakoon {1}'.format(node_id, cluster), add_to_result=False)
elif not tlog_dir:
missing_tlog_per_cluster.update({node_id: tlog_dir})
result_handler.warning('Arakoon {1} seems to have no tlog_dir on this node.'.format(node_id, cluster), add_to_result=False)
else:
nodes_per_cluster_result.update({node_id: tlog_dir})
present_nodes[cluster] = nodes_per_cluster_result
missing_nodes[cluster] = missing_nodes_per_cluster
missing_tlog[cluster] = missing_tlog_per_cluster
return cluster_results
def _configure_arakoon_to_volumedriver(cluster_name):
StorageDriverController._logger.info('Update existing vPools')
config = ArakoonClusterConfig(cluster_id=cluster_name)
arakoon_nodes = []
for node in config.nodes:
arakoon_nodes.append({
'host': node.ip,
'port': node.client_port,
'node_id': node.name
})
if Configuration.dir_exists('/ovs/vpools'):
for vpool_guid in Configuration.list('/ovs/vpools'):
for storagedriver_id in Configuration.list(
'/ovs/vpools/{0}/hosts'.format(vpool_guid)):
storagedriver_config = StorageDriverConfiguration(
'storagedriver', vpool_guid, storagedriver_id)
storagedriver_config.load()
storagedriver_config.configure_volume_registry(
vregistry_arakoon_cluster_id=cluster_name,
vregistry_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.configure_distributed_lock_store(
dls_type='Arakoon',
dls_arakoon_cluster_id=cluster_name,
dls_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.save()
def _get_nsm_state(abm):
state = {}
data = VirtualAlbaBackend.data[abm]
for nsm in data['nsms']:
config = ArakoonClusterConfig(cluster_id=nsm['id'])
state[nsm['id']] = [node.name for node in config.nodes]
return state
def load(vpool):
"""
Initializes the wrapper for a given vPool
:param vpool: vPool for which the FileSystemMetaDataClient needs to be loaded
"""
if FileSystemMetaDataClient is None:
raise FeatureNotAvailableException()
if os.environ.get('RUNNING_UNITTESTS') == 'True':
return FileSystemMetaDataClient(str(vpool.guid), None, None)
key = vpool.identifier
if key not in fsmclient_vpool_cache:
arakoon_cluster_name = str(
Configuration.get('/ovs/framework/arakoon_clusters|voldrv'))
config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name)
arakoon_node_configs = []
for node in config.nodes:
arakoon_node_configs.append(
ArakoonNodeConfig(str(node.name), str(node.ip),
node.client_port))
client = FileSystemMetaDataClient(str(vpool.guid),
arakoon_cluster_name,
arakoon_node_configs)
fsmclient_vpool_cache[key] = client
return fsmclient_vpool_cache[key]
def _get_arakoon_clusters(cls, result_handler):
"""
Retrieves all Arakoon clusters registered in this OVSCluster
:param result_handler: Logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:return: Dict with the Arakoon cluster types as key and list with dicts which contain cluster names and pyrakoon clients
:rtype: dict(str, list[dict])
"""
result_handler.info('Fetching available arakoon clusters.',
add_to_result=False)
arakoon_clusters = {}
for cluster_name in list(
Configuration.list('/ovs/arakoon')) + ['cacc']:
# Determine Arakoon type
is_cacc = cluster_name == 'cacc'
arakoon_config = ArakoonClusterConfig(cluster_id=cluster_name,
load_config=not is_cacc)
if is_cacc is True:
with open(Configuration.CACC_LOCATION) as config_file:
contents = config_file.read()
arakoon_config.read_config(contents=contents)
try:
arakoon_client = ArakoonInstaller.build_client(arakoon_config)
except (ArakoonNoMaster, ArakoonNoMasterResult) as ex:
result_handler.failure(
'Unable to find a master for Arakoon cluster {0}. (Message: {1})'
.format(cluster_name, str(ex)),
code=ErrorCodes.master_none)
except Exception as ex:
msg = 'Unable to connect to Arakoon cluster {0}. (Message: {1})'.format(
cluster_name, str(ex))
result_handler.exception(msg,
code=ErrorCodes.unhandled_exception)
cls.logger.exception(msg)
continue
metadata = json.loads(
arakoon_client.get(ArakoonInstaller.METADATA_KEY))
cluster_type = metadata['cluster_type']
if cluster_type not in arakoon_clusters:
arakoon_clusters[cluster_type] = []
arakoon_clusters[cluster_type].append({
'cluster_name': cluster_name,
'client': arakoon_client,
'config': arakoon_config
})
return arakoon_clusters
def collapse_arakoon():
"""
Collapse Arakoon's Tlogs
:return: None
"""
from ovs_extensions.generic.toolbox import ExtensionsToolbox
GenericController._logger.info('Arakoon collapse started')
cluster_info = []
storagerouters = StorageRouterList.get_storagerouters()
if os.environ.get('RUNNING_UNITTESTS') != 'True':
cluster_info = [('cacc', storagerouters[0])]
cluster_names = []
for service in ServiceList.get_services():
if service.is_internal is True and service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR):
cluster = ExtensionsToolbox.remove_prefix(service.name, 'arakoon-')
if cluster in cluster_names and cluster not in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST]:
continue
cluster_names.append(cluster)
cluster_info.append((cluster, service.storagerouter))
workload = {}
cluster_config_map = {}
for cluster, storagerouter in cluster_info:
GenericController._logger.debug(' Collecting info for cluster {0}'.format(cluster))
ip = storagerouter.ip if cluster in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST] else None
try:
config = ArakoonClusterConfig(cluster_id=cluster, source_ip=ip)
cluster_config_map[cluster] = config
except:
GenericController._logger.exception(' Retrieving cluster information on {0} for {1} failed'.format(storagerouter.ip, cluster))
continue
for node in config.nodes:
if node.ip not in workload:
workload[node.ip] = {'node_id': node.name,
'clusters': []}
workload[node.ip]['clusters'].append((cluster, ip))
for storagerouter in storagerouters:
try:
if storagerouter.ip not in workload:
continue
node_workload = workload[storagerouter.ip]
client = SSHClient(storagerouter)
for cluster, ip in node_workload['clusters']:
try:
GenericController._logger.debug(' Collapsing cluster {0} on {1}'.format(cluster, storagerouter.ip))
client.run(['arakoon', '--collapse-local', node_workload['node_id'], '2', '-config', cluster_config_map[cluster].external_config_path])
GenericController._logger.debug(' Collapsing cluster {0} on {1} completed'.format(cluster, storagerouter.ip))
except:
GenericController._logger.exception(' Collapsing cluster {0} on {1} failed'.format(cluster, storagerouter.ip))
except UnableToConnectException:
GenericController._logger.error(' Could not collapse any cluster on {0} (not reachable)'.format(storagerouter.name))
GenericController._logger.info('Arakoon collapse finished')
def load(vpool):
"""
Initializes the wrapper for a given vpool
:param vpool: vPool for which the ClusterRegistryClient needs to be loaded
"""
if os.environ.get('RUNNING_UNITTESTS') == 'True':
return ClusterRegistry(str(vpool.guid), None, None)
key = vpool.identifier
if key not in crclient_vpool_cache:
arakoon_cluster_name = str(
Configuration.get('/ovs/framework/arakoon_clusters|voldrv'))
config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name)
arakoon_node_configs = []
for node in config.nodes:
arakoon_node_configs.append(
ArakoonNodeConfig(str(node.name), str(node.ip),
node.client_port))
client = ClusterRegistry(str(vpool.guid), arakoon_cluster_name,
arakoon_node_configs)
crclient_vpool_cache[key] = client
return crclient_vpool_cache[key]
def promote_or_demote_node(node_action,
cluster_ip=None,
execute_rollback=False):
"""
Promotes or demotes the local node
:param node_action: Demote or promote
:type node_action: str
:param cluster_ip: IP of node to promote or demote
:type cluster_ip: str
:param execute_rollback: In case of failure revert the changes made
:type execute_rollback: bool
:return: None
"""
if node_action not in ('promote', 'demote'):
raise ValueError('Nodes can only be promoted or demoted')
Toolbox.log(logger=NodeTypeController._logger,
messages='Open vStorage Setup - {0}'.format(
node_action.capitalize()),
boxed=True)
try:
Toolbox.log(logger=NodeTypeController._logger,
messages='Collecting information',
title=True)
machine_id = System.get_my_machine_id()
if Configuration.get('/ovs/framework/hosts/{0}/setupcompleted'.
format(machine_id)) is False:
raise RuntimeError('No local OVS setup found.')
if cluster_ip and not re.match(Toolbox.regex_ip, cluster_ip):
raise RuntimeError(
'Incorrect IP provided ({0})'.format(cluster_ip))
if cluster_ip:
client = SSHClient(endpoint=cluster_ip)
machine_id = System.get_my_machine_id(client)
node_type = Configuration.get(
'/ovs/framework/hosts/{0}/type'.format(machine_id))
if node_action == 'promote' and node_type == 'MASTER':
raise RuntimeError('This node is already master.')
elif node_action == 'demote' and node_type == 'EXTRA':
raise RuntimeError('This node should be a master.')
elif node_type not in ['MASTER', 'EXTRA']:
raise RuntimeError('This node is not correctly configured.')
master_ip = None
offline_nodes = []
online = True
target_client = None
if node_action == 'demote' and cluster_ip: # Demote an offline node
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.lib.storagedriver import StorageDriverController
ip = cluster_ip
unique_id = None
ip_client_map = {}
for storage_router in StorageRouterList.get_storagerouters():
try:
client = SSHClient(storage_router.ip, username='******')
if storage_router.node_type == 'MASTER':
master_ip = storage_router.ip
ip_client_map[storage_router.ip] = client
except UnableToConnectException:
if storage_router.ip == cluster_ip:
online = False
unique_id = storage_router.machine_id
StorageDriverController.mark_offline(
storagerouter_guid=storage_router.guid)
offline_nodes.append(storage_router)
if online is True:
raise RuntimeError(
"If the node is online, please use 'ovs setup demote' executed on the node you wish to demote"
)
if master_ip is None:
raise RuntimeError(
'Failed to retrieve another responsive MASTER node')
else:
target_password = Toolbox.ask_validate_password(
ip='127.0.0.1', logger=NodeTypeController._logger)
target_client = SSHClient('127.0.0.1',
username='******',
password=target_password)
unique_id = System.get_my_machine_id(target_client)
ip = Configuration.get(
'/ovs/framework/hosts/{0}/ip'.format(unique_id))
storagerouter_info = NodeTypeController.retrieve_storagerouter_info_via_host(
ip=target_client.ip, password=target_password)
node_ips = [
sr_info['ip']
for sr_info in storagerouter_info.itervalues()
]
master_node_ips = [
sr_info['ip']
for sr_info in storagerouter_info.itervalues()
if sr_info['type'] == 'master' and sr_info['ip'] != ip
]
if len(master_node_ips) == 0:
if node_action == 'promote':
raise RuntimeError('No master node could be found')
else:
raise RuntimeError(
'It is not possible to remove the only master')
master_ip = master_node_ips[0]
ip_client_map = dict(
(node_ip, SSHClient(node_ip, username='******'))
for node_ip in node_ips)
if node_action == 'demote':
for cluster_name in Configuration.list('/ovs/arakoon'):
config = ArakoonClusterConfig(cluster_id=cluster_name)
arakoon_client = ArakoonInstaller.build_client(config)
metadata = json.loads(
arakoon_client.get(ArakoonInstaller.METADATA_KEY))
if len(config.nodes) == 1 and config.nodes[
0].ip == ip and metadata.get('internal') is True:
raise RuntimeError(
'Demote is not supported when single node Arakoon cluster(s) are present on the node to be demoted.'
)
configure_rabbitmq = Toolbox.is_service_internally_managed(
service='rabbitmq')
configure_memcached = Toolbox.is_service_internally_managed(
service='memcached')
if node_action == 'promote':
try:
NodeTypeController.promote_node(
cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
configure_memcached=configure_memcached,
configure_rabbitmq=configure_rabbitmq)
except Exception:
if execute_rollback is True:
NodeTypeController.demote_node(
cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
unconfigure_memcached=configure_memcached,
unconfigure_rabbitmq=configure_rabbitmq,
offline_nodes=offline_nodes)
elif target_client is not None:
target_client.file_write('/tmp/ovs_rollback', 'demote')
raise
else:
try:
NodeTypeController.demote_node(
cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
unconfigure_memcached=configure_memcached,
unconfigure_rabbitmq=configure_rabbitmq,
offline_nodes=offline_nodes)
except Exception:
if execute_rollback is True:
NodeTypeController.promote_node(
cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
configure_memcached=configure_memcached,
configure_rabbitmq=configure_rabbitmq)
elif target_client is not None:
target_client.file_write('/tmp/ovs_rollback',
'promote')
raise
Toolbox.log(logger=NodeTypeController._logger, messages='\n')
Toolbox.log(logger=NodeTypeController._logger,
messages='{0} complete.'.format(
node_action.capitalize()),
boxed=True)
except Exception as exception:
Toolbox.log(logger=NodeTypeController._logger, messages='\n')
Toolbox.log(
logger=NodeTypeController._logger,
messages=['An unexpected error occurred:',
str(exception)],
boxed=True,
loglevel='exception')
sys.exit(1)
except KeyboardInterrupt:
Toolbox.log(logger=NodeTypeController._logger, messages='\n')
Toolbox.log(
logger=NodeTypeController._logger,
messages=
'This setup was aborted. Open vStorage may be in an inconsistent state, make sure to validate the installation.',
boxed=True,
loglevel='error')
sys.exit(1)
def migrate(previous_version):
"""
Migrates from a given version to the current version. It uses 'previous_version' to be smart
wherever possible, but the code should be able to migrate any version towards the expected version.
When this is not possible, the code can set a minimum version and raise when it is not met.
:param previous_version: The previous version from which to start the migration
:type previous_version: float
"""
working_version = previous_version
if working_version == 0:
from ovs.dal.hybrids.servicetype import ServiceType
# Initial version:
# * Add any basic configuration or model entries
# Add backends
for backend_type_info in [('ALBA', 'alba')]:
code = backend_type_info[1]
backend_type = BackendTypeList.get_backend_type_by_code(code)
if backend_type is None:
backend_type = BackendType()
backend_type.name = backend_type_info[0]
backend_type.code = code
backend_type.save()
# Add service types
for service_type_info in [
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR,
ServiceType.SERVICE_TYPES.ALBA_S3_TRANSACTION
]:
service_type = ServiceType()
service_type.name = service_type_info
service_type.save()
# From here on, all actual migration should happen to get to the expected state for THIS RELEASE
elif working_version < DALMigrator.THIS_VERSION:
import hashlib
from ovs.dal.exceptions import ObjectNotFoundException
from ovs.dal.helpers import HybridRunner, Descriptor
from ovs.dal.hybrids.albaabmcluster import ABMCluster
from ovs.dal.hybrids.albaosd import AlbaOSD
from ovs.dal.hybrids.albansmcluster import NSMCluster
from ovs.dal.hybrids.j_abmservice import ABMService
from ovs.dal.hybrids.j_nsmservice import NSMService
from ovs.dal.hybrids.service import Service
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.lists.albabackendlist import AlbaBackendList
from ovs.dal.lists.albanodelist import AlbaNodeList
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.extensions.db.arakooninstaller import ArakoonClusterConfig, ArakoonInstaller
from ovs.extensions.generic.configuration import Configuration, NotFoundException
from ovs_extensions.generic.toolbox import ExtensionsToolbox
from ovs.extensions.plugins.albacli import AlbaCLI
from ovs.extensions.storage.persistentfactory import PersistentFactory
# Migrate unique constraints & indexes
client = PersistentFactory.get_client()
hybrid_structure = HybridRunner.get_hybrids()
for class_descriptor in hybrid_structure.values():
cls = Descriptor().load(class_descriptor).get_object()
classname = cls.__name__.lower()
unique_key = 'ovs_unique_{0}_{{0}}_'.format(classname)
index_prefix = 'ovs_index_{0}|{{0}}|'.format(classname)
index_key = 'ovs_index_{0}|{{0}}|{{1}}'.format(classname)
uniques = []
indexes = []
# noinspection PyProtectedMember
for prop in cls._properties:
if prop.unique is True and len([
k for k in client.prefix(
unique_key.format(prop.name))
]) == 0:
uniques.append(prop.name)
if prop.indexed is True and len([
k for k in client.prefix(
index_prefix.format(prop.name))
]) == 0:
indexes.append(prop.name)
if len(uniques) > 0 or len(indexes) > 0:
prefix = 'ovs_data_{0}_'.format(classname)
for key, data in client.prefix_entries(prefix):
for property_name in uniques:
ukey = '{0}{1}'.format(
unique_key.format(property_name),
hashlib.sha1(str(
data[property_name])).hexdigest())
client.set(ukey, key)
for property_name in indexes:
if property_name not in data:
continue # This is the case when there's a new indexed property added.
ikey = index_key.format(
property_name,
hashlib.sha1(str(
data[property_name])).hexdigest())
index = list(
client.get_multi([ikey], must_exist=False))[0]
transaction = client.begin_transaction()
if index is None:
client.assert_value(ikey,
None,
transaction=transaction)
client.set(ikey, [key],
transaction=transaction)
elif key not in index:
client.assert_value(ikey,
index[:],
transaction=transaction)
client.set(ikey,
index + [key],
transaction=transaction)
client.apply_transaction(transaction)
#############################################
# Introduction of ABMCluster and NSMCluster #
#############################################
# Verify presence of unchanged ALBA Backends
alba_backends = AlbaBackendList.get_albabackends()
changes_required = False
for alba_backend in alba_backends:
if alba_backend.abm_cluster is None or len(
alba_backend.nsm_clusters) == 0:
changes_required = True
break
if changes_required:
# Retrieve ABM and NSM clusters
abm_cluster_info = []
nsm_cluster_info = []
for cluster_name in Configuration.list('/ovs/arakoon'):
try:
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=cluster_name)
if metadata[
'cluster_type'] == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
abm_cluster_info.append(metadata)
elif metadata[
'cluster_type'] == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
nsm_cluster_info.append(metadata)
except NotFoundException:
continue
# Retrieve NSM Arakoon cluster information
cluster_arakoon_map = {}
for cluster_info in abm_cluster_info + nsm_cluster_info:
cluster_name = cluster_info['cluster_name']
arakoon_config = ArakoonClusterConfig(
cluster_id=cluster_name)
cluster_arakoon_map[
cluster_name] = arakoon_config.export_dict()
storagerouter_map = dict(
(storagerouter.machine_id, storagerouter) for storagerouter
in StorageRouterList.get_storagerouters())
alba_backend_id_map = dict((alba_backend.alba_id, alba_backend)
for alba_backend in alba_backends)
for cluster_info in abm_cluster_info:
internal = cluster_info['internal']
cluster_name = cluster_info['cluster_name']
config_location = Configuration.get_configuration_path(
key=ArakoonClusterConfig.CONFIG_KEY.format(
cluster_name))
try:
alba_id = AlbaCLI.run(command='get-alba-id',
config=config_location,
named_params={'attempts':
3})['id']
nsm_hosts = AlbaCLI.run(command='list-nsm-hosts',
config=config_location,
named_params={'attempts': 3})
except RuntimeError:
continue
alba_backend = alba_backend_id_map.get(alba_id)
if alba_backend is None: # ALBA Backend with ID not found in model
continue
if alba_backend.abm_cluster is not None and len(
alba_backend.nsm_clusters
) > 0: # Clusters already exist
continue
# Create ABM Cluster
if alba_backend.abm_cluster is None:
abm_cluster = ABMCluster()
abm_cluster.name = cluster_name
abm_cluster.alba_backend = alba_backend
abm_cluster.config_location = ArakoonClusterConfig.CONFIG_KEY.format(
cluster_name)
abm_cluster.save()
else:
abm_cluster = alba_backend.abm_cluster
# Create ABM Services
abm_arakoon_config = cluster_arakoon_map[cluster_name]
abm_arakoon_config.pop('global')
arakoon_nodes = abm_arakoon_config.keys()
if internal is False:
services_to_create = 1
else:
if set(arakoon_nodes).difference(
set(storagerouter_map.keys())):
continue
services_to_create = len(arakoon_nodes)
for index in range(services_to_create):
service = Service()
service.name = 'arakoon-{0}-abm'.format(
alba_backend.name)
service.type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ALBA_MGR)
if internal is True:
arakoon_node_config = abm_arakoon_config[
arakoon_nodes[index]]
service.ports = [
arakoon_node_config['client_port'],
arakoon_node_config['messaging_port']
]
service.storagerouter = storagerouter_map[
arakoon_nodes[index]]
else:
service.ports = []
service.storagerouter = None
service.save()
abm_service = ABMService()
abm_service.service = service
abm_service.abm_cluster = abm_cluster
abm_service.save()
# Create NSM Clusters
for cluster_index, nsm_host in enumerate(
sorted(nsm_hosts,
key=lambda host: ExtensionsToolbox.
advanced_sort(host['cluster_id'], '_'))):
nsm_cluster_name = nsm_host['cluster_id']
nsm_arakoon_config = cluster_arakoon_map.get(
nsm_cluster_name)
if nsm_arakoon_config is None:
continue
number = cluster_index if internal is False else int(
nsm_cluster_name.split('_')[-1])
nsm_cluster = NSMCluster()
nsm_cluster.name = nsm_cluster_name
nsm_cluster.number = number
nsm_cluster.alba_backend = alba_backend
nsm_cluster.config_location = ArakoonClusterConfig.CONFIG_KEY.format(
nsm_cluster_name)
nsm_cluster.save()
# Create NSM Services
nsm_arakoon_config.pop('global')
arakoon_nodes = nsm_arakoon_config.keys()
if internal is False:
services_to_create = 1
else:
if set(arakoon_nodes).difference(
set(storagerouter_map.keys())):
continue
services_to_create = len(arakoon_nodes)
for service_index in range(services_to_create):
service = Service()
service.name = 'arakoon-{0}-nsm_{1}'.format(
alba_backend.name, number)
service.type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.NS_MGR)
if internal is True:
arakoon_node_config = nsm_arakoon_config[
arakoon_nodes[service_index]]
service.ports = [
arakoon_node_config['client_port'],
arakoon_node_config['messaging_port']
]
service.storagerouter = storagerouter_map[
arakoon_nodes[service_index]]
else:
service.ports = []
service.storagerouter = None
service.save()
nsm_service = NSMService()
nsm_service.service = service
nsm_service.nsm_cluster = nsm_cluster
nsm_service.save()
# Clean up all junction services no longer linked to an ALBA Backend
all_nsm_services = [
service.nsm_service for service in ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.NS_MGR).services
if service.nsm_service.nsm_cluster is None
]
all_abm_services = [
service.abm_service for service in ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ALBA_MGR).services
if service.abm_service.abm_cluster is None
]
for abm_service in all_abm_services:
abm_service.delete()
abm_service.service.delete()
for nsm_service in all_nsm_services:
nsm_service.delete()
nsm_service.service.delete()
################################
# Introduction of Active Drive #
################################
# Update slot_id and Alba Node relation for all OSDs
client = PersistentFactory.get_client()
disk_osd_map = {}
for key, data in client.prefix_entries('ovs_data_albaosd_'):
alba_disk_guid = data.get('alba_disk', {}).get('guid')
if alba_disk_guid is not None:
if alba_disk_guid not in disk_osd_map:
disk_osd_map[alba_disk_guid] = []
disk_osd_map[alba_disk_guid].append(
key.replace('ovs_data_albaosd_', ''))
try:
value = client.get(key)
value.pop('alba_disk', None)
client.set(key=key, value=value)
except Exception:
pass # We don't care if we would have any leftover AlbaDisk information in _data, but its cleaner not to
alba_guid_node_map = dict(
(an.guid, an) for an in AlbaNodeList.get_albanodes())
for key, data in client.prefix_entries('ovs_data_albadisk_'):
alba_disk_guid = key.replace('ovs_data_albadisk_', '')
alba_node_guid = data.get('alba_node', {}).get('guid')
if alba_disk_guid in disk_osd_map and alba_node_guid in alba_guid_node_map and len(
data.get('aliases', [])) > 0:
slot_id = data['aliases'][0].split('/')[-1]
for osd_guid in disk_osd_map[alba_disk_guid]:
try:
osd = AlbaOSD(osd_guid)
except ObjectNotFoundException:
continue
osd.slot_id = slot_id
osd.alba_node = alba_guid_node_map[alba_node_guid]
osd.save()
client.delete(key=key, must_exist=False)
# Remove unique constraints for AlbaNode IP
for key in client.prefix('ovs_unique_albanode_ip_'):
client.delete(key=key, must_exist=False)
# Remove relation for all Alba Disks
for key in client.prefix('ovs_reverseindex_albadisk_'):
client.delete(key=key, must_exist=False)
# Remove the relation between AlbaNode and AlbaDisk
for key in client.prefix('ovs_reverseindex_albanode_'):
if '|disks|' in key:
client.delete(key=key, must_exist=False)
return DALMigrator.THIS_VERSION
def create_hprm_config_files(vpool_guid, local_storagerouter_guid,
parameters):
"""
Create the required configuration files to be able to make use of HPRM (aka PRACC)
This configuration will be zipped and made available for download
:param vpool_guid: The guid of the VPool for which a HPRM manager needs to be deployed
:type vpool_guid: str
:param local_storagerouter_guid: The guid of the StorageRouter the API was requested on
:type local_storagerouter_guid: str
:param parameters: Additional information required for the HPRM configuration files
:type parameters: dict
:return: Name of the zipfile containing the configuration files
:rtype: str
"""
# Validations
required_params = {
'port': (int, {
'min': 1,
'max': 65535
}),
'identifier': (str, ExtensionsToolbox.regex_vpool)
}
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
vpool = VPool(vpool_guid)
identifier = parameters['identifier']
config_path = None
local_storagerouter = StorageRouter(local_storagerouter_guid)
for sd in vpool.storagedrivers:
if len(sd.alba_proxies) == 0:
raise ValueError(
'No ALBA proxies configured for vPool {0} on StorageRouter {1}'
.format(vpool.name, sd.storagerouter.name))
config_path = '/ovs/vpools/{0}/proxies/{1}/config/{{0}}'.format(
vpool.guid, sd.alba_proxies[0].guid)
if config_path is None:
raise ValueError(
'vPool {0} has not been extended any StorageRouter'.format(
vpool.name))
proxy_cfg = Configuration.get(key=config_path.format('main'))
cache_info = {}
arakoons = {}
cache_types = VPool.CACHES.values()
if not any(ctype in parameters for ctype in cache_types):
raise ValueError(
'At least one cache type should be passed: {0}'.format(
', '.join(cache_types)))
for ctype in cache_types:
if ctype not in parameters:
continue
required_dict = {'read': (bool, None), 'write': (bool, None)}
required_params.update({ctype: (dict, required_dict)})
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
read = parameters[ctype]['read']
write = parameters[ctype]['write']
if read is False and write is False:
cache_info[ctype] = ['none']
continue
path = parameters[ctype].get('path')
if path is not None:
path = path.strip()
if not path or path.endswith(
'/.') or '..' in path or '/./' in path:
raise ValueError('Invalid path specified')
required_dict.update({
'path': (str, None),
'size': (int, {
'min': 1,
'max': 10 * 1024
})
})
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
while '//' in path:
path = path.replace('//', '/')
cache_info[ctype] = [
'local', {
'path': path,
'max_size': parameters[ctype]['size'] * 1024**3,
'cache_on_read': read,
'cache_on_write': write
}
]
else:
required_dict.update({
'backend_info': (dict, {
'preset': (str, ExtensionsToolbox.regex_preset),
'alba_backend_guid':
(str, ExtensionsToolbox.regex_guid),
'alba_backend_name':
(str, ExtensionsToolbox.regex_backend)
}),
'connection_info': (dict, {
'host': (str, ExtensionsToolbox.regex_ip, False),
'port': (int, {
'min': 1,
'max': 65535
}, False),
'client_id':
(str, ExtensionsToolbox.regex_guid, False),
'client_secret': (str, None, False)
})
})
ExtensionsToolbox.verify_required_params(
actual_params=parameters, required_params=required_params)
connection_info = parameters[ctype]['connection_info']
if connection_info[
'host']: # Remote Backend for accelerated Backend
alba_backend_guid = parameters[ctype]['backend_info'][
'alba_backend_guid']
ovs_client = OVSClient.get_instance(
connection_info=connection_info)
arakoon_config = VPoolShared.retrieve_alba_arakoon_config(
alba_backend_guid=alba_backend_guid,
ovs_client=ovs_client)
arakoons[ctype] = ArakoonClusterConfig.convert_config_to(
arakoon_config, return_type='INI')
else: # Local Backend for accelerated Backend
alba_backend_name = parameters[ctype]['backend_info'][
'alba_backend_name']
if Configuration.exists(key='/ovs/arakoon/{0}-abm/config'.
format(alba_backend_name),
raw=True) is False:
raise ValueError(
'Arakoon cluster for ALBA Backend {0} could not be retrieved'
.format(alba_backend_name))
arakoons[ctype] = Configuration.get(
key='/ovs/arakoon/{0}-abm/config'.format(
alba_backend_name),
raw=True)
cache_info[ctype] = [
'alba', {
'albamgr_cfg_url':
'/etc/hprm/{0}/{1}_cache_arakoon.ini'.format(
identifier, ctype),
'bucket_strategy': [
'1-to-1', {
'prefix':
vpool.guid,
'preset':
parameters[ctype]['backend_info']['preset']
}
],
'manifest_cache_size':
proxy_cfg['manifest_cache_size'],
'cache_on_read':
read,
'cache_on_write':
write
}
]
tgz_name = 'hprm_config_files_{0}_{1}.tgz'.format(
identifier, vpool.name)
config = {
'ips': ['127.0.0.1'],
'port': parameters['port'],
'pracc': {
'uds_path': '/var/run/hprm/{0}/uds_path'.format(identifier),
'max_clients': 1000,
'max_read_buf_size':
64 * 1024, # Buffer size for incoming requests (in bytes)
'thread_pool_size': 64
}, # Amount of threads
'transport': 'tcp',
'log_level': 'info',
'read_preference': proxy_cfg['read_preference'],
'albamgr_cfg_url': '/etc/hprm/{0}/arakoon.ini'.format(identifier),
'manifest_cache_size': proxy_cfg['manifest_cache_size']
}
file_contents_map = {}
for ctype in cache_types:
if ctype in cache_info:
config['{0}_cache'.format(ctype)] = cache_info[ctype]
if ctype in arakoons:
file_contents_map[
'/opt/OpenvStorage/config/{0}/{1}_cache_arakoon.ini'.
format(identifier, ctype)] = arakoons[ctype]
file_contents_map.update({
'/opt/OpenvStorage/config/{0}/config.json'.format(identifier):
json.dumps(config, indent=4),
'/opt/OpenvStorage/config/{0}/arakoon.ini'.format(identifier):
Configuration.get(key=config_path.format('abm'), raw=True)
})
local_client = SSHClient(endpoint=local_storagerouter)
local_client.dir_create(
directories='/opt/OpenvStorage/config/{0}'.format(identifier))
local_client.dir_create(
directories='/opt/OpenvStorage/webapps/frontend/downloads')
for file_name, contents in file_contents_map.iteritems():
local_client.file_write(contents=contents, filename=file_name)
local_client.run(command=[
'tar', '--transform', 's#^config/{0}#{0}#'.format(identifier),
'-czf', '/opt/OpenvStorage/webapps/frontend/downloads/{0}'.format(
tgz_name), 'config/{0}'.format(identifier)
])
local_client.dir_delete(
directories='/opt/OpenvStorage/config/{0}'.format(identifier))
return tgz_name
def services_running(self):
# type: () -> bool
"""
Check if all services are running
:return: Boolean
"""
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if self.target in [WatcherTypes.CONFIG, WatcherTypes.FWK]:
self.log_message('Testing configuration store...')
try:
Configuration.list('/')
except Exception as ex:
self.log_message(' Error during configuration store test: {0}'.format(ex), 2)
return False
with open(CACC_LOCATION) as config_file:
contents = config_file.read()
config = ArakoonClusterConfig(cluster_id=ARAKOON_NAME, load_config=False)
config.read_config(contents=contents)
client = ArakoonInstaller.build_client(config)
contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY, consistency=NoGuarantee())
if Watcher.LOG_CONTENTS != contents:
try:
config.read_config(contents=contents) # Validate whether the contents are not corrupt
except Exception as ex:
self.log_message(' Configuration stored in configuration store seems to be corrupt: {0}'.format(ex), 2)
return False
temp_filename = '{0}~'.format(CACC_LOCATION)
with open(temp_filename, 'w') as config_file:
config_file.write(contents)
config_file.flush()
os.fsync(config_file)
os.rename(temp_filename, CACC_LOCATION)
Watcher.LOG_CONTENTS = contents
self.log_message(' Configuration store OK', 0)
if self.target == WatcherTypes.FWK:
self._test_store('volatile', key, value)
self._test_store('persistent')
if self.target == WatcherTypes.VOLDRV:
# Arakoon, voldrv cluster
self._test_store('arakoon_voldrv')
if self.target in [WatcherTypes.FWK, WatcherTypes.VOLDRV]:
# RabbitMQ
self.log_message('Test rabbitMQ...', 0)
messagequeue = Configuration.get('/ovs/framework/messagequeue')
rmq_servers = messagequeue['endpoints']
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(messagequeue['protocol'],
messagequeue['user'],
messagequeue['password'],
server)
connection = pika.BlockingConnection(pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish('', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain', delivery_mode=1))
connection.close()
good_node = True
except Exception as message:
self.log_message(' Error during rabbitMQ test on node {0}: {1}'.format(server, message), 2)
if good_node is False:
self.log_message(' No working rabbitMQ node could be found', 2)
return False
self.log_message(' RabbitMQ test OK')
self.log_message('All tests OK')
return True
except Exception as ex:
self.log_message('Unexpected exception: {0}'.format(ex), 2)
return False
def setup_proxy_configs(self):
"""
Sets up the proxies their configuration data in the configuration management
:return: None
:rtype: NoneType
"""
def _generate_proxy_cache_config(cache_settings, cache_type, proxy_index):
if cache_settings['read'] is False and cache_settings['write'] is False:
return ['none']
if cache_settings['is_backend'] is True:
cfg_tree_name = 'abm_bc' if cache_type == StorageDriverConfiguration.CACHE_BLOCK else 'abm_aa'
return ['alba', {'cache_on_read': cache_settings['read'],
'cache_on_write': cache_settings['write'],
'albamgr_cfg_url': Configuration.get_configuration_path(config_tree.format(cfg_tree_name)),
'bucket_strategy': ['1-to-1', {'prefix': vpool.guid,
'preset': cache_settings['backend_info']['preset']}],
'manifest_cache_size': manifest_cache_size}]
if cache_type == StorageDriverConfiguration.CACHE_BLOCK:
path = '{0}/bc'.format(self.storagedriver_partitions_caches[proxy_index].path)
else:
path = '{0}/fc'.format(self.storagedriver_partitions_caches[proxy_index].path)
return ['local', {'path': path,
'max_size': self.cache_size_local / self.sr_installer.requested_local_proxies,
'cache_on_read': cache_settings['read'],
'cache_on_write': cache_settings['write']}]
def _generate_scrub_proxy_cache_config(cache_settings, main_proxy_cache_config):
scrub_cache_info = ['none']
if cache_settings['is_backend'] is True and cache_settings['write'] is True:
scrub_cache_info = copy.deepcopy(main_proxy_cache_config)
scrub_cache_info[1]['cache_on_read'] = False
return scrub_cache_info
def _generate_proxy_config(proxy_type, proxy_service):
proxy_config = {'log_level': 'info',
'port': proxy_service.service.ports[0] if proxy_type == 'main' else 0,
'ips': [self.storagedriver.storage_ip] if proxy_type == 'main' else ['127.0.0.1'],
'manifest_cache_size': manifest_cache_size,
'fragment_cache': fragment_cache_main_proxy if proxy_type == 'main' else fragment_cache_scrub_proxy,
'transport': 'tcp',
'read_preference': read_preferences,
'albamgr_cfg_url': Configuration.get_configuration_path(config_tree.format('abm'))}
if self.sr_installer.block_cache_supported:
proxy_config['block_cache'] = block_cache_main_proxy if proxy_type == 'main' else block_cache_scrub_proxy
return proxy_config
vpool = self.vp_installer.vpool
read_preferences = self.vp_installer.calculate_read_preferences()
manifest_cache_size = 500 * 1024 ** 2
block_cache_settings = vpool.metadata['caching_info'][self.storagedriver.storagerouter_guid][StorageDriverConfiguration.CACHE_BLOCK]
fragment_cache_settings = vpool.metadata['caching_info'][self.storagedriver.storagerouter_guid][StorageDriverConfiguration.CACHE_FRAGMENT]
# Obtain all arakoon configurations for each Backend (main, block cache, fragment cache)
arakoon_data = {'abm': vpool.metadata['backend']['backend_info']['arakoon_config']}
if block_cache_settings['is_backend'] is True:
arakoon_data['abm_bc'] = block_cache_settings['backend_info']['arakoon_config']
if fragment_cache_settings['is_backend'] is True:
arakoon_data['abm_aa'] = fragment_cache_settings['backend_info']['arakoon_config']
for proxy_id, alba_proxy in enumerate(self.storagedriver.alba_proxies):
config_tree = '/ovs/vpools/{0}/proxies/{1}/config/{{0}}'.format(vpool.guid, alba_proxy.guid)
for arakoon_entry, arakoon_config in arakoon_data.iteritems():
arakoon_config = ArakoonClusterConfig.convert_config_to(config=arakoon_config, return_type='INI')
Configuration.set(config_tree.format(arakoon_entry), arakoon_config, raw=True)
# Generate cache information for main proxy
block_cache_main_proxy = _generate_proxy_cache_config(cache_type=StorageDriverConfiguration.CACHE_BLOCK, cache_settings=block_cache_settings, proxy_index=proxy_id)
fragment_cache_main_proxy = _generate_proxy_cache_config(cache_type=StorageDriverConfiguration.CACHE_FRAGMENT, cache_settings=fragment_cache_settings, proxy_index=proxy_id)
# Generate cache information for scrub proxy
block_cache_scrub_proxy = _generate_scrub_proxy_cache_config(cache_settings=block_cache_settings, main_proxy_cache_config=block_cache_main_proxy)
fragment_cache_scrub_proxy = _generate_scrub_proxy_cache_config(cache_settings=fragment_cache_settings, main_proxy_cache_config=fragment_cache_main_proxy)
# Generate complete main and proxy configuration
main_proxy_config = _generate_proxy_config(proxy_type='main', proxy_service=alba_proxy)
scrub_proxy_config = _generate_proxy_config(proxy_type='scrub', proxy_service=alba_proxy)
# Add configurations to configuration management
Configuration.set(config_tree.format('main'), json.dumps(main_proxy_config, indent=4), raw=True)
Configuration.set('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid), json.dumps(scrub_proxy_config, indent=4), raw=True)
def configure_storagedriver_service(self):
"""
Configure the StorageDriver service
:return: None
:rtype: NoneType
"""
def _generate_queue_urls():
mq_user = Configuration.get('/ovs/framework/messagequeue|user')
mq_protocol = Configuration.get('/ovs/framework/messagequeue|protocol')
mq_password = Configuration.get('/ovs/framework/messagequeue|password')
return [{'amqp_uri': '{0}://{1}:{2}@{3}:5672'.format(mq_protocol, mq_user, mq_password, sr.ip)} for sr in StorageRouterList.get_masters()]
def _generate_config_file_system():
config = {'fs_dtl_host': '',
'fs_enable_shm_interface': 0,
'fs_enable_network_interface': 1,
'fs_metadata_backend_arakoon_cluster_nodes': [],
'fs_metadata_backend_mds_nodes': [],
'fs_metadata_backend_type': 'MDS',
'fs_virtual_disk_format': 'raw',
'fs_raw_disk_suffix': '.raw',
'fs_file_event_rules': [{'fs_file_event_rule_calls': ['Rename'],
'fs_file_event_rule_path_regex': '.*'}]}
if self.dtl_mode == StorageDriverClient.FRAMEWORK_DTL_NO_SYNC:
config['fs_dtl_config_mode'] = StorageDriverClient.VOLDRV_DTL_MANUAL_MODE
else:
config['fs_dtl_mode'] = StorageDriverClient.VPOOL_DTL_MODE_MAP[self.dtl_mode]
config['fs_dtl_config_mode'] = StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE
return config
def _generate_config_backend_connection_manager():
config = {'backend_type': 'MULTI',
'backend_interface_retries_on_error': 5,
'backend_interface_retry_interval_secs': 1,
'backend_interface_retry_backoff_multiplier': 2.0}
for index, proxy in enumerate(sorted(self.storagedriver.alba_proxies, key=lambda k: k.service.ports[0])):
config[str(index)] = {'alba_connection_host': self.storagedriver.storage_ip,
'alba_connection_port': proxy.service.ports[0],
'alba_connection_preset': vpool.metadata['backend']['backend_info']['preset'],
'alba_connection_timeout': 30,
'alba_connection_use_rora': True,
'alba_connection_transport': 'TCP',
'alba_connection_rora_manifest_cache_capacity': 25000,
'alba_connection_asd_connection_pool_capacity': 10,
'alba_connection_rora_timeout_msecs': 50,
'backend_type': 'ALBA'}
return config
if self.sr_installer is None:
raise RuntimeError('No StorageRouterInstaller instance found')
if len(self.write_caches) == 0:
raise RuntimeError('The StorageDriverPartition junctions have not been created yet')
vpool = self.vp_installer.vpool
gap_configuration = StorageDriverController.calculate_trigger_and_backoff_gap(cache_size=self.sr_installer.smallest_write_partition_size)
arakoon_cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|voldrv'))
arakoon_nodes = [{'host': node.ip,
'port': node.client_port,
'node_id': node.name} for node in ArakoonClusterConfig(cluster_id=arakoon_cluster_name).nodes]
storagedriver_config = StorageDriverConfiguration(vpool.guid, self.storagedriver.storagedriver_id)
storagedriver_config.configure_scocache(scocache_mount_points=self.write_caches,
trigger_gap=ExtensionsToolbox.convert_byte_size_to_human_readable(size=gap_configuration['trigger']),
backoff_gap=ExtensionsToolbox.convert_byte_size_to_human_readable(size=gap_configuration['backoff']))
storagedriver_config.configure_file_driver(fd_cache_path=self.storagedriver_partition_file_driver.path,
fd_extent_cache_capacity='1024',
fd_namespace='fd-{0}-{1}'.format(vpool.name, vpool.guid))
storagedriver_config.configure_volume_router(vrouter_id=self.storagedriver.storagedriver_id,
vrouter_redirect_timeout_ms='120000',
vrouter_keepalive_time_secs='15',
vrouter_keepalive_interval_secs='5',
vrouter_keepalive_retries='2',
vrouter_routing_retries=10,
vrouter_volume_read_threshold=0,
vrouter_volume_write_threshold=0,
vrouter_file_read_threshold=0,
vrouter_file_write_threshold=0,
vrouter_min_workers=4,
vrouter_max_workers=16,
vrouter_sco_multiplier=self.sco_size * 1024 / self.cluster_size,
vrouter_backend_sync_timeout_ms=60000,
vrouter_migrate_timeout_ms=60000,
vrouter_use_fencing=True)
storagedriver_config.configure_volume_manager(tlog_path=self.storagedriver_partition_tlogs.path,
metadata_path=self.storagedriver_partition_metadata.path,
clean_interval=1,
dtl_throttle_usecs=4000,
default_cluster_size=self.cluster_size * 1024,
number_of_scos_in_tlog=self.tlog_multiplier,
non_disposable_scos_factor=float(self.write_buffer) / self.tlog_multiplier / self.sco_size)
storagedriver_config.configure_event_publisher(events_amqp_routing_key=Configuration.get('/ovs/framework/messagequeue|queues.storagedriver'),
events_amqp_uris=_generate_queue_urls())
storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id=arakoon_cluster_name,
vregistry_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.configure_network_interface(network_max_neighbour_distance=StorageDriver.DISTANCES.FAR - 1)
storagedriver_config.configure_threadpool_component(num_threads=16)
storagedriver_config.configure_volume_router_cluster(vrouter_cluster_id=vpool.guid)
storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon',
dls_arakoon_cluster_id=arakoon_cluster_name,
dls_arakoon_cluster_nodes=arakoon_nodes)
storagedriver_config.configure_content_addressed_cache(serialize_read_cache=False,
read_cache_serialization_path=[])
storagedriver_config.configure_distributed_transaction_log(dtl_path=self.storagedriver_partition_dtl.path, # Not used, but required
dtl_transport=StorageDriverClient.VPOOL_DTL_TRANSPORT_MAP[self.dtl_transport])
storagedriver_config.configure_filesystem(**_generate_config_file_system())
storagedriver_config.configure_backend_connection_manager(**_generate_config_backend_connection_manager())
storagedriver_config.save(client=self.sr_installer.root_client)
def services_running(self, target):
"""
Check all services are running
:param target: Target to check
:return: Boolean
"""
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target in ['config', 'framework']:
self.log_message(target, 'Testing configuration store...', 0)
from ovs.extensions.generic.configuration import Configuration
try:
Configuration.list('/')
except Exception as ex:
self.log_message(
target,
' Error during configuration store test: {0}'.format(
ex), 2)
return False
from ovs.extensions.db.arakooninstaller import ArakoonInstaller, ArakoonClusterConfig
from ovs_extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee
from ovs.extensions.generic.configuration import Configuration
with open(Configuration.CACC_LOCATION) as config_file:
contents = config_file.read()
config = ArakoonClusterConfig(
cluster_id=Configuration.ARAKOON_NAME, load_config=False)
config.read_config(contents=contents)
client = ArakoonInstaller.build_client(config)
contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY,
consistency=NoGuarantee())
if Watcher.LOG_CONTENTS != contents:
try:
config.read_config(
contents=contents
) # Validate whether the contents are not corrupt
except Exception as ex:
self.log_message(
target,
' Configuration stored in configuration store seems to be corrupt: {0}'
.format(ex), 2)
return False
temp_filename = '{0}~'.format(Configuration.CACC_LOCATION)
with open(temp_filename, 'w') as config_file:
config_file.write(contents)
config_file.flush()
os.fsync(config_file)
os.rename(temp_filename, Configuration.CACC_LOCATION)
Watcher.LOG_CONTENTS = contents
self.log_message(target, ' Configuration store OK', 0)
if target == 'framework':
# Volatile
self.log_message(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during volatile store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target,
' Volatile store not working correctly',
2)
return False
self.log_message(
target,
' Volatile store OK after {0} tries'.format(tries), 0)
# Persistent
self.log_message(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
persistent = PersistentFactory.get_client()
persistent.nop()
break
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during persistent store test: {0}'.format(
message), 2)
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Persistent store not working correctly', 2)
return False
self.log_message(
target,
' Persistent store OK after {0} tries'.format(tries), 0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
self.log_message(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.generic.configuration import Configuration
from ovs_extensions.storage.persistent.pyrakoonstore import PyrakoonStore
cluster_name = str(
Configuration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
configuration = Configuration.get(
'/ovs/arakoon/{0}/config'.format(cluster_name),
raw=True)
client = PyrakoonStore(cluster=cluster_name,
configuration=configuration)
client.nop()
break
except Exception as message:
self.log_message(
target,
' Error during arakoon (voldrv) test: {0}'.format(
message), 2)
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Arakoon (voldrv) not working correctly', 2)
return False
self.log_message(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
self.log_message(target, 'Test rabbitMQ...', 0)
import pika
from ovs.extensions.generic.configuration import Configuration
messagequeue = Configuration.get('/ovs/framework/messagequeue')
rmq_servers = messagequeue['endpoints']
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(
messagequeue['protocol'], messagequeue['user'],
messagequeue['password'], server)
connection = pika.BlockingConnection(
pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish(
'', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain',
delivery_mode=1))
connection.close()
good_node = True
except Exception as message:
self.log_message(
target,
' Error during rabbitMQ test on node {0}: {1}'.
format(server, message), 2)
if good_node is False:
self.log_message(
target, ' No working rabbitMQ node could be found', 2)
return False
self.log_message(target, ' RabbitMQ test OK', 0)
self.log_message(target, 'All tests OK', 0)
return True
except Exception as ex:
self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
def promote_node(cluster_ip, master_ip, ip_client_map, unique_id,
configure_memcached, configure_rabbitmq):
"""
Promotes a given node
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.servicelist import ServiceList
from ovs.dal.hybrids.service import Service
Toolbox.log(logger=NodeTypeController._logger,
messages='Promoting node',
title=True)
service_manager = ServiceFactory.get_manager()
if configure_memcached is True:
if NodeTypeController._validate_local_memcache_servers(
ip_client_map) is False:
raise RuntimeError(
'Not all memcache nodes can be reached which is required for promoting a node.'
)
target_client = ip_client_map[cluster_ip]
machine_id = System.get_my_machine_id(target_client)
node_name, _ = target_client.get_hostname()
master_client = ip_client_map[master_ip]
storagerouter = StorageRouterList.get_by_machine_id(unique_id)
storagerouter.node_type = 'MASTER'
storagerouter.save()
external_config = Configuration.get('/ovs/framework/external_config')
if external_config is None:
Toolbox.log(logger=NodeTypeController._logger,
messages='Joining Arakoon configuration cluster')
arakoon_installer = ArakoonInstaller(cluster_name='config')
arakoon_installer.load(ip=master_ip)
arakoon_installer.extend_cluster(
new_ip=cluster_ip,
base_dir=Configuration.get('/ovs/framework/paths|ovsdb'))
arakoon_installer.restart_cluster_after_extending(
new_ip=cluster_ip)
service_manager.register_service(
node_name=machine_id,
service_metadata=arakoon_installer.service_metadata[cluster_ip]
)
# Find other (arakoon) master nodes
arakoon_cluster_name = str(
Configuration.get('/ovs/framework/arakoon_clusters|ovsdb'))
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=arakoon_cluster_name)
config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name)
master_node_ips = [node.ip for node in config.nodes]
if cluster_ip in master_node_ips:
master_node_ips.remove(cluster_ip)
if len(master_node_ips) == 0:
raise RuntimeError(
'There should be at least one other master node')
arakoon_ports = []
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger,
messages='Joining Arakoon OVS DB cluster')
arakoon_installer = ArakoonInstaller(
cluster_name=arakoon_cluster_name)
arakoon_installer.load()
arakoon_installer.extend_cluster(
new_ip=cluster_ip,
base_dir=Configuration.get('/ovs/framework/paths|ovsdb'))
arakoon_installer.restart_cluster_after_extending(
new_ip=cluster_ip)
arakoon_ports = arakoon_installer.ports[cluster_ip]
if configure_memcached is True:
NodeTypeController.configure_memcached(
client=target_client, logger=NodeTypeController._logger)
NodeTypeController.add_services(client=target_client,
node_type='master',
logger=NodeTypeController._logger)
Toolbox.log(logger=NodeTypeController._logger,
messages='Update configurations')
if configure_memcached is True:
endpoints = Configuration.get('/ovs/framework/memcache|endpoints')
endpoint = '{0}:11211'.format(cluster_ip)
if endpoint not in endpoints:
endpoints.append(endpoint)
Configuration.set('/ovs/framework/memcache|endpoints',
endpoints)
if configure_rabbitmq is True:
endpoints = Configuration.get(
'/ovs/framework/messagequeue|endpoints')
endpoint = '{0}:5672'.format(cluster_ip)
if endpoint not in endpoints:
endpoints.append(endpoint)
Configuration.set('/ovs/framework/messagequeue|endpoints',
endpoints)
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger,
messages='Restarting master node services')
PersistentFactory.store = None
VolatileFactory.store = None
if 'arakoon-ovsdb' not in [
s.name for s in ServiceList.get_services() if
s.is_internal is False or s.storagerouter.ip == cluster_ip
]:
service = Service()
service.name = 'arakoon-ovsdb'
service.type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
service.ports = arakoon_ports
service.storagerouter = storagerouter
service.save()
if configure_rabbitmq is True:
NodeTypeController.configure_rabbitmq(
client=target_client, logger=NodeTypeController._logger)
# Copy rabbitmq cookie
rabbitmq_cookie_file = '/var/lib/rabbitmq/.erlang.cookie'
Toolbox.log(logger=NodeTypeController._logger,
messages='Copying RabbitMQ cookie')
contents = master_client.file_read(rabbitmq_cookie_file)
master_hostname, _ = master_client.get_hostname()
target_client.dir_create(os.path.dirname(rabbitmq_cookie_file))
target_client.file_write(rabbitmq_cookie_file, contents)
target_client.file_chmod(rabbitmq_cookie_file, mode=0400)
target_client.run(['rabbitmq-server', '-detached'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop_app'])
time.sleep(5)
target_client.run([
'rabbitmqctl', 'join_cluster',
'rabbit@{0}'.format(master_hostname)
])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop'])
time.sleep(5)
# Enable HA for the rabbitMQ queues
ServiceFactory.change_service_state(target_client,
'rabbitmq-server', 'start',
NodeTypeController._logger)
NodeTypeController.check_rabbitmq_and_enable_ha_mode(
client=target_client, logger=NodeTypeController._logger)
NodeTypeController._configure_amqp_to_volumedriver()
Toolbox.log(logger=NodeTypeController._logger,
messages='Starting services')
services = ['memcached', 'arakoon-ovsdb', 'rabbitmq-server']
if arakoon_metadata['internal'] is True:
services.remove('arakoon-ovsdb')
for service in services:
if service_manager.has_service(service, client=target_client):
ServiceFactory.change_service_state(target_client, service,
'start',
NodeTypeController._logger)
Toolbox.log(logger=NodeTypeController._logger,
messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(
clients=ip_client_map, logger=NodeTypeController._logger)
if Toolbox.run_hooks(component='nodetype',
sub_component='promote',
logger=NodeTypeController._logger,
cluster_ip=cluster_ip,
master_ip=master_ip):
Toolbox.log(logger=NodeTypeController._logger,
messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(
clients=ip_client_map, logger=NodeTypeController._logger)
if NodeTypeController.avahi_installed(
client=target_client,
logger=NodeTypeController._logger) is True:
NodeTypeController.configure_avahi(
client=target_client,
node_name=node_name,
node_type='master',
logger=NodeTypeController._logger)
Configuration.set('/ovs/framework/hosts/{0}/type'.format(machine_id),
'MASTER')
target_client.run(
['chown', '-R', 'ovs:ovs', '/opt/OpenvStorage/config'])
Configuration.set(
'/ovs/framework/hosts/{0}/promotecompleted'.format(machine_id),
True)
if target_client.file_exists('/tmp/ovs_rollback'):
target_client.file_delete('/tmp/ovs_rollback')
Toolbox.log(logger=NodeTypeController._logger,
messages='Promote complete')
def demote_node(cluster_ip,
master_ip,
ip_client_map,
unique_id,
unconfigure_memcached,
unconfigure_rabbitmq,
offline_nodes=None):
"""
Demotes a given node
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
Toolbox.log(logger=NodeTypeController._logger,
messages='Demoting node',
title=True)
service_manager = ServiceFactory.get_manager()
if offline_nodes is None:
offline_nodes = []
if unconfigure_memcached is True and len(offline_nodes) == 0:
if NodeTypeController._validate_local_memcache_servers(
ip_client_map) is False:
raise RuntimeError(
'Not all memcache nodes can be reached which is required for demoting a node.'
)
# Find other (arakoon) master nodes
arakoon_cluster_name = str(
Configuration.get('/ovs/framework/arakoon_clusters|ovsdb'))
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=arakoon_cluster_name)
config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name)
master_node_ips = [node.ip for node in config.nodes]
shrink = False
if cluster_ip in master_node_ips:
shrink = True
master_node_ips.remove(cluster_ip)
if len(master_node_ips) == 0:
raise RuntimeError(
'There should be at least one other master node')
storagerouter = StorageRouterList.get_by_machine_id(unique_id)
storagerouter.node_type = 'EXTRA'
storagerouter.save()
offline_node_ips = [node.ip for node in offline_nodes]
if arakoon_metadata['internal'] is True and shrink is True:
Toolbox.log(logger=NodeTypeController._logger,
messages='Leaving Arakoon {0} cluster'.format(
arakoon_cluster_name))
arakoon_installer = ArakoonInstaller(
cluster_name=arakoon_cluster_name)
arakoon_installer.load()
arakoon_installer.shrink_cluster(removal_ip=cluster_ip,
offline_nodes=offline_node_ips)
arakoon_installer.restart_cluster_after_shrinking()
try:
external_config = Configuration.get(
'/ovs/framework/external_config')
if external_config is None and shrink is True:
Toolbox.log(logger=NodeTypeController._logger,
messages='Leaving Arakoon config cluster')
arakoon_installer = ArakoonInstaller(cluster_name='config')
arakoon_installer.load(ip=master_node_ips[0])
arakoon_installer.shrink_cluster(
removal_ip=cluster_ip, offline_nodes=offline_node_ips)
arakoon_installer.restart_cluster_after_shrinking()
except Exception as ex:
Toolbox.log(
logger=NodeTypeController._logger,
messages=['\nFailed to leave configuration cluster', ex],
loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger,
messages='Update configurations')
try:
if unconfigure_memcached is True:
endpoints = Configuration.get(
'/ovs/framework/memcache|endpoints')
endpoint = '{0}:{1}'.format(cluster_ip, 11211)
if endpoint in endpoints:
endpoints.remove(endpoint)
Configuration.set('/ovs/framework/memcache|endpoints',
endpoints)
if unconfigure_rabbitmq is True:
endpoints = Configuration.get(
'/ovs/framework/messagequeue|endpoints')
endpoint = '{0}:{1}'.format(cluster_ip, 5672)
if endpoint in endpoints:
endpoints.remove(endpoint)
Configuration.set('/ovs/framework/messagequeue|endpoints',
endpoints)
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger,
messages=['\nFailed to update configurations', ex],
loglevel='exception')
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger,
messages='Restarting master node services')
remaining_nodes = ip_client_map.keys()[:]
if cluster_ip in remaining_nodes:
remaining_nodes.remove(cluster_ip)
PersistentFactory.store = None
VolatileFactory.store = None
for service in storagerouter.services:
if service.name == 'arakoon-ovsdb':
service.delete()
target_client = None
if storagerouter in offline_nodes:
if unconfigure_rabbitmq is True:
Toolbox.log(
logger=NodeTypeController._logger,
messages='Removing/unconfiguring offline RabbitMQ node')
client = ip_client_map[master_ip]
try:
client.run([
'rabbitmqctl', 'forget_cluster_node',
'rabbit@{0}'.format(storagerouter.name)
])
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger,
messages=[
'\nFailed to forget RabbitMQ cluster node',
ex
],
loglevel='exception')
else:
target_client = ip_client_map[cluster_ip]
if unconfigure_rabbitmq is True:
Toolbox.log(logger=NodeTypeController._logger,
messages='Removing/unconfiguring RabbitMQ')
try:
if service_manager.has_service('rabbitmq-server',
client=target_client):
ServiceFactory.change_service_state(
target_client, 'rabbitmq-server', 'stop',
NodeTypeController._logger)
target_client.run(['rabbitmq-server', '-detached'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop_app'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'reset'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop'])
time.sleep(5)
target_client.file_unlink(
"/var/lib/rabbitmq/.erlang.cookie")
ServiceFactory.change_service_state(
target_client, 'rabbitmq-server', 'stop',
NodeTypeController._logger) # To be sure
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger,
messages=[
'\nFailed to remove/unconfigure RabbitMQ',
ex
],
loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger,
messages='Stopping services')
services = ['memcached', 'rabbitmq-server']
if unconfigure_rabbitmq is False:
services.remove('rabbitmq-server')
if unconfigure_memcached is False:
services.remove('memcached')
for service in services:
if service_manager.has_service(service, client=target_client):
Toolbox.log(
logger=NodeTypeController._logger,
messages='Stopping service {0}'.format(service))
try:
ServiceFactory.change_service_state(
target_client, service, 'stop',
NodeTypeController._logger)
except Exception as ex:
Toolbox.log(
logger=NodeTypeController._logger,
messages=[
'\nFailed to stop service'.format(service), ex
],
loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger,
messages='Removing services')
services = [
'scheduled-tasks', 'webapp-api', 'volumerouter-consumer'
]
for service in services:
if service_manager.has_service(service, client=target_client):
Toolbox.log(
logger=NodeTypeController._logger,
messages='Removing service {0}'.format(service))
try:
ServiceFactory.change_service_state(
target_client, service, 'stop',
NodeTypeController._logger)
service_manager.remove_service(service,
client=target_client)
except Exception as ex:
Toolbox.log(
logger=NodeTypeController._logger,
messages=[
'\nFailed to remove service'.format(service),
ex
],
loglevel='exception')
if service_manager.has_service('workers', client=target_client):
service_manager.add_service(
name='workers',
client=target_client,
params={'WORKER_QUEUE': '{0}'.format(unique_id)})
try:
NodeTypeController._configure_amqp_to_volumedriver()
except Exception as ex:
Toolbox.log(
logger=NodeTypeController._logger,
messages=['\nFailed to configure AMQP to Storage Driver', ex],
loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger,
messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(
clients=ip_client_map,
logger=NodeTypeController._logger,
offline_node_ips=offline_node_ips)
if Toolbox.run_hooks(component='nodetype',
sub_component='demote',
logger=NodeTypeController._logger,
cluster_ip=cluster_ip,
master_ip=master_ip,
offline_node_ips=offline_node_ips):
Toolbox.log(logger=NodeTypeController._logger,
messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(
clients=ip_client_map,
logger=NodeTypeController._logger,
offline_node_ips=offline_node_ips)
if storagerouter not in offline_nodes:
target_client = ip_client_map[cluster_ip]
node_name, _ = target_client.get_hostname()
if NodeTypeController.avahi_installed(
client=target_client,
logger=NodeTypeController._logger) is True:
NodeTypeController.configure_avahi(
client=target_client,
node_name=node_name,
node_type='extra',
logger=NodeTypeController._logger)
Configuration.set(
'/ovs/framework/hosts/{0}/type'.format(storagerouter.machine_id),
'EXTRA')
if target_client is not None and target_client.file_exists(
'/tmp/ovs_rollback'):
target_client.file_write('/tmp/ovs_rollback', 'rollback')
Toolbox.log(logger=NodeTypeController._logger,
messages='Demote complete',
title=True)
