def test_alba_arakoon_checkup(self):
"""
Validates whether the ALBA Arakoon checkup works (Manual and Scheduled)
"""
ovs_structure = DalHelper.build_dal_structure(
structure={'storagerouters': [1]})
alba_structure = AlbaDalHelper.build_dal_structure(
structure={'alba_backends': [[1, 'LOCAL']]})
#############################
# SCHEDULED_ARAKOON_CHECKUP #
#############################
# Create an ABM and NSM cluster for ALBA Backend 1 and do some basic validations
sr_1 = ovs_structure['storagerouters'][1]
ab_1 = alba_structure['alba_backends'][1]
MockedSSHClient._run_returns[sr_1.ip] = {}
MockedSSHClient._run_returns[sr_1.ip][
'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-abm/db'] = None
MockedSSHClient._run_returns[sr_1.ip][
'ln -s /usr/lib/alba/nsm_host_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-nsm_0/db'] = None
AlbaController.add_cluster(ab_1.guid)
abm_cluster_name = '{0}-abm'.format(ab_1.name)
nsm_cluster_name = '{0}-nsm_0'.format(ab_1.name)
arakoon_clusters = sorted(Configuration.list('/ovs/arakoon'))
self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name],
list2=arakoon_clusters)
abm_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=abm_cluster_name)
nsm_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=nsm_cluster_name)
self.assertTrue(expr=abm_metadata['in_use'])
self.assertTrue(expr=nsm_metadata['in_use'])
# Run scheduled Arakoon checkup and validate amount of Arakoon clusters did not change
AlbaArakoonController.scheduled_alba_arakoon_checkup()
self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name],
list2=arakoon_clusters)
self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=1)
self.assertEqual(first=len(ab_1.nsm_clusters), second=1)
self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services),
second=1)
# Create 2 additional StorageRouters
srs = DalHelper.build_dal_structure(
structure={'storagerouters': [2, 3]},
previous_structure=ovs_structure)['storagerouters']
sr_2 = srs[2]
sr_3 = srs[3]
# Run scheduled checkup again and do some validations
MockedSSHClient._run_returns[sr_2.ip] = {}
MockedSSHClient._run_returns[sr_3.ip] = {}
MockedSSHClient._run_returns[sr_2.ip][
'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_2/disk_1/partition_1/arakoon/backend_1-abm/db'] = None
MockedSSHClient._run_returns[sr_3.ip][
'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_3/disk_1/partition_1/arakoon/backend_1-abm/db'] = None
MockedSSHClient._run_returns[sr_2.ip][
'arakoon --node {0} -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/backend_1-abm/config -catchup-only'
.format(sr_2.machine_id)] = None
MockedSSHClient._run_returns[sr_3.ip][
'arakoon --node {0} -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/backend_1-abm/config -catchup-only'
.format(sr_3.machine_id)] = None
AlbaArakoonController.scheduled_alba_arakoon_checkup()
self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name],
list2=arakoon_clusters)
self.assertEqual(first=len(ab_1.abm_cluster.abm_services),
second=3) # Gone up from 1 to 3
self.assertEqual(first=len(ab_1.nsm_clusters), second=1)
self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services),
second=1) # Still 1 since NSM checkup hasn't ran yet
# Make sure 1 StorageRouter is unreachable
SSHClient._raise_exceptions[sr_3.ip] = {
'users': ['ovs'],
'exception': UnableToConnectException('No route to host')
}
AlbaArakoonController.scheduled_alba_arakoon_checkup()
alba_logs = Logger._logs.get('lib', [])
self.assertIn(
member='Storage Router with IP {0} is not reachable'.format(
sr_3.ip),
container=alba_logs)
##########################
# MANUAL_ARAKOON_CHECKUP #
##########################
AlbaDalHelper.setup() # Clear everything
ovs_structure = DalHelper.build_dal_structure(
structure={'storagerouters': [1]})
alba_structure = AlbaDalHelper.build_dal_structure(
structure={'alba_backends': [[1, 'LOCAL']]})
sr_1 = ovs_structure['storagerouters'][1]
ab_1 = alba_structure['alba_backends'][1]
MockedSSHClient._run_returns[sr_1.ip] = {}
MockedSSHClient._run_returns[sr_1.ip][
'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-abm/db'] = None
MockedSSHClient._run_returns[sr_1.ip][
'ln -s /usr/lib/alba/nsm_host_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-nsm_0/db'] = None
AlbaController.add_cluster(ab_1.guid)
# Run manual Arakoon checkup and validate amount of Arakoon clusters did not change
AlbaArakoonController.manual_alba_arakoon_checkup(
alba_backend_guid=ab_1.guid, nsm_clusters=[], abm_cluster=None)
self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name],
list2=arakoon_clusters)
self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=1)
self.assertEqual(first=len(ab_1.nsm_clusters), second=1)
self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services),
second=1)
# Test some error paths
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.manual_alba_arakoon_checkup(
alba_backend_guid=ab_1.guid,
nsm_clusters=['no_abm_cluster_passed'])
self.assertEqual(
first=raise_info.exception.message,
second='Both ABM cluster and NSM clusters must be provided')
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.manual_alba_arakoon_checkup(
alba_backend_guid=ab_1.guid,
nsm_clusters=[],
abm_cluster='no_nsm_clusters_passed')
self.assertEqual(
first=raise_info.exception.message,
second='Both ABM cluster and NSM clusters must be provided')
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.manual_alba_arakoon_checkup(
alba_backend_guid=ab_1.guid,
nsm_clusters=[nsm_cluster_name],
abm_cluster=abm_cluster_name)
self.assertEqual(first=raise_info.exception.message,
second='Cluster {0} has already been claimed'.format(
abm_cluster_name))
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.manual_alba_arakoon_checkup(
alba_backend_guid=ab_1.guid,
nsm_clusters=['non-existing-nsm-cluster'],
abm_cluster='non-existing-abm-cluster')
self.assertEqual(
first=raise_info.exception.message,
second=
'Could not find an Arakoon cluster with name: non-existing-abm-cluster'
)
# Recreate ALBA Backend with Arakoon clusters
AlbaDalHelper.setup() # Clear everything
ovs_structure = DalHelper.build_dal_structure(
structure={'storagerouters': [1]})
alba_structure = AlbaDalHelper.build_dal_structure(
structure={'alba_backends': [[1, 'LOCAL']]})
sr_1 = ovs_structure['storagerouters'][1]
ab_1 = alba_structure['alba_backends'][1]
# Create some Arakoon clusters to be claimed by the manual checkup
for cluster_name, cluster_type in {
'manual-abm-1': ServiceType.ARAKOON_CLUSTER_TYPES.ABM,
'manual-abm-2': ServiceType.ARAKOON_CLUSTER_TYPES.ABM,
'manual-nsm-1': ServiceType.ARAKOON_CLUSTER_TYPES.NSM,
'manual-nsm-2': ServiceType.ARAKOON_CLUSTER_TYPES.NSM,
'manual-nsm-3': ServiceType.ARAKOON_CLUSTER_TYPES.NSM
}.iteritems():
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
arakoon_installer.create_cluster(
cluster_type=cluster_type,
ip=sr_1.ip,
base_dir=DalHelper.CLUSTER_DIR.format(cluster_name),
internal=False)
arakoon_installer.start_cluster()
arakoon_installer.unclaim_cluster()
AlbaArakoonController.manual_alba_arakoon_checkup(
alba_backend_guid=ab_1.guid,
nsm_clusters=['manual-nsm-1', 'manual-nsm-3'],
abm_cluster='manual-abm-2')
# Validate the correct clusters have been claimed by the manual checkup
unused_abms = ArakoonInstaller.get_unused_arakoon_clusters(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.ABM)
unused_nsms = ArakoonInstaller.get_unused_arakoon_clusters(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM)
self.assertEqual(first=len(unused_abms), second=1)
self.assertEqual(first=len(unused_nsms), second=1)
self.assertEqual(first=unused_abms[0]['cluster_name'],
second='manual-abm-1')
self.assertEqual(first=unused_nsms[0]['cluster_name'],
second='manual-nsm-2')
def add_arakoon(cluster_name,
storagerouter_ip,
cluster_basedir,
service_type=ServiceType.ARAKOON_CLUSTER_TYPES.FWK):
"""
Adds a external arakoon to a storagerouter
:param cluster_name: name of the new arakoon cluster
:type cluster_name: str
:param service_type: type of plugin for arakoon (DEFAULT=ServiceType.ARAKOON_CLUSTER_TYPES.FWK)
* FWK
* ABM
* NSM
:type service_type: ovs.dal.hybrids.ServiceType.ARAKOON_CLUSTER_TYPES
:param storagerouter_ip: ip of a storagerouter
:type storagerouter_ip: str
:param cluster_basedir: absolute path for the new arakoon cluster
:type cluster_basedir: str
:return:
"""
client = SSHClient(storagerouter_ip, username='******')
# create required directories
if not client.dir_exists(cluster_basedir):
client.dir_create(cluster_basedir)
# determine plugin
if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.FWK:
plugins = None
elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
plugins = {
AlbaController.ABM_PLUGIN: AlbaController.ALBA_VERSION_GET
}
elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
plugins = {
AlbaController.NSM_PLUGIN: AlbaController.ALBA_VERSION_GET
}
else:
raise RuntimeError(
"Incompatible Arakoon cluster type selected: {0}".format(
service_type))
ArakoonSetup.LOGGER.info(
"Starting creation of new arakoon cluster with name `{0}`, servicetype `{1}`, ip `{2}`, base_dir `{3}`"
.format(cluster_name, service_type, storagerouter_ip,
cluster_basedir))
arakoon_installer = ArakoonInstaller(cluster_name)
arakoon_installer.create_cluster(
cluster_type=service_type,
ip=storagerouter_ip,
base_dir=cluster_basedir,
plugins=plugins,
locked=False,
internal=False,
log_sinks=Logger.get_sink_path('automation_lib_arakoon_server'),
crash_log_sinks=Logger.get_sink_path(
'automation_lib_arakoon_server_crash'))
if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
client.run([
'ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs',
'{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)
])
elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
client.run([
'ln', '-s', '/usr/lib/alba/nsm_host_plugin.cmxs',
'{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name)
])
arakoon_installer.start_cluster()
arakoon_installer.unclaim_cluster()
ArakoonSetup.LOGGER.info(
"Finished creation of new arakoon cluster with name `{0}`, servicetype `{1}`, ip `{2}`, base_dir `{3}`"
.format(cluster_name, service_type, storagerouter_ip,
cluster_basedir))
def test_nsm_checkup_external(self):
"""
Validates whether the NSM checkup works for externally managed Arakoon clusters
"""
Configuration.set('/ovs/framework/plugins/alba/config|nsm.safety', 1)
Configuration.set('/ovs/framework/plugins/alba/config|nsm.maxload', 10)
structure = DalHelper.build_dal_structure(structure={'storagerouters': [1, 2, 3]})
alba_structure = AlbaDalHelper.build_dal_structure(structure={'alba_backends': [[1, 'LOCAL']]})
alba_backend = alba_structure['alba_backends'][1]
storagerouter_1 = structure['storagerouters'][1]
storagerouter_2 = structure['storagerouters'][2]
# Validate some logic for externally managed arakoons during NSM checkup
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.nsm_checkup(external_nsm_cluster_names=['test']) # No ALBA Backend specified
self.assertEqual(first=str(raise_info.exception), second='Additional NSMs can only be configured for a specific ALBA Backend')
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, min_internal_nsms=2, external_nsm_cluster_names=['test'])
self.assertEqual(first=str(raise_info.exception), second="'min_internal_nsms' and 'external_nsm_cluster_names' are mutually exclusive")
with self.assertRaises(ValueError) as raise_info:
# noinspection PyTypeChecker
AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names={}) # NSM cluster names must be a list
self.assertEqual(first=str(raise_info.exception), second="'external_nsm_cluster_names' must be of type 'list'")
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=['non-existing-cluster']) # non-existing cluster names should raise
self.assertEqual(first=str(raise_info.exception), second="Arakoon cluster with name non-existing-cluster does not exist")
# Create an external ABM and NSM Arakoon cluster
external_abm_1 = 'backend_1-abm'
external_nsm_1 = 'backend_1-nsm_0'
external_nsm_2 = 'backend_1-nsm_1'
for cluster_name, cluster_type in {external_abm_1: 'ABM', external_nsm_1: 'NSM', external_nsm_2: 'NSM'}.iteritems():
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
arakoon_installer.create_cluster(cluster_type=cluster_type, ip=storagerouter_1.ip, base_dir='/tmp', internal=False)
arakoon_installer.extend_cluster(new_ip=storagerouter_2.ip, base_dir='/tmp')
arakoon_installer.start_cluster()
arakoon_installer.unclaim_cluster()
self.assertDictEqual(d1={'cluster_name': cluster_name,
'cluster_type': cluster_type,
'internal': False,
'in_use': False},
d2=arakoon_installer.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name))
# Let the 'add_cluster` claim the externally managed clusters and model the services
Logger._logs = {}
AlbaController.add_cluster(alba_backend_guid=alba_backend.guid,
abm_cluster=external_abm_1,
nsm_clusters=[external_nsm_1]) # Only claim external_nsm_1
for cluster_name, cluster_type in {external_abm_1: 'ABM', external_nsm_1: 'NSM', external_nsm_2: 'NSM'}.iteritems():
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
self.assertDictEqual(d1={'cluster_name': cluster_name,
'cluster_type': cluster_type,
'internal': False,
'in_use': False if cluster_name == external_nsm_2 else True},
d2=arakoon_installer.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name))
log_found = False
for log_record in Logger._logs.get('lib', []):
if 'NSM load OK' in log_record:
log_found = True
break
self.assertTrue(expr=log_found)
self.assertEqual(first=1, second=len(alba_backend.abm_cluster.abm_services))
self.assertEqual(first=1, second=len(alba_backend.nsm_clusters))
self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[0].nsm_services))
self.assertIsNone(obj=alba_backend.abm_cluster.abm_services[0].service.storagerouter)
self.assertIsNone(obj=alba_backend.nsm_clusters[0].nsm_services[0].service.storagerouter)
self.assertListEqual(VirtualAlbaBackend.run_log['backend_1-abm'], [['update_abm_client_config'],
['add_nsm_host', 'backend_1-nsm_0'],
['update_maintenance_config','--eviction-type-random'],
['update_maintenance_config','enable-auto-cleanup-deleted-namespaces-days']])
# Add cluster already invokes a NSM checkup, so nothing should have changed
VirtualAlbaBackend.run_log['backend_1-abm'] = []
AlbaArakoonController.nsm_checkup()
self.assertListEqual(list1=[], list2=VirtualAlbaBackend.run_log['backend_1-abm'])
# Overload the only NSM and run NSM checkup. This should log a critical message, but change nothing
VirtualAlbaBackend.data['backend_1-abm']['nsms'][0]['namespaces_count'] = 25
Logger._logs = {}
AlbaArakoonController.nsm_checkup()
log_found = False
for log_record in Logger._logs.get('lib', []):
if 'All NSM clusters are overloaded' in log_record:
log_found = True
break
self.assertTrue(expr=log_found)
self.assertEqual(first=1, second=len(alba_backend.abm_cluster.abm_services))
self.assertEqual(first=1, second=len(alba_backend.nsm_clusters))
self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[0].nsm_services))
self.assertIsNone(obj=alba_backend.abm_cluster.abm_services[0].service.storagerouter)
self.assertIsNone(obj=alba_backend.nsm_clusters[0].nsm_services[0].service.storagerouter)
self.assertListEqual(list1=[], list2=VirtualAlbaBackend.run_log['backend_1-abm'])
# Validate a maximum of 50 NSMs can be deployed
current_nsms = [nsm_cluster.number for nsm_cluster in alba_backend.nsm_clusters]
alba_structure = AlbaDalHelper.build_dal_structure(
structure={'alba_nsm_clusters': [(1, 50)]}, # (<abackend_id>, <amount_of_nsm_clusters>)
previous_structure=alba_structure
)
# Try to add 1 additional NSM
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=[external_nsm_2])
self.assertEqual(first=str(raise_info.exception), second='The maximum of 50 NSM Arakoon clusters will be exceeded. Amount of clusters that can be deployed for this ALBA Backend: 0')
# Remove the unused NSM clusters again
for nsm_cluster in alba_structure['alba_nsm_clusters'][1][len(current_nsms):]:
for nsm_service in nsm_cluster.nsm_services:
nsm_service.delete()
nsm_service.service.delete()
nsm_cluster.delete()
# Try to add a previously claimed NSM cluster
with self.assertRaises(ValueError) as raise_info:
AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=[external_nsm_1]) # The provided cluster_name to claim has already been claimed
self.assertEqual(first=str(raise_info.exception), second='Some of the provided cluster_names have already been claimed before')
# Add a 2nd NSM cluster
AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=[external_nsm_2])
self.assertEqual(first=1, second=len(alba_backend.abm_cluster.abm_services))
self.assertEqual(first=2, second=len(alba_backend.nsm_clusters))
self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[0].nsm_services))
self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[1].nsm_services))
self.assertIsNone(obj=alba_backend.abm_cluster.abm_services[0].service.storagerouter)
self.assertIsNone(obj=alba_backend.nsm_clusters[0].nsm_services[0].service.storagerouter)
self.assertIsNone(obj=alba_backend.nsm_clusters[1].nsm_services[0].service.storagerouter)
self.assertListEqual(list1=[['add_nsm_host', 'backend_1-nsm_1']], list2=VirtualAlbaBackend.run_log['backend_1-abm'])
for cluster_name, cluster_type in {external_abm_1: 'ABM', external_nsm_1: 'NSM', external_nsm_2: 'NSM'}.iteritems():
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
self.assertDictEqual(d1={'cluster_name': cluster_name,
'cluster_type': cluster_type,
'internal': False,
'in_use': True},
d2=arakoon_installer.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name))