def test_cluster_maintenance(self):
"""
Validates whether a cluster can be correctly created
"""
Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]})
structure = Helper.build_service_structure(
{'storagerouters': [1, 2]}
)
storagerouters = structure['storagerouters']
System._machine_id = {storagerouters[1].ip: '1',
storagerouters[2].ip: '2'}
# Create new cluster
mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir = mountpoint + '/test_create_cluster'
info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir)
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
'files': []},
'db': {'dirs': {},
'files': []}},
'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[1].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))
ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata'])
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
# Extending cluster
mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir2 = mountpoint + '/test_extend_cluster'
ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2)
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
'files': []},
'db': {'dirs': {},
'files': []}},
'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001
),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
SSHClient._run_returns[catchup_command] = None
SSHClient._run_recordings = []
ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False)
self.assertIn(catchup_command, SSHClient._run_recordings)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
# Shrinking cluster
ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test')
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
def promote_or_demote_node(node_action, cluster_ip=None, execute_rollback=False):
"""
Promotes or demotes the local node
:param node_action: Demote or promote
:type node_action: str
:param cluster_ip: IP of node to promote or demote
:type cluster_ip: str
:param execute_rollback: In case of failure revert the changes made
:type execute_rollback: bool
:return: None
"""
if node_action not in ('promote', 'demote'):
raise ValueError('Nodes can only be promoted or demoted')
Toolbox.log(logger=NodeTypeController._logger, messages='Open vStorage Setup - {0}'.format(node_action.capitalize()), boxed=True)
try:
Toolbox.log(logger=NodeTypeController._logger, messages='Collecting information', title=True)
machine_id = System.get_my_machine_id()
if Configuration.get('/ovs/framework/hosts/{0}/setupcompleted'.format(machine_id)) is False:
raise RuntimeError('No local OVS setup found.')
if cluster_ip and not re.match(Toolbox.regex_ip, cluster_ip):
raise RuntimeError('Incorrect IP provided ({0})'.format(cluster_ip))
if cluster_ip:
client = SSHClient(endpoint=cluster_ip)
machine_id = System.get_my_machine_id(client)
node_type = Configuration.get('/ovs/framework/hosts/{0}/type'.format(machine_id))
if node_action == 'promote' and node_type == 'MASTER':
raise RuntimeError('This node is already master.')
elif node_action == 'demote' and node_type == 'EXTRA':
raise RuntimeError('This node should be a master.')
elif node_type not in ['MASTER', 'EXTRA']:
raise RuntimeError('This node is not correctly configured.')
master_ip = None
offline_nodes = []
online = True
target_client = None
if node_action == 'demote' and cluster_ip: # Demote an offline node
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.lib.storagedriver import StorageDriverController
ip = cluster_ip
unique_id = None
ip_client_map = {}
for storage_router in StorageRouterList.get_storagerouters():
try:
client = SSHClient(storage_router.ip, username='******')
if storage_router.node_type == 'MASTER':
master_ip = storage_router.ip
ip_client_map[storage_router.ip] = client
except UnableToConnectException:
if storage_router.ip == cluster_ip:
online = False
unique_id = storage_router.machine_id
StorageDriverController.mark_offline(storagerouter_guid=storage_router.guid)
offline_nodes.append(storage_router)
if online is True:
raise RuntimeError("If the node is online, please use 'ovs setup demote' executed on the node you wish to demote")
if master_ip is None:
raise RuntimeError('Failed to retrieve another responsive MASTER node')
else:
target_password = Toolbox.ask_validate_password(ip='127.0.0.1', logger=NodeTypeController._logger)
target_client = SSHClient('127.0.0.1', username='******', password=target_password)
unique_id = System.get_my_machine_id(target_client)
ip = Configuration.get('/ovs/framework/hosts/{0}/ip'.format(unique_id))
storagerouter_info = NodeTypeController.retrieve_storagerouter_info_via_host(ip=target_client.ip, password=target_password)
node_ips = [sr_info['ip'] for sr_info in storagerouter_info.itervalues()]
master_node_ips = [sr_info['ip'] for sr_info in storagerouter_info.itervalues() if sr_info['type'] == 'master' and sr_info['ip'] != ip]
if len(master_node_ips) == 0:
if node_action == 'promote':
raise RuntimeError('No master node could be found')
else:
raise RuntimeError('It is not possible to remove the only master')
master_ip = master_node_ips[0]
ip_client_map = dict((node_ip, SSHClient(node_ip, username='******')) for node_ip in node_ips)
if node_action == 'demote':
for cluster_name in Configuration.list('/ovs/arakoon'):
config = ArakoonClusterConfig(cluster_name, False)
config.load_config()
arakoon_client = ArakoonInstaller.build_client(config)
metadata = json.loads(arakoon_client.get(ArakoonInstaller.METADATA_KEY))
if len(config.nodes) == 1 and config.nodes[0].ip == ip and metadata.get('internal') is True:
raise RuntimeError('Demote is not supported when single node Arakoon cluster(s) are present on the node to be demoted.')
configure_rabbitmq = Toolbox.is_service_internally_managed(service='rabbitmq')
configure_memcached = Toolbox.is_service_internally_managed(service='memcached')
if node_action == 'promote':
try:
NodeTypeController.promote_node(cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
configure_memcached=configure_memcached,
configure_rabbitmq=configure_rabbitmq)
except Exception:
if execute_rollback is True:
NodeTypeController.demote_node(cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
unconfigure_memcached=configure_memcached,
unconfigure_rabbitmq=configure_rabbitmq,
offline_nodes=offline_nodes)
elif target_client is not None:
target_client.file_write('/tmp/ovs_rollback', 'demote')
raise
else:
try:
NodeTypeController.demote_node(cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
unconfigure_memcached=configure_memcached,
unconfigure_rabbitmq=configure_rabbitmq,
offline_nodes=offline_nodes)
except Exception:
if execute_rollback is True:
NodeTypeController.promote_node(cluster_ip=ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=unique_id,
configure_memcached=configure_memcached,
configure_rabbitmq=configure_rabbitmq)
elif target_client is not None:
target_client.file_write('/tmp/ovs_rollback', 'promote')
raise
Toolbox.log(logger=NodeTypeController._logger, messages='\n')
Toolbox.log(logger=NodeTypeController._logger, messages='{0} complete.'.format(node_action.capitalize()), boxed=True)
except Exception as exception:
Toolbox.log(logger=NodeTypeController._logger, messages='\n')
Toolbox.log(logger=NodeTypeController._logger, messages=['An unexpected error occurred:', str(exception)], boxed=True, loglevel='exception')
sys.exit(1)
except KeyboardInterrupt:
Toolbox.log(logger=NodeTypeController._logger, messages='\n')
Toolbox.log(logger=NodeTypeController._logger,
messages='This setup was aborted. Open vStorage may be in an inconsistent state, make sure to validate the installation.',
boxed=True,
loglevel='error')
sys.exit(1)
def services_running(self, target):
"""
Check all services are running
:param target: Target to check
:return: Boolean
"""
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target == 'config':
self.log_message(target, 'Testing configuration store...', 0)
from ovs.extensions.generic.configuration import Configuration
try:
Configuration.list('/')
except Exception as ex:
self.log_message(target, ' Error during configuration store test: {0}'.format(ex), 2)
return False
if Configuration.get_store() == 'arakoon':
from ovs.extensions.db.arakoon.configuration import ArakoonConfiguration
from ovs.extensions.db.arakoon.ArakoonInstaller import ArakoonInstaller, ArakoonClusterConfig
from ovs.extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee
with open(ArakoonConfiguration.CACC_LOCATION) as config_file:
contents = config_file.read()
config = ArakoonClusterConfig(cluster_id='cacc', filesystem=True)
config.read_config(contents)
client = ArakoonInstaller.build_client(config)
contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY, consistency=NoGuarantee())
if Watcher.LOG_CONTENTS != contents:
try:
config.read_config(contents) # Validate whether the contents are not corrupt
except Exception as ex:
self.log_message(target, ' Configuration stored in configuration store seems to be corrupt: {0}'.format(ex), 2)
return False
temp_filename = '{0}~'.format(ArakoonConfiguration.CACC_LOCATION)
with open(temp_filename, 'w') as config_file:
config_file.write(contents)
config_file.flush()
os.fsync(config_file)
os.rename(temp_filename, ArakoonConfiguration.CACC_LOCATION)
Watcher.LOG_CONTENTS = contents
self.log_message(target, ' Configuration store OK', 0)
return True
if target == 'framework':
# Volatile
self.log_message(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(target, ' Error during volatile store test: {0}'.format(message), 2)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target, ' Volatile store not working correctly', 2)
return False
self.log_message(target, ' Volatile store OK after {0} tries'.format(tries), 0)
# Persistent
self.log_message(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
persistent = PersistentFactory.get_client()
persistent.set(key, value)
if persistent.get(key) == value:
persistent.delete(key)
break
persistent.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(target, ' Error during persistent store test: {0}'.format(message), 2)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target, ' Persistent store not working correctly', 2)
return False
self.log_message(target, ' Persistent store OK after {0} tries'.format(tries), 0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
self.log_message(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.generic.configuration import Configuration
from ovs.extensions.storage.persistent.pyrakoonstore import PyrakoonStore
cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|voldrv'))
client = PyrakoonStore(cluster=cluster_name)
client.set(key, value)
if client.get(key) == value:
client.delete(key)
break
client.delete(key)
except Exception as message:
self.log_message(target, ' Error during arakoon (voldrv) test: {0}'.format(message), 2)
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target, ' Arakoon (voldrv) not working correctly', 2)
return False
self.log_message(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
self.log_message(target, 'Test rabbitMQ...', 0)
import pika
from ovs.extensions.generic.configuration import Configuration
messagequeue = Configuration.get('/ovs/framework/messagequeue')
rmq_servers = messagequeue['endpoints']
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(messagequeue['protocol'],
messagequeue['user'],
messagequeue['password'],
server)
connection = pika.BlockingConnection(pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish('', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain', delivery_mode=1))
connection.close()
good_node = True
except Exception as message:
self.log_message(target, ' Error during rabbitMQ test on node {0}: {1}'.format(server, message), 2)
if good_node is False:
self.log_message(target, ' No working rabbitMQ node could be found', 2)
return False
self.log_message(target, ' RabbitMQ test OK', 0)
self.log_message(target, 'All tests OK', 0)
return True
except Exception as ex:
self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
def test_cluster_maintenance(self):
"""
Validates whether a cluster can be correctly created
"""
Configuration.set('/ovs/framework/hosts/1/ports',
{'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/hosts/2/ports',
{'arakoon': [20000, 20100]})
structure = Helper.build_service_structure({'storagerouters': [1, 2]})
storagerouters = structure['storagerouters']
System._machine_id = {
storagerouters[1].ip: '1',
storagerouters[2].ip: '2'
}
# Create new cluster
mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir = mountpoint + '/test_create_cluster'
info = ArakoonInstaller.create_cluster(
'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
storagerouters[1].ip, base_dir)
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {
'tlogs': {
'dirs': {},
'files': []
},
'db': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'1', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.start_cluster('test',
storagerouters[1].ip,
filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[1].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))
ArakoonInstaller.claim_cluster('test',
storagerouters[1].ip,
filesystem=False,
metadata=info['metadata'])
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
# Extending cluster
mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir2 = mountpoint + '/test_extend_cluster'
ArakoonInstaller.extend_cluster(storagerouters[1].ip,
storagerouters[2].ip, 'test',
base_dir2)
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {
'tlogs': {
'dirs': {},
'files': []
},
'db': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n{2}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'1,2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
SSHClient._run_returns[catchup_command] = None
SSHClient._run_recordings = []
ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip],
storagerouters[2].ip,
filesystem=False)
self.assertIn(catchup_command, SSHClient._run_recordings)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
# Shrinking cluster
ArakoonInstaller.shrink_cluster(storagerouters[1].ip,
storagerouters[2].ip, 'test')
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip],
filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
def services_running(self, target):
"""
Check all services are running
:param target: Target to check
:return: Boolean
"""
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target == 'config':
self.log_message(target, 'Testing configuration store...', 0)
from ovs.extensions.generic.configuration import Configuration
try:
Configuration.list('/')
except Exception as ex:
self.log_message(
target,
' Error during configuration store test: {0}'.format(
ex), 2)
return False
if Configuration.get_store() == 'arakoon':
from ovs.extensions.db.arakoon.configuration import ArakoonConfiguration
from ovs.extensions.db.arakoon.ArakoonInstaller import ArakoonInstaller, ArakoonClusterConfig
from ovs.extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee
with open(
ArakoonConfiguration.CACC_LOCATION) as config_file:
contents = config_file.read()
config = ArakoonClusterConfig(cluster_id='cacc',
filesystem=True)
config.read_config(contents)
client = ArakoonInstaller.build_client(config)
contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY,
consistency=NoGuarantee())
if Watcher.LOG_CONTENTS != contents:
try:
config.read_config(
contents
) # Validate whether the contents are not corrupt
except Exception as ex:
self.log_message(
target,
' Configuration stored in configuration store seems to be corrupt: {0}'
.format(ex), 2)
return False
temp_filename = '{0}~'.format(
ArakoonConfiguration.CACC_LOCATION)
with open(temp_filename, 'w') as config_file:
config_file.write(contents)
config_file.flush()
os.fsync(config_file)
os.rename(temp_filename,
ArakoonConfiguration.CACC_LOCATION)
Watcher.LOG_CONTENTS = contents
self.log_message(target, ' Configuration store OK', 0)
return True
if target == 'framework':
# Volatile
self.log_message(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during volatile store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target,
' Volatile store not working correctly',
2)
return False
self.log_message(
target,
' Volatile store OK after {0} tries'.format(tries), 0)
# Persistent
self.log_message(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
persistent = PersistentFactory.get_client()
persistent.set(key, value)
if persistent.get(key) == value:
persistent.delete(key)
break
persistent.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during persistent store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Persistent store not working correctly', 2)
return False
self.log_message(
target,
' Persistent store OK after {0} tries'.format(tries), 0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
self.log_message(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.generic.configuration import Configuration
from ovs.extensions.storage.persistent.pyrakoonstore import PyrakoonStore
cluster_name = str(
Configuration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
client = PyrakoonStore(cluster=cluster_name)
client.set(key, value)
if client.get(key) == value:
client.delete(key)
break
client.delete(key)
except Exception as message:
self.log_message(
target,
' Error during arakoon (voldrv) test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Arakoon (voldrv) not working correctly', 2)
return False
self.log_message(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
self.log_message(target, 'Test rabbitMQ...', 0)
import pika
from ovs.extensions.generic.configuration import Configuration
messagequeue = Configuration.get('/ovs/framework/messagequeue')
rmq_servers = messagequeue['endpoints']
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(
messagequeue['protocol'], messagequeue['user'],
messagequeue['password'], server)
connection = pika.BlockingConnection(
pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish(
'', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain',
delivery_mode=1))
connection.close()
good_node = True
except Exception as message:
self.log_message(
target,
' Error during rabbitMQ test on node {0}: {1}'.
format(server, message), 2)
if good_node is False:
self.log_message(
target, ' No working rabbitMQ node could be found', 2)
return False
self.log_message(target, ' RabbitMQ test OK', 0)
self.log_message(target, 'All tests OK', 0)
return True
except Exception as ex:
self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
