def ovs_4509_validate_arakoon_collapse_test():
"""
Validate arakoon collapse
"""
node_ips = [sr.ip for sr in GeneralStorageRouter.get_storage_routers()]
node_ips.sort()
for node_ip in node_ips:
root_client = SSHClient(node_ip, username='******')
arakoon_clusters = []
for service in ServiceList.get_services():
if service.is_internal is True and service.storagerouter.ip == node_ip and \
service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR):
arakoon_clusters.append(service.name.replace('arakoon-', ''))
for arakoon_cluster in arakoon_clusters:
arakoon_config_path = Configuration.get_configuration_path('/ovs/arakoon/{0}/config'.format(arakoon_cluster))
tlog_location = '/opt/OpenvStorage/db/arakoon/{0}/tlogs'.format(arakoon_cluster)
# read_tlog_dir
with remote(node_ip, [Configuration]) as rem:
config_contents = rem.Configuration.get('/ovs/arakoon/{0}/config'.format(arakoon_cluster), raw=True)
for line in config_contents.splitlines():
if 'tlog_dir' in line:
tlog_location = line.split()[-1]
nr_of_tlogs = TestArakoon.get_nr_of_tlogs_in_folder(root_client, tlog_location)
old_headdb_timestamp = 0
if root_client.file_exists('/'.join([tlog_location, 'head.db'])):
old_headdb_timestamp = root_client.run(['stat', '--format=%Y', tlog_location + '/head.db'])
if nr_of_tlogs <= 2:
benchmark_command = ['arakoon', '--benchmark', '-n_clients', '1', '-max_n', '5_000', '-config', arakoon_config_path]
root_client.run(benchmark_command)
GenericController.collapse_arakoon()
nr_of_tlogs = TestArakoon.get_nr_of_tlogs_in_folder(root_client, tlog_location)
new_headdb_timestamp = root_client.run(['stat', '--format=%Y', tlog_location + '/head.db'])
assert nr_of_tlogs <= 2,\
'Arakoon collapse left {0} tlogs on the environment, expecting less than 2'.format(nr_of_tlogs)
assert old_headdb_timestamp != new_headdb_timestamp,\
'Timestamp of the head_db file was not changed in the process of collapsing tlogs'
def test_arakoon_collapse(self):
"""
Test the Arakoon collapse functionality
"""
# Set up the test
structure = DalHelper.build_dal_structure(
structure={'storagerouters': [1, 2]})
storagerouter_1 = structure['storagerouters'][1]
storagerouter_2 = structure['storagerouters'][2]
MockedSSHClient._run_returns[storagerouter_1.ip] = {}
MockedSSHClient._run_returns[storagerouter_2.ip] = {}
# Make sure we cover all Arakoon cluster types
clusters_to_create = {
ServiceType.ARAKOON_CLUSTER_TYPES.SD: [{
'name': 'unittest-voldrv',
'internal': True,
'success': True
}],
ServiceType.ARAKOON_CLUSTER_TYPES.CFG: [{
'name': 'unittest-cacc',
'internal': True,
'success': True
}],
ServiceType.ARAKOON_CLUSTER_TYPES.FWK: [{
'name': 'unittest-ovsdb',
'internal': True,
'success': False
}],
ServiceType.ARAKOON_CLUSTER_TYPES.ABM: [{
'name': 'unittest-cluster-1-abm',
'internal': True,
'success': False
}, {
'name': 'unittest-random-abm-name',
'internal': False,
'success': True
}],
ServiceType.ARAKOON_CLUSTER_TYPES.NSM: [{
'name': 'unittest-cluster-1-nsm_0',
'internal': True,
'success': True
}]
}
self.assertEqual(
first=sorted(clusters_to_create.keys()),
second=sorted(ServiceType.ARAKOON_CLUSTER_TYPES.keys()),
msg=
'An Arakoon cluster type has been removed or added, please update this test accordingly'
)
# Create all Arakoon clusters and related services
failed_clusters = []
external_clusters = []
successful_clusters = []
for cluster_type, cluster_infos in clusters_to_create.iteritems():
filesystem = cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.CFG
for cluster_info in cluster_infos:
internal = cluster_info['internal']
cluster_name = cluster_info['name']
base_dir = DalHelper.CLUSTER_DIR.format(cluster_name)
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
arakoon_installer.create_cluster(cluster_type=cluster_type,
ip=storagerouter_1.ip,
base_dir=base_dir,
internal=internal)
arakoon_installer.start_cluster()
arakoon_installer.extend_cluster(new_ip=storagerouter_2.ip,
base_dir=base_dir)
service_name = ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)
if cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM:
service_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ALBA_MGR)
elif cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM:
service_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.NS_MGR)
else:
service_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
if internal is True:
DalHelper.create_service(
service_name=service_name,
service_type=service_type,
storagerouter=storagerouter_1,
ports=arakoon_installer.ports[storagerouter_1.ip])
DalHelper.create_service(
service_name=service_name,
service_type=service_type,
storagerouter=storagerouter_2,
ports=arakoon_installer.ports[storagerouter_2.ip])
else:
DalHelper.create_service(service_name=service_name,
service_type=service_type)
external_clusters.append(cluster_name)
continue
if cluster_info['success'] is True:
if filesystem is True:
config_path = ArakoonClusterConfig.CONFIG_FILE.format(
cluster_name)
else:
config_path = Configuration.get_configuration_path(
ArakoonClusterConfig.CONFIG_KEY.format(
cluster_name))
MockedSSHClient._run_returns[storagerouter_1.ip][
'arakoon --collapse-local 1 2 -config {0}'.format(
config_path)] = None
MockedSSHClient._run_returns[storagerouter_2.ip][
'arakoon --collapse-local 2 2 -config {0}'.format(
config_path)] = None
successful_clusters.append(cluster_name)
else: # For successful False clusters we don't emulate the collapse, thus making it fail
failed_clusters.append(cluster_name)
# Start collapse and make it fail for all clusters on StorageRouter 2
SSHClient._raise_exceptions[storagerouter_2.ip] = {
'users': ['ovs'],
'exception': UnableToConnectException('No route to host')
}
GenericController.collapse_arakoon()
# Verify all log messages for each type of cluster
generic_logs = Logger._logs.get('lib', {})
for cluster_name in successful_clusters + failed_clusters + external_clusters:
collect_msg = (
'DEBUG',
'Collecting info for cluster {0}'.format(cluster_name))
unreachable_msg = (
'ERROR',
'Could not collapse any cluster on {0} (not reachable)'.format(
storagerouter_2.name))
end_collapse_msg = (
'DEBUG', 'Collapsing cluster {0} on {1} completed'.format(
cluster_name, storagerouter_1.ip))
start_collapse_msg = ('DEBUG',
'Collapsing cluster {0} on {1}'.format(
cluster_name, storagerouter_1.ip))
failed_collapse_msg = (
'ERROR', 'Collapsing cluster {0} on {1} failed'.format(
cluster_name, storagerouter_1.ip))
messages_to_validate = []
if cluster_name in successful_clusters:
assert_function = self.assertIn
messages_to_validate.append(collect_msg)
messages_to_validate.append(unreachable_msg)
messages_to_validate.append(start_collapse_msg)
messages_to_validate.append(end_collapse_msg)
elif cluster_name in failed_clusters:
assert_function = self.assertIn
messages_to_validate.append(collect_msg)
messages_to_validate.append(unreachable_msg)
messages_to_validate.append(start_collapse_msg)
messages_to_validate.append(failed_collapse_msg)
else:
assert_function = self.assertNotIn
messages_to_validate.append(collect_msg)
messages_to_validate.append(start_collapse_msg)
messages_to_validate.append(end_collapse_msg)
for severity, message in messages_to_validate:
if assert_function == self.assertIn:
assert_message = 'Expected to find log message: {0}'.format(
message)
else:
assert_message = 'Did not expect to find log message: {0}'.format(
message)
assert_function(member=message,
container=generic_logs,
msg=assert_message)
if assert_function == self.assertIn:
self.assertEqual(
first=severity,
second=generic_logs[message],
msg='Log message {0} is of severity {1} expected {2}'.
format(message, generic_logs[message], severity))
# Collapse should always have a 'finished' message since each cluster should be attempted to be collapsed
for general_message in [
'Arakoon collapse started', 'Arakoon collapse finished'
]:
self.assertIn(member=general_message,
container=generic_logs,
msg='Expected to find log message: {0}'.format(
general_message))
def test_collapse():
"""
Test the arakoon collapsing
:return:
"""
ArakoonCollapse.LOGGER.info("Starting validating arakoon collapse")
node_ips = StoragerouterHelper.get_storagerouter_ips()
node_ips.sort()
for node_ip in node_ips:
ArakoonCollapse.LOGGER.info(
"Fetching arakoons on node `{0}`".format(node_ip))
arakoon_clusters = []
root_client = SSHClient(node_ip, username='******')
# fetch arakoon clusters
for service in ServiceList.get_services():
if service.is_internal is True and service.storagerouter.ip == node_ip and \
service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR):
arakoon_clusters.append(
service.name.replace('arakoon-', ''))
# perform collapse
ArakoonCollapse.LOGGER.info(
"Starting arakoon collapse on node `{0}`".format(node_ip))
for arakoon_cluster in arakoon_clusters:
ArakoonCollapse.LOGGER.info(
"Fetching `{0}` arakoon on node `{1}`".format(
arakoon_cluster, node_ip))
arakoon_config_path = Configuration.get_configuration_path(
'/ovs/arakoon/{0}/config'.format(arakoon_cluster))
tlog_location = '/opt/OpenvStorage/db/arakoon/{0}/tlogs'.format(
arakoon_cluster)
# read_tlog_dir
with remote(node_ip, [Configuration]) as rem:
config_contents = rem.Configuration.get(
'/ovs/arakoon/{0}/config'.format(arakoon_cluster),
raw=True)
for line in config_contents.splitlines():
if 'tlog_dir' in line:
tlog_location = line.split()[-1]
nr_of_tlogs = ArakoonCollapse.get_nr_of_tlogs_in_folder(
root_client, tlog_location)
old_headdb_timestamp = 0
if root_client.file_exists('/'.join([tlog_location,
'head.db'])):
old_headdb_timestamp = root_client.run([
'stat', '--format=%Y',
'{0}/{1}'.format(tlog_location, 'head.db')
])
if nr_of_tlogs <= 2:
benchmark_command = [
'arakoon', '--benchmark', '-n_clients', '1', '-max_n',
'5_000', '-config', arakoon_config_path
]
root_client.run(benchmark_command)
ArakoonCollapse.LOGGER.info(
"Collapsing arakoon `{0}` on node `{1}` ...".format(
arakoon_cluster, node_ip))
GenericController.collapse_arakoon()
nr_of_tlogs = ArakoonCollapse.get_nr_of_tlogs_in_folder(
root_client, tlog_location)
new_headdb_timestamp = root_client.run([
'stat', '--format=%Y',
'{0}/{1}'.format(tlog_location, 'head.db')
])
# perform assertion
assert nr_of_tlogs <= 2,\
'Arakoon collapse left {0} tlogs on the environment, expecting less than 2 in `{1}` on node `{1}`'\
.format(nr_of_tlogs, arakoon_cluster, node_ip)
assert old_headdb_timestamp != new_headdb_timestamp,\
'Timestamp of the head_db file was not changed ' \
'in the process of collapsing tlogs of arakoon `{0}` on node `{1}`'\
.format(arakoon_cluster, node_ip)
ArakoonCollapse.LOGGER.info(
"Successfully collapsed arakoon `{0}` on node `{1}`".
format(arakoon_cluster, node_ip))
ArakoonCollapse.LOGGER.info("Finished validating arakoon collapsing")
