def monitor_services(self):
# type: () -> None
"""
Monitor the local services
:return: None
:rtype: NoneType
"""
try:
grep = ['egrep "{0}"'.format(prefix) for prefix in self._monitor_prefixes]
previous_output = None
while True:
# Gather service states
running_services = {}
non_running_services = {}
longest_service_name = 0
for service_name in check_output('systemctl list-unit-files --full --type=service --no-legend --no-pager | {0} | tr -s " " | cut -d " " -f 1'.format(' | '.join(grep)), shell=True).splitlines():
try:
service_state = check_output('systemctl is-active {0}'.format(service_name), shell=True).strip()
except CalledProcessError as cpe:
service_state = cpe.output.strip()
service_name = service_name.replace('.service', '')
if service_state == 'active':
service_pid = check_output('systemctl show {0} --property=MainPID'.format(service_name), shell=True).strip().split('=')[1]
running_services[service_name] = (service_state, service_pid)
else:
non_running_services[service_name] = service_state
if len(service_name) > longest_service_name:
longest_service_name = len(service_name)
# Put service states in list
output = ['Running processes',
'=================\n']
for service_name in sorted(running_services, key=lambda service: ExtensionsToolbox.advanced_sort(service, '_')):
output.append('{0} {1} {2} {3}'.format(service_name, ' ' * (longest_service_name - len(service_name)), running_services[service_name][0], running_services[service_name][1]))
output.extend(['\n\nNon-running processes',
'=====================\n'])
for service_name in sorted(non_running_services, key=lambda service: ExtensionsToolbox.advanced_sort(service, '_')):
output.append('{0} {1} {2}'.format(service_name, ' ' * (longest_service_name - len(service_name)), non_running_services[service_name]))
# Print service states (only if changes)
if previous_output != output:
print '\x1b[2J\x1b[H'
for line in output:
print line
previous_output = list(output)
time.sleep(1)
except KeyboardInterrupt:
pass
def check_arakoon_ports(cls, result_handler):
"""
Verifies that the Arakoon clusters still respond to connections
:param result_handler: logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:return: None
:rtype: NoneType
"""
arakoon_clusters = cls._get_arakoon_clusters(result_handler)
result_handler.info('Starting Arakoon ports test.',
add_to_result=False)
result_handler.info(
'Retrieving all collapsing statistics. This might take a while',
add_to_result=False)
start = time.time()
arakoon_stats = cls._get_port_connections(result_handler,
arakoon_clusters)
result_handler.info(
'Retrieving all collapsing statistics succeeded (duration: {0})'.
format(time.time() - start),
add_to_result=False)
for cluster_type, clusters in arakoon_stats.iteritems():
result_handler.info(
'Testing the collapse of {0} Arakoons'.format(cluster_type),
add_to_result=False)
for cluster in clusters:
cluster_name = cluster['cluster_name']
connection_result = cluster['connection_result']
connection_result = OrderedDict(
sorted(connection_result.items(),
key=lambda item: ExtensionsToolbox.advanced_sort(
item[0].ip, separator='.')))
for node, stats in connection_result.iteritems():
identifier_log = 'Arakoon cluster {0} on node {1}'.format(
cluster_name, node.ip)
if len(stats['errors']) > 0:
# Determine where issues were found
for step, exception in stats['errors']:
if step == 'test_connection':
try:
# Raise the thrown exception
raise exception
except Exception:
message = 'Connection to {0} could not be established due to an unhandled exception.'.format(
identifier_log)
cls.logger.exception(message)
result_handler.exception(
message,
code=ErrorCodes.unhandled_exception)
continue
if stats['result'] is True:
result_handler.success(
'Connection established to {0}'.format(
identifier_log),
code=ErrorCodes.arakoon_connection_ok)
else:
result_handler.failure(
'Connection could not be established to {0}'.
format(identifier_log),
code=ErrorCodes.arakoon_connection_failure)
def mds_checkup():
"""
Validates the current MDS setup/configuration and takes actions where required
Actions:
* Verify which StorageRouters are available
* Make mapping between vPools and its StorageRouters
* For each vPool make sure every StorageRouter has at least 1 MDS service with capacity available
* For each vPool retrieve the optimal configuration and store it for each StorageDriver
* For each vPool run an ensure safety for all vDisks
:raises RuntimeError: When ensure safety fails for any vDisk
:return: None
:rtype: NoneType
"""
MDSServiceController._logger.info('Started')
# Verify StorageRouter availability
root_client_cache = {}
storagerouters = StorageRouterList.get_storagerouters()
storagerouters.sort(key=lambda _sr: ExtensionsToolbox.advanced_sort(
element=_sr.ip, separator='.'))
offline_nodes = []
for storagerouter in storagerouters:
try:
root_client = SSHClient(endpoint=storagerouter,
username='******')
MDSServiceController._logger.debug(
'StorageRouter {0} - ONLINE'.format(storagerouter.name))
except UnableToConnectException:
root_client = None
offline_nodes.append(storagerouter)
MDSServiceController._logger.error(
'StorageRouter {0} - OFFLINE'.format(storagerouter.name))
root_client_cache[storagerouter] = root_client
# Create mapping per vPool and its StorageRouters
mds_dict = collections.OrderedDict()
for vpool in sorted(VPoolList.get_vpools(), key=lambda k: k.name):
MDSServiceController._logger.info('vPool {0}'.format(vpool.name))
mds_dict[vpool] = {}
# Loop all StorageDrivers and add StorageDriver to mapping
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': root_client_cache.get(storagerouter),
'services': [],
'storagedriver': storagedriver
}
# Loop all MDS Services and append services to appropriate vPool / StorageRouter combo
mds_services = vpool.mds_services
mds_services.sort(
key=lambda _mds_service: ExtensionsToolbox.advanced_sort(
element=_mds_service.service.storagerouter.ip,
separator='.'))
for mds_service in mds_services:
service = mds_service.service
storagerouter = service.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': root_client_cache.get(storagerouter),
'services': [],
'storagedriver': None
}
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Service on port {2}'.
format(vpool.name, storagerouter.name, service.ports[0]))
mds_dict[vpool][storagerouter]['services'].append(mds_service)
failures = []
for vpool, storagerouter_info in mds_dict.iteritems():
# Make sure there's at least 1 MDS on every StorageRouter that's not overloaded
# Remove all MDS Services which have been manually marked for removal (by setting its capacity to 0)
max_load = Configuration.get(
'/ovs/vpools/{0}/mds_config|mds_maxload'.format(vpool.guid))
for storagerouter in sorted(storagerouter_info,
key=lambda k: k.ip):
total_load = 0.0
root_client = mds_dict[vpool][storagerouter]['client']
mds_services = mds_dict[vpool][storagerouter]['services']
for mds_service in list(
sorted(mds_services, key=lambda k: k.number)):
port = mds_service.service.ports[0]
number = mds_service.number
# Manual intervention required here in order for the MDS to be cleaned up
# @TODO: Remove this and make a dynamic calculation to check which MDSes to remove
if mds_service.capacity == 0 and len(
mds_service.vdisks_guids) == 0:
MDSServiceController._logger.warning(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: Removing'
.format(vpool.name, storagerouter.name, number,
port))
try:
MDSServiceController.remove_mds_service(
mds_service=mds_service,
reconfigure=True,
allow_offline=root_client is None)
except Exception:
MDSServiceController._logger.exception(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: Failed to remove'
.format(vpool.name, storagerouter.name, number,
port))
mds_services.remove(mds_service)
else:
_, next_load = MDSServiceController.get_mds_load(
mds_service=mds_service)
if next_load == float('inf'):
total_load = sys.maxint * -1 # Cast to lowest possible value if any MDS service capacity is set to infinity
else:
total_load += next_load
if next_load < max_load:
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: Capacity available - Load at {4}%'
.format(vpool.name, storagerouter.name, number,
port, next_load))
else:
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - MDS Service {2} on port {3}: No capacity available - Load at {4}%'
.format(vpool.name, storagerouter.name, number,
port, next_load))
if total_load >= max_load * len(mds_services):
mds_services_to_add = int(
math.ceil((total_load - max_load * len(mds_services)) /
max_load))
MDSServiceController._logger.info(
'vPool {0} - StorageRouter {1} - Average load per service {2:.2f}% - Max load per service {3:.2f}% - {4} MDS service{5} will be added'
.format(vpool.name, storagerouter.name,
total_load / len(mds_services), max_load,
mds_services_to_add,
'' if mds_services_to_add == 1 else 's'))
for _ in range(mds_services_to_add):
MDSServiceController._logger.info(
'vPool {0} - StorageRouter {1} - Adding new MDS Service'
.format(vpool.name, storagerouter.name))
try:
mds_services.append(
MDSServiceController.prepare_mds_service(
storagerouter=storagerouter, vpool=vpool))
except Exception:
MDSServiceController._logger.exception(
'vPool {0} - StorageRouter {1} - Failed to create new MDS Service'
.format(vpool.name, storagerouter.name))
# After potentially having added new MDSes, retrieve the optimal configuration
mds_config_set = {}
try:
mds_config_set = MDSServiceController.get_mds_storagedriver_config_set(
vpool=vpool, offline_nodes=offline_nodes)
MDSServiceController._logger.debug(
'vPool {0} - Optimal configuration {1}'.format(
vpool.name, mds_config_set))
except (NotFoundException, RuntimeError):
MDSServiceController._logger.exception(
'vPool {0} - Failed to retrieve the optimal configuration'.
format(vpool.name))
# Apply the optimal MDS configuration per StorageDriver
for storagerouter in sorted(storagerouter_info,
key=lambda k: k.ip):
root_client = mds_dict[vpool][storagerouter]['client']
storagedriver = mds_dict[vpool][storagerouter]['storagedriver']
if storagedriver is None:
MDSServiceController._logger.critical(
'vPool {0} - StorageRouter {1} - No matching StorageDriver found'
.format(vpool.name, storagerouter.name))
continue
if storagerouter.guid not in mds_config_set:
MDSServiceController._logger.critical(
'vPool {0} - StorageRouter {1} - Not marked as offline, but could not retrieve an optimal MDS config'
.format(vpool.name, storagerouter.name))
continue
if root_client is None:
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Marked as offline, not setting optimal MDS configuration'
.format(vpool.name, storagerouter.name))
continue
storagedriver_config = StorageDriverConfiguration(
vpool_guid=vpool.guid,
storagedriver_id=storagedriver.storagedriver_id)
if storagedriver_config.config_missing is False:
optimal_mds_config = mds_config_set[storagerouter.guid]
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Storing optimal MDS configuration: {2}'
.format(vpool.name, storagerouter.name,
optimal_mds_config))
# Filesystem section in StorageDriver configuration are all parameters used for vDisks created directly on the filesystem
# So when a vDisk gets created on the filesystem, these MDSes will be assigned to them
storagedriver_config.configure_filesystem(
fs_metadata_backend_mds_nodes=optimal_mds_config)
storagedriver_config.save(root_client)
# Execute a safety check, making sure the master/slave configuration is optimal.
MDSServiceController._logger.info(
'vPool {0} - Ensuring safety for all vDisks'.format(
vpool.name))
for vdisk in vpool.vdisks:
try:
MDSServiceController.ensure_safety(vdisk_guid=vdisk.guid)
except Exception:
message = 'Ensure safety for vDisk {0} with guid {1} failed'.format(
vdisk.name, vdisk.guid)
MDSServiceController._logger.exception(message)
failures.append(message)
if len(failures) > 0:
raise RuntimeError('\n - ' + '\n - '.join(failures))
MDSServiceController._logger.info('Finished')
def refresh_package_information():
"""
Retrieve and store the package information of all StorageRouters
:return: None
"""
GenericController._logger.info('Updating package information')
client_map = {}
prerequisites = []
package_info_cluster = {}
all_storagerouters = StorageRouterList.get_storagerouters()
all_storagerouters.sort(key=lambda sr: ExtensionsToolbox.advanced_sort(
element=sr.ip, separator='.'))
for storagerouter in all_storagerouters:
package_info_cluster[storagerouter.ip] = {}
try:
# We make use of these clients in Threads --> cached = False
client_map[storagerouter] = SSHClient(endpoint=storagerouter,
username='******',
cached=False)
except (NotAuthenticatedException, UnableToConnectException):
GenericController._logger.warning(
'StorageRouter {0} is inaccessible'.format(
storagerouter.ip))
prerequisites.append(['node_down', storagerouter.name])
package_info_cluster[storagerouter.ip]['errors'] = [
'StorageRouter {0} is inaccessible'.format(
storagerouter.name)
]
# Retrieve for each StorageRouter in the cluster the installed and candidate versions of related packages
# This also validates whether all required packages have been installed
GenericController._logger.debug(
'Retrieving package information for the cluster')
threads = []
for storagerouter, client in client_map.iteritems():
for fct in Toolbox.fetch_hooks(
component='update',
sub_component='get_package_update_info_cluster'):
thread = Thread(target=fct,
args=(client, package_info_cluster))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
# Retrieve the related downtime / service restart information
GenericController._logger.debug(
'Retrieving update information for the cluster')
update_info_cluster = {}
for storagerouter, client in client_map.iteritems():
update_info_cluster[storagerouter.ip] = {
'errors':
package_info_cluster[storagerouter.ip].get('errors', [])
}
for fct in Toolbox.fetch_hooks(
component='update',
sub_component='get_update_info_cluster'):
fct(client, update_info_cluster,
package_info_cluster[storagerouter.ip])
# Retrieve the update information for plugins (eg: ALBA, iSCSI)
GenericController._logger.debug(
'Retrieving package and update information for the plugins')
threads = []
update_info_plugin = {}
for fct in Toolbox.fetch_hooks('update', 'get_update_info_plugin'):
thread = Thread(target=fct, args=(update_info_plugin, ))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
# Add the prerequisites
if len(prerequisites) > 0:
for ip, component_info in update_info_cluster.iteritems():
if PackageFactory.COMP_FWK in component_info:
component_info[PackageFactory.COMP_FWK][
'prerequisites'].extend(prerequisites)
# Store information in model and collect errors for OVS cluster
errors = set()
for storagerouter in all_storagerouters:
GenericController._logger.debug(
'Storing update information for StorageRouter {0}'.format(
storagerouter.ip))
update_info = update_info_cluster.get(storagerouter.ip, {})
# Remove the errors from the update information
sr_errors = update_info.pop('errors', [])
if len(sr_errors) > 0:
errors.update([
'{0}: {1}'.format(storagerouter.ip, error)
for error in sr_errors
])
update_info = {
} # If any error occurred, we store no update information for this StorageRouter
# Remove the components without updates from the update information
update_info_copy = copy.deepcopy(update_info)
for component, info in update_info_copy.iteritems():
if len(info['packages']) == 0:
update_info.pop(component)
# Store the update information
storagerouter.package_information = update_info
storagerouter.save()
# Collect errors for plugins
for ip, plugin_errors in update_info_plugin.iteritems():
if len(plugin_errors) > 0:
errors.update(
['{0}: {1}'.format(ip, error) for error in plugin_errors])
if len(errors) > 0:
raise Exception('\n - {0}'.format('\n - '.join(errors)))
GenericController._logger.info('Finished updating package information')
def _get_mds_information(vpools=None):
# type: (Optional[List[VPool]]) -> Tuple[collections.OrderedDict, List[StorageRouter]]
"""
Retrieve a complete overview of all storagerouters and their mds layout
:param vpools: VPools to get the overview for
:type vpools: List[VPool]
:return: - An overview with the vpool as keys, storagerouter - client, services and storagedriver map
- All storagerouters that were offline
:rtype: Tuple[collection.OrderedDict, List[StorageRouter]]
"""
# Verify StorageRouter availability
if vpools is None:
vpools = VPoolList.get_vpools()
root_client_cache = {}
storagerouters = StorageRouterList.get_storagerouters()
storagerouters.sort(key=lambda _sr: ExtensionsToolbox.advanced_sort(
element=_sr.ip, separator='.'))
offline_nodes = []
for storagerouter in storagerouters:
try:
root_client = SSHClient(endpoint=storagerouter,
username='******')
MDSServiceController._logger.debug(
'StorageRouter {0} - ONLINE'.format(storagerouter.name))
except UnableToConnectException:
root_client = None
offline_nodes.append(storagerouter)
MDSServiceController._logger.error(
'StorageRouter {0} - OFFLINE'.format(storagerouter.name))
root_client_cache[storagerouter] = root_client
# Create mapping per vPool and its StorageRouters
mds_dict = collections.OrderedDict()
for vpool in sorted(vpools, key=lambda k: k.name):
MDSServiceController._logger.info('vPool {0}'.format(vpool.name))
mds_dict[vpool] = {}
# Loop all StorageDrivers and add StorageDriver to mapping
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': root_client_cache.get(storagerouter),
'services': [],
'storagedriver': storagedriver
}
# Loop all MDS Services and append services to appropriate vPool / StorageRouter combo
mds_services = vpool.mds_services
mds_services.sort(
key=lambda _mds_service: ExtensionsToolbox.advanced_sort(
element=_mds_service.service.storagerouter.ip,
separator='.'))
for mds_service in mds_services:
service = mds_service.service
storagerouter = service.storagerouter
if storagerouter not in mds_dict[vpool]:
mds_dict[vpool][storagerouter] = {
'client': root_client_cache.get(storagerouter),
'services': [],
'storagedriver': None
}
MDSServiceController._logger.debug(
'vPool {0} - StorageRouter {1} - Service on port {2}'.
format(vpool.name, storagerouter.name, service.ports[0]))
mds_dict[vpool][storagerouter]['services'].append(mds_service)
return mds_dict, offline_nodes
def check_arakoon_fd(cls, result_handler, fd_limit=30, passed_connections=None):
"""
Checks all current open tcp file descriptors for all Arakoon clusters in the OVS cluster
Will raise warnings when these reach a certain threshold
:param result_handler: Logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:param fd_limit: Threshold for the number number of tcp connections for which to start logging warnings
:type fd_limit: int
:param passed_connections: checked TCP connections
:type passed_connections: list
:return: None
:rtype: NoneType
"""
if passed_connections is None:
passed_connections = ['ESTABLISHED', 'TIME_WAIT']
warning_threshold = fd_limit * 80 / 100
error_threshold = fd_limit * 95 / 100
result_handler.info('Starting Arakoon integrity test', add_to_result=False)
arakoon_clusters = cls._get_arakoon_clusters(result_handler)
start = time.time()
arakoon_fd_results = cls._get_filedescriptors(result_handler, arakoon_clusters)
result_handler.info('Retrieving all file descriptor information succeeded (duration: {0})'.format(time.time() - start), add_to_result=False)
for cluster_type, clusters in arakoon_fd_results.iteritems():
result_handler.info('Checking the file descriptors of {0} Arakoons'.format(cluster_type), add_to_result=False)
for cluster in clusters:
cluster_name = cluster['cluster_name']
fd_result = cluster['fd_result']
fd_result = OrderedDict(sorted(fd_result.items(), key=lambda item: ExtensionsToolbox.advanced_sort(item[0].ip, separator='.')))
for node, stats in fd_result.iteritems():
identifier_log = 'Arakoon cluster {0} on node {1}'.format(cluster_name, node.ip)
if len(stats['errors']) > 0:
# Determine where issues were found
for step, exception in stats['errors']:
if step == 'build_client':
try:
# Raise the thrown exception
raise exception
except TimeOutException:
result_handler.warning('Connection to {0} has timed out'.format(identifier_log), code=ErrorCodes.ssh_connection_time)
except (socket.error, UnableToConnectException):
result_handler.failure(
'Connection to {0} could not be established'.format(identifier_log), code=ErrorCodes.ssh_connection_fail)
except NotAuthenticatedException:
result_handler.skip('Connection to {0} could not be authenticated. This node has no access to the Arakoon node.'.format(identifier_log),
code=ErrorCodes.ssh_connection_authentication)
except Exception:
message = 'Connection to {0} could not be established due to an unhandled exception.'.format(identifier_log)
cls.logger.exception(message)
result_handler.exception(message, code=ErrorCodes.unhandled_exception)
elif step == 'lsof':
try:
raise exception
except Exception:
message = 'Unable to list the file descriptors for {0}'.format(identifier_log)
cls.logger.exception(message)
result_handler.exception(message, ErrorCodes.unhandled_exception)
continue
fds = stats['result']['fds']
filtered_fds = [i for i in fds if i.split()[-1].strip('(').strip(')') in passed_connections]
if len(filtered_fds) >= warning_threshold:
if len(filtered_fds) >= error_threshold:
result_handler.warning('Number of TCP connections exceeded the 95% warning threshold for {0}, ({1}/{2})'.format(identifier_log, len(filtered_fds), fd_limit),
code=ErrorCodes.arakoon_fd_95)
else:
result_handler.warning('Number of TCP connections exceeded the 80% warning threshold for {0}, ({1}/{2})'.format(identifier_log, len(filtered_fds), fd_limit),
code=ErrorCodes.arakoon_fd_80)
else:
result_handler.success('Number of TCP connections for {0} is healthy ({1}/{2})'.format(identifier_log, len(filtered_fds), fd_limit),
code=ErrorCodes.arakoon_fd_ok)
def check_collapse(cls, result_handler, max_collapse_age=3, min_tlx_amount=10):
"""
Verifies collapsing has occurred for all Arakoons
:param result_handler: logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:param max_collapse_age: tlx files may not be longer than x days
:type max_collapse_age: int
:param min_tlx_amount: Minimum amount of tlxes before making collapsing mandatory (defaults to 10)
:type min_tlx_amount: int
:return: None
:rtype: NoneType
"""
arakoon_clusters = cls._get_arakoon_clusters(result_handler)
result_handler.info('Starting Arakoon collapse test', add_to_result=False)
max_age_seconds = timedelta(days=max_collapse_age).total_seconds()
result_handler.info('Retrieving all collapsing statistics. This might take a while', add_to_result=False)
start = time.time()
arakoon_stats = cls._retrieve_stats(result_handler, arakoon_clusters)
result_handler.info('Retrieving all collapsing statistics succeeded (duration: {0})'.format(time.time() - start), add_to_result=False)
for cluster_type, clusters in arakoon_stats.iteritems():
result_handler.info('Testing the collapse of {0} Arakoons'.format(cluster_type), add_to_result=False)
for cluster in clusters:
cluster_name = cluster['cluster_name']
collapse_result = cluster['collapse_result']
collapse_result = OrderedDict(sorted(collapse_result.items(), key=lambda item: ExtensionsToolbox.advanced_sort(item[0].ip, separator='.')))
for node, stats in collapse_result.iteritems():
identifier_log = 'Arakoon cluster {0} on node {1}'.format(cluster_name, node.ip)
if len(stats['errors']) > 0:
# Determine where issues were found
for step, exception in stats['errors']:
if step == 'build_client':
try:
# Raise the thrown exception
raise exception
except TimeOutException:
result_handler.warning('Connection to {0} has timed out'.format(identifier_log), code=ErrorCodes.ssh_connection_time)
except (socket.error, UnableToConnectException):
result_handler.failure(
'Connection to {0} could not be established'.format(identifier_log), code=ErrorCodes.ssh_connection_fail)
except NotAuthenticatedException:
result_handler.skip('Connection to {0} could not be authenticated. This node has no access to the Arakoon node.'.format(identifier_log),
code=ErrorCodes.ssh_connection_authentication)
except Exception:
message = 'Connection to {0} could not be established due to an unhandled exception.'.format(identifier_log)
cls.logger.exception(message)
result_handler.exception(message, code=ErrorCodes.unhandled_exception)
elif step == 'stat_dir':
try:
raise exception
except Exception:
message = 'Unable to list the contents of the tlog directory ({0}) for {1}'.format(node.tlog_dir, identifier_log)
cls.logger.exception(message)
result_handler.exception(message, code=ErrorCodes.unhandled_exception)
continue
tlx_files = stats['result']['tlx']
tlog_files = stats['result']['tlog']
headdb_files = stats['result']['headDB']
avail_size = stats['result']['avail_size']
if any(item is None for item in [tlx_files, tlog_files, avail_size]):
# Exception occurred but no errors were logged
result_handler.exception('Either the tlx or tlog files or available size could be found in/of the tlog directory ({0}) for {1}'.format(node.tlog_dir, identifier_log),
code=ErrorCodes.tlx_tlog_not_found)
continue
if len(headdb_files) > 0:
headdb_size = sum([int(i[2]) for i in headdb_files])
collapse_size_msg = 'Spare space for local collapse is'
if avail_size >= headdb_size * 4:
result_handler.success('{0} sufficient (n > 4x head.db size)'.format(collapse_size_msg))
elif avail_size >= headdb_size * 3:
result_handler.warning('{0} running short (n > 3x head.db size)'.format(collapse_size_msg))
elif avail_size >= headdb_size * 2:
result_handler.failure('{0} just enough (n > 2x head.db size'.format(collapse_size_msg))
else:
result_handler.failure('{0} insufficient (n <2 x head.db size'.format(collapse_size_msg))
if len(tlog_files) == 0:
# A tlog should always be present
result_handler.failure('{0} has no open tlog'.format(identifier_log), code=ErrorCodes.tlog_not_found)
continue
if len(tlx_files) < min_tlx_amount:
result_handler.skip('{0} only has {1} tlx, not worth collapsing (required: {2})'.format(identifier_log, len(tlx_files), min_tlx_amount))
continue
# Compare youngest tlog and oldest tlx timestamp
seconds_difference = int(tlog_files[-1][0]) - int(tlx_files[0][0])
if max_age_seconds > seconds_difference:
result_handler.success('{0} should not be collapsed. The oldest tlx is at least {1} days younger than the youngest tlog (actual age: {2})'.format(identifier_log, max_collapse_age, str(timedelta(seconds=seconds_difference))),
code=ErrorCodes.collapse_ok)
else:
result_handler.failure('{0} should be collapsed. The oldest tlx is currently {1} old'.format(identifier_log, str(timedelta(seconds=seconds_difference))), code=ErrorCodes.collapse_not_ok)
def check_arakoon_ports(cls, result_handler):
"""
Verifies that the Arakoon clusters still respond to connections
:param result_handler: logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:return: None
:rtype: NoneType
"""
arakoon_clusters = cls._get_arakoon_clusters(result_handler)
result_handler.info('Starting Arakoon ports test.', add_to_result=False)
result_handler.info('Retrieving all collapsing statistics. This might take a while', add_to_result=False)
start = time.time()
arakoon_stats = cls._get_port_connections(result_handler, arakoon_clusters)
result_handler.info('Retrieving all collapsing statistics succeeded (duration: {0})'.format(time.time() - start), add_to_result=False)
for cluster_type, clusters in arakoon_stats.iteritems():
result_handler.info('Testing the collapse of {0} Arakoons'.format(cluster_type), add_to_result=False)
for cluster in clusters:
cluster_name = cluster['cluster_name']
connection_result = cluster['connection_result']
connection_result = OrderedDict(sorted(connection_result.items(), key=lambda item: ExtensionsToolbox.advanced_sort(item[0].ip, separator='.')))
for node, stats in connection_result.iteritems():
identifier_log = 'Arakoon cluster {0} on node {1}'.format(cluster_name, node.ip)
if len(stats['errors']) > 0:
# Determine where issues were found
for step, exception in stats['errors']:
if step == 'test_connection':
try:
# Raise the thrown exception
raise exception
except Exception:
message = 'Connection to {0} could not be established due to an unhandled exception.'.format(identifier_log)
cls.logger.exception(message)
result_handler.exception(message, code=ErrorCodes.unhandled_exception)
continue
if stats['result'] is True:
result_handler.success('Connection established to {0}'.format(identifier_log),
code=ErrorCodes.arakoon_connection_ok)
else:
result_handler.failure('Connection could not be established to {0}'.format(identifier_log),
code=ErrorCodes.arakoon_connection_failure)
def check_arakoon_fd(cls,
result_handler,
fd_limit=30,
passed_connections=None):
"""
Checks all current open tcp file descriptors for all Arakoon clusters in the OVS cluster
Will raise warnings when these reach a certain threshold
:param result_handler: Logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:param fd_limit: Threshold for the number number of tcp connections for which to start logging warnings
:type fd_limit: int
:param passed_connections: checked TCP connections
:type passed_connections: list
:return: None
:rtype: NoneType
"""
if passed_connections is None:
passed_connections = ['ESTABLISHED', 'TIME_WAIT']
warning_threshold = fd_limit * 80 / 100
error_threshold = fd_limit * 95 / 100
result_handler.info('Starting Arakoon integrity test',
add_to_result=False)
arakoon_clusters = cls._get_arakoon_clusters(result_handler)
start = time.time()
arakoon_fd_results = cls._get_filedescriptors(result_handler,
arakoon_clusters)
result_handler.info(
'Retrieving all file descriptor information succeeded (duration: {0})'
.format(time.time() - start),
add_to_result=False)
for cluster_type, clusters in arakoon_fd_results.iteritems():
result_handler.info(
'Checking the file descriptors of {0} Arakoons'.format(
cluster_type),
add_to_result=False)
for cluster in clusters:
cluster_name = cluster['cluster_name']
fd_result = cluster['fd_result']
fd_result = OrderedDict(
sorted(fd_result.items(),
key=lambda item: ExtensionsToolbox.advanced_sort(
item[0].ip, separator='.')))
for node, stats in fd_result.iteritems():
identifier_log = 'Arakoon cluster {0} on node {1}'.format(
cluster_name, node.ip)
if len(stats['errors']) > 0:
# Determine where issues were found
for step, exception in stats['errors']:
if step == 'build_client':
try:
# Raise the thrown exception
raise exception
except TimeOutException:
result_handler.warning(
'Connection to {0} has timed out'.
format(identifier_log),
code=ErrorCodes.ssh_connection_time)
except (socket.error,
UnableToConnectException):
result_handler.failure(
'Connection to {0} could not be established'
.format(identifier_log),
code=ErrorCodes.ssh_connection_fail)
except NotAuthenticatedException:
result_handler.skip(
'Connection to {0} could not be authenticated. This node has no access to the Arakoon node.'
.format(identifier_log),
code=ErrorCodes.
ssh_connection_authentication)
except Exception:
message = 'Connection to {0} could not be established due to an unhandled exception.'.format(
identifier_log)
cls.logger.exception(message)
result_handler.exception(
message,
code=ErrorCodes.unhandled_exception)
elif step == 'lsof':
try:
raise exception
except Exception:
message = 'Unable to list the file descriptors for {0}'.format(
identifier_log)
cls.logger.exception(message)
result_handler.exception(
message,
ErrorCodes.unhandled_exception)
continue
fds = stats['result']['fds']
filtered_fds = [
i for i in fds if i.split()[-1].strip('(').strip(')')
in passed_connections
]
if len(filtered_fds) >= warning_threshold:
if len(filtered_fds) >= error_threshold:
result_handler.warning(
'Number of TCP connections exceeded the 95% warning threshold for {0}, ({1}/{2})'
.format(identifier_log, len(filtered_fds),
fd_limit),
code=ErrorCodes.arakoon_fd_95)
else:
result_handler.warning(
'Number of TCP connections exceeded the 80% warning threshold for {0}, ({1}/{2})'
.format(identifier_log, len(filtered_fds),
fd_limit),
code=ErrorCodes.arakoon_fd_80)
else:
result_handler.success(
'Number of TCP connections for {0} is healthy ({1}/{2})'
.format(identifier_log, len(filtered_fds),
fd_limit),
code=ErrorCodes.arakoon_fd_ok)
def check_collapse(cls,
result_handler,
max_collapse_age=3,
min_tlx_amount=10):
"""
Verifies collapsing has occurred for all Arakoons
:param result_handler: logging object
:type result_handler: ovs.extensions.healthcheck.result.HCResults
:param max_collapse_age: tlx files may not be longer than x days
:type max_collapse_age: int
:param min_tlx_amount: Minimum amount of tlxes before making collapsing mandatory (defaults to 10)
:type min_tlx_amount: int
:return: None
:rtype: NoneType
"""
arakoon_clusters = cls._get_arakoon_clusters(result_handler)
result_handler.info('Starting Arakoon collapse test',
add_to_result=False)
max_age_seconds = timedelta(days=max_collapse_age).total_seconds()
result_handler.info(
'Retrieving all collapsing statistics. This might take a while',
add_to_result=False)
start = time.time()
arakoon_stats = cls._retrieve_stats(result_handler, arakoon_clusters)
result_handler.info(
'Retrieving all collapsing statistics succeeded (duration: {0})'.
format(time.time() - start),
add_to_result=False)
for cluster_type, clusters in arakoon_stats.iteritems():
result_handler.info(
'Testing the collapse of {0} Arakoons'.format(cluster_type),
add_to_result=False)
for cluster in clusters:
cluster_name = cluster['cluster_name']
collapse_result = cluster['collapse_result']
collapse_result = OrderedDict(
sorted(collapse_result.items(),
key=lambda item: ExtensionsToolbox.advanced_sort(
item[0].ip, separator='.')))
for node, stats in collapse_result.iteritems():
identifier_log = 'Arakoon cluster {0} on node {1}'.format(
cluster_name, node.ip)
if len(stats['errors']) > 0:
# Determine where issues were found
for step, exception in stats['errors']:
if step == 'build_client':
try:
# Raise the thrown exception
raise exception
except TimeOutException:
result_handler.warning(
'Connection to {0} has timed out'.
format(identifier_log),
code=ErrorCodes.ssh_connection_time)
except (socket.error,
UnableToConnectException):
result_handler.failure(
'Connection to {0} could not be established'
.format(identifier_log),
code=ErrorCodes.ssh_connection_fail)
except NotAuthenticatedException:
result_handler.skip(
'Connection to {0} could not be authenticated. This node has no access to the Arakoon node.'
.format(identifier_log),
code=ErrorCodes.
ssh_connection_authentication)
except Exception:
message = 'Connection to {0} could not be established due to an unhandled exception.'.format(
identifier_log)
cls.logger.exception(message)
result_handler.exception(
message,
code=ErrorCodes.unhandled_exception)
elif step == 'stat_dir':
try:
raise exception
except Exception:
message = 'Unable to list the contents of the tlog directory ({0}) for {1}'.format(
node.tlog_dir, identifier_log)
cls.logger.exception(message)
result_handler.exception(
message,
code=ErrorCodes.unhandled_exception)
continue
tlx_files = stats['result']['tlx']
tlog_files = stats['result']['tlog']
headdb_files = stats['result']['headDB']
avail_size = stats['result']['avail_size']
if any(item is None
for item in [tlx_files, tlog_files, avail_size]):
# Exception occurred but no errors were logged
result_handler.exception(
'Either the tlx or tlog files or available size could be found in/of the tlog directory ({0}) for {1}'
.format(node.tlog_dir, identifier_log),
code=ErrorCodes.tlx_tlog_not_found)
continue
if len(headdb_files) > 0:
headdb_size = sum([int(i[2]) for i in headdb_files])
collapse_size_msg = 'Spare space for local collapse is'
if avail_size >= headdb_size * 4:
result_handler.success(
'{0} sufficient (n > 4x head.db size)'.format(
collapse_size_msg))
elif avail_size >= headdb_size * 3:
result_handler.warning(
'{0} running short (n > 3x head.db size)'.
format(collapse_size_msg))
elif avail_size >= headdb_size * 2:
result_handler.failure(
'{0} just enough (n > 2x head.db size'.format(
collapse_size_msg))
else:
result_handler.failure(
'{0} insufficient (n <2 x head.db size'.format(
collapse_size_msg))
if len(tlog_files) == 0:
# A tlog should always be present
result_handler.failure(
'{0} has no open tlog'.format(identifier_log),
code=ErrorCodes.tlog_not_found)
continue
if len(tlx_files) < min_tlx_amount:
result_handler.skip(
'{0} only has {1} tlx, not worth collapsing (required: {2})'
.format(identifier_log, len(tlx_files),
min_tlx_amount))
continue
# Compare youngest tlog and oldest tlx timestamp
seconds_difference = int(tlog_files[-1][0]) - int(
tlx_files[0][0])
if max_age_seconds > seconds_difference:
result_handler.success(
'{0} should not be collapsed. The oldest tlx is at least {1} days younger than the youngest tlog (actual age: {2})'
.format(
identifier_log, max_collapse_age,
str(timedelta(seconds=seconds_difference))),
code=ErrorCodes.collapse_ok)
else:
result_handler.failure(
'{0} should be collapsed. The oldest tlx is currently {1} old'
.format(
identifier_log,
str(timedelta(seconds=seconds_difference))),
code=ErrorCodes.collapse_not_ok)
def get_primary_and_secondary_storagerouters(self):
# type: () -> Tuple[List[StorageRouter], List[StorageRouter]]
"""
Retrieve the primary and secondary storagerouters for MDS deployment
:return: Both primary and secondary storagerouters
:rtype: Tuple[List[StorageRouter], List[StorageRouter]]
"""
# Create a pool of StorageRouters being a part of the primary and secondary domains of this StorageRouter
vdisk = self.vdisk
vdisk_storagerouter = StorageRouter(vdisk.storagerouter_guid)
primary_domains = [
junction.domain for junction in vdisk_storagerouter.domains
if junction.backup is False
]
secondary_domains = [
junction.domain for junction in vdisk_storagerouter.domains
if junction.backup is True
]
primary_storagerouters = set()
secondary_storagerouters = set()
for domain in primary_domains:
primary_storagerouters.update(
StorageRouterList.get_primary_storagerouters_for_domain(
domain))
for domain in secondary_domains:
secondary_storagerouters.update(
StorageRouterList.get_primary_storagerouters_for_domain(
domain))
# In case no domains have been configured
if len(primary_storagerouters) == 0:
primary_storagerouters = set(
StorageRouterList.get_storagerouters())
# Remove all excluded StorageRouters from primary StorageRouters
primary_storagerouters = primary_storagerouters.difference(
self.excluded_storagerouters)
# Remove all StorageRouters from secondary which are present in primary, all excluded
secondary_storagerouters = secondary_storagerouters.difference(
primary_storagerouters)
secondary_storagerouters = secondary_storagerouters.difference(
self.excluded_storagerouters)
# Make sure to only use the StorageRouters related to the current vDisk's vPool
related_storagerouters = [
sd.storagerouter for sd in vdisk.vpool.storagedrivers
if sd.storagerouter is not None
]
primary_storagerouters = list(
primary_storagerouters.intersection(related_storagerouters))
secondary_storagerouters = list(
secondary_storagerouters.intersection(related_storagerouters))
if vdisk_storagerouter not in primary_storagerouters:
raise RuntimeError(
'Host of vDisk {0} ({1}) should be part of the primary domains'
.format(vdisk.name, vdisk_storagerouter.name))
primary_storagerouters.sort(
key=lambda sr: ExtensionsToolbox.advanced_sort(element=sr.ip,
separator='.'))
secondary_storagerouters.sort(
key=lambda sr: ExtensionsToolbox.advanced_sort(element=sr.ip,
separator='.'))
for primary_storagerouter in primary_storagerouters:
self._logger.debug(
'vDisk {0} - Primary StorageRouter {1} with IP {2}'.format(
vdisk.guid, primary_storagerouter.name,
primary_storagerouter.ip))
for secondary_storagerouter in secondary_storagerouters:
self._logger.debug(
'vDisk {0} - Secondary StorageRouter {1} with IP {2}'.format(
vdisk.guid, secondary_storagerouter.name,
secondary_storagerouter.ip))
for excluded_storagerouter in self.excluded_storagerouters:
self._logger.debug(
'vDisk {0} - Excluded StorageRouter {1} with IP {2}'.format(
vdisk.guid, excluded_storagerouter.name,
excluded_storagerouter.ip))
return primary_storagerouters, secondary_storagerouters
def _post_update_alba_plugin_alba(cls, components):
"""
Execute some functionality after the ALBA plugin packages have been updated for the ASD manager nodes
:param components: Update components which have been executed
:type components: list
:return: None
:rtype: NoneType
"""
if PackageFactory.COMP_ALBA not in components:
return
# First run post-update migrations to update services, config mgmt, ... and restart services afterwards
for method_name in ['migrate', 'migrate_sdm']:
try:
# noinspection PyUnresolvedReferences
from ovs.lib.albamigration import AlbaMigrationController
cls._logger.debug(
'Executing migration code: AlbaMigrationController.{0}()'.
format(method_name))
getattr(AlbaMigrationController, method_name)()
except ImportError:
cls._logger.error('Could not import AlbaMigrationController')
except Exception:
cls._logger.exception(
'Migration code for the ALBA plugin failed to be executed')
# Update ALBA nodes
method_name = inspect.currentframe().f_code.co_name
cls._logger.info('Executing hook {0}'.format(method_name))
alba_nodes = sorted(
AlbaNodeList.get_albanodes_by_type(AlbaNode.NODE_TYPES.ASD),
key=lambda an: ExtensionsToolbox.advanced_sort(element=an.ip,
separator='.'))
for alba_node in alba_nodes:
services_to_restart = []
for component in components:
if component not in alba_node.package_information:
continue
component_info = alba_node.package_information[component]
if 'services_post_update' not in component_info:
# Package_information still has the old format, so refresh update information
# This can occur when updating from earlier than 2.11.0 to 2.11.0 and older
try:
GenericController.refresh_package_information()
except:
cls._logger.exception(
'{0}: Refreshing package information failed'.
format(alba_node.ip))
alba_node.discard()
component_info = alba_node.package_information.get(
component, {})
services_post_update = dict(
(int(key), value) for key, value in component_info.get(
'services_post_update', {}).iteritems())
for restart_order in sorted(services_post_update):
for service_name in sorted(
services_post_update[restart_order]):
if service_name not in services_to_restart:
services_to_restart.append(service_name)
if len(services_to_restart) > 0:
alba_node.client.restart_services(
service_names=services_to_restart)
# Renew maintenance services
cls._logger.info('Checkup maintenance agents')
AlbaController.checkup_maintenance_agents.delay()
cls._logger.info('Executed hook {0}'.format(method_name))
def _package_install_plugin_alba(cls, components=None):
"""
Update the packages related to the ASD manager
:param components: Components which have been selected for update
:type components: list
:return: Boolean indicating whether to continue with the update or not
:rtype: bool
"""
cls._logger.info('Updating packages for ALBA plugin')
if components is None:
components = [PackageFactory.COMP_ALBA]
abort = False
alba_nodes = sorted(
AlbaNodeList.get_albanodes_by_type(AlbaNode.NODE_TYPES.ASD),
key=lambda an: ExtensionsToolbox.advanced_sort(element=an.ip,
separator='.'))
for alba_node in alba_nodes:
cls._logger.debug('ALBA Node {0}: Verifying packages'.format(
alba_node.ip))
for component in components:
packages = alba_node.package_information.get(
component, {}).get('packages', {})
package_names = sorted(packages)
# Always install the extensions package first
if PackageFactory.PKG_OVS_EXTENSIONS in package_names:
package_names.remove(PackageFactory.PKG_OVS_EXTENSIONS)
package_names.insert(0, PackageFactory.PKG_OVS_EXTENSIONS)
if len(package_names) > 0:
cls._logger.debug(
'ALBA Node {0}: Packages for component {1}: {2}'.
format(alba_node.ip, component, package_names))
for package_name in package_names:
try:
installed = packages[package_name]['installed']
candidate = packages[package_name]['candidate']
if candidate == alba_node.client.update_installed_version_package(
package_name=package_name):
# Package has already been installed by another hook
continue
cls._logger.debug(
'ALBA Node {0}: Updating package {1} ({2} --> {3})'
.format(alba_node.ip, package_name, installed,
candidate))
alba_node.client.execute_update(package_name)
cls._logger.debug(
'ALBA Node {0}: Updated package {1}'.format(
alba_node.ip, package_name))
except requests.ConnectionError as ce:
if 'Connection aborted.' not in ce.message: # This error is thrown due the post-update code of the SDM package which restarts the asd-manager service
cls._logger.exception(
'ALBA Node {0}: Failed to update package {1}'.
format(alba_node.ip, package_name))
abort = True
except Exception:
cls._logger.exception(
'ALBA Node {0}: Failed to update package {1}'.
format(alba_node.ip, package_name))
abort = True
if abort is False:
cls._logger.info('Updated packages for ALBA plugin')
return abort
def monitor_services(cls):
"""
Monitor the local OVS services
:return: None
:rtype: NoneType
"""
try:
previous_output = None
while True:
# Gather service states
running_services = {}
non_running_services = {}
longest_service_name = 0
for service_info in check_output('initctl list',
shell=True).splitlines():
if not service_info.startswith('ovs-'):
continue
service_info = service_info.split(',')[0].strip()
service_name = service_info.split()[0].strip()
service_state = service_info.split()[1].strip()
if service_state == "start/running":
running_services[service_name] = service_state
else:
non_running_services[service_name] = service_state
if len(service_name) > longest_service_name:
longest_service_name = len(service_name)
# Put service states in list
output = ['OVS running processes', '=====================\n']
for service_name in sorted(
running_services,
key=lambda service: ExtensionsToolbox.advanced_sort(
service, '_')):
output.append('{0} {1} {2}'.format(
service_name,
' ' * (longest_service_name - len(service_name)),
running_services[service_name]))
output.extend([
'\n\nOVS non-running processes',
'=========================\n'
])
for service_name in sorted(
non_running_services,
key=lambda service: ExtensionsToolbox.advanced_sort(
service, '_')):
output.append('{0} {1} {2}'.format(
service_name,
' ' * (longest_service_name - len(service_name)),
non_running_services[service_name]))
# Print service states (only if changes)
if previous_output != output:
print '\x1b[2J\x1b[H'
for line in output:
print line
previous_output = list(output)
time.sleep(1)
except KeyboardInterrupt:
pass