def post_update_core(client, components):
"""
Execute functionality after the openvstorage core packages have been updated
For framework:
* Restart support-agent on every client
* Restart arakoon-ovsdb on every client (if present and required)
For storagedriver:
* ALBA proxies on every client
* Restart arakoon-voldrv on every client (if present and required)
:param client: Client on which to execute this post update functionality
:type client: SSHClient
:param components: Update components which have been executed
:type components: list
:return: None
"""
if 'framework' not in components and 'storagedriver' not in components:
return
update_information = UpdateController.get_update_information_core({})
services_to_restart = set()
if 'storagedriver' in components:
services_to_restart.update(update_information.get('storagedriver', {}).get('services_post_update', set()))
if 'framework' in components:
services_to_restart.update(update_information.get('framework', {}).get('services_post_update', set()))
services_to_restart.add('support-agent')
if services_to_restart:
UpdateController._logger.debug('{0}: Executing hook {1}'.format(client.ip, inspect.currentframe().f_code.co_name))
for service_name in sorted(services_to_restart):
if not service_name.startswith('ovs-arakoon-'):
UpdateController.change_services_state(services=[service_name], ssh_clients=[client], action='restart')
else:
cluster_name = ArakoonClusterConfig.get_cluster_name(ExtensionToolbox.remove_prefix(service_name, 'ovs-arakoon-'))
if cluster_name == 'config':
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name='cacc', filesystem=True, ip=System.get_my_storagerouter().ip)
else:
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if arakoon_metadata['internal'] is True:
UpdateController._logger.debug('{0}: Restarting arakoon node {1}'.format(client.ip, cluster_name))
ArakoonInstaller.restart_node(cluster_name=cluster_name,
client=client)
UpdateController._logger.debug('{0}: Executed hook {1}'.format(client.ip, inspect.currentframe().f_code.co_name))
def get_package_information_core(client, package_info):
"""
Called by GenericController.refresh_package_information() every hour
Retrieve information about the currently installed versions of the core packages
Retrieve information about the versions to which each package can potentially be updated
If installed version is different from candidate version --> store this information in model
Additionally check the services with a 'run' file
Verify whether the running version is up-to-date with the candidate version
If different --> store this information in the model
Result: Every package with updates or which requires services to be restarted is stored in the model
:param client: Client on which to collect the version information
:type client: SSHClient
:param package_info: Dictionary passed in by the thread calling this function
:type package_info: dict
:return: Package information
:rtype: dict
"""
try:
if client.username != 'root':
raise RuntimeError('Only the "root" user can retrieve the package information')
binaries = PackageManager.get_binary_versions(client=client, package_names=UpdateController.core_packages_with_binaries)
installed = PackageManager.get_installed_versions(client=client, package_names=UpdateController.all_core_packages)
candidate = PackageManager.get_candidate_versions(client=client, package_names=UpdateController.all_core_packages)
if set(installed.keys()) != set(UpdateController.all_core_packages) or set(candidate.keys()) != set(UpdateController.all_core_packages):
raise RuntimeError('Failed to retrieve the installed and candidate versions for packages: {0}'.format(', '.join(UpdateController.all_core_packages)))
# Retrieve Arakoon information
framework_arakoons = []
storagedriver_arakoons = []
for cluster, arakoon_list in {'cacc': framework_arakoons,
'ovsdb': framework_arakoons,
'voldrv': storagedriver_arakoons}.iteritems():
cluster_name = ArakoonClusterConfig.get_cluster_name(cluster)
if cluster_name is None:
continue
if cluster == 'cacc':
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name, filesystem=True, ip=client.ip)
else:
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if arakoon_metadata['internal'] is True:
arakoon_list.append(ArakoonInstaller.get_service_name_for_cluster(cluster_name=arakoon_metadata['cluster_name']))
storagerouter = StorageRouterList.get_by_ip(client.ip)
alba_proxies = []
for service in storagerouter.services:
if service.type.name == ServiceType.SERVICE_TYPES.ALBA_PROXY:
alba_proxies.append(service.name)
storagedriver_services = []
for sd in storagerouter.storagedrivers:
storagedriver_services.append('ovs-dtl_{0}'.format(sd.vpool.name))
storagedriver_services.append('ovs-volumedriver_{0}'.format(sd.vpool.name))
default_entry = {'candidate': None,
'installed': None,
'services_to_restart': []}
# component: package_name: services_with_run_file
for component, info in {'framework': {'arakoon': framework_arakoons,
'openvstorage': []},
'storagedriver': {'alba': alba_proxies,
'arakoon': storagedriver_arakoons,
'volumedriver-no-dedup-base': [],
'volumedriver-no-dedup-server': storagedriver_services}}.iteritems():
component_info = {}
for package, services in info.iteritems():
for service in services:
service = ExtensionToolbox.remove_prefix(service, 'ovs-')
version_file = '/opt/OpenvStorage/run/{0}.version'.format(service)
if not client.file_exists(version_file):
UpdateController._logger.warning('{0}: Failed to find a version file in /opt/OpenvStorage/run for service {1}'.format(client.ip, service))
continue
package_name = package
running_versions = client.file_read(version_file).strip()
for version in running_versions.split(';'):
version = version.strip()
running_version = None
if '=' in version:
package_name = version.split('=')[0]
running_version = version.split('=')[1]
elif version:
running_version = version
if package_name not in UpdateController.all_core_packages:
raise ValueError('Unknown package dependency found in {0}'.format(version_file))
if package_name not in binaries:
raise RuntimeError('Binary version for package {0} was not retrieved'.format(package_name))
if running_version is not None and running_version != binaries[package_name]:
if package_name not in component_info:
component_info[package_name] = copy.deepcopy(default_entry)
component_info[package_name]['installed'] = running_version
component_info[package_name]['candidate'] = binaries[package_name]
component_info[package_name]['services_to_restart'].append('ovs-{0}'.format(service))
if installed[package] != candidate[package] and package not in component_info:
component_info[package] = copy.deepcopy(default_entry)
component_info[package]['installed'] = installed[package]
component_info[package]['candidate'] = candidate[package]
if component_info:
if component not in package_info[client.ip]:
package_info[client.ip][component] = {}
package_info[client.ip][component].update(component_info)
except Exception as ex:
if 'errors' not in package_info[client.ip]:
package_info[client.ip]['errors'] = []
package_info[client.ip]['errors'].append(ex)
return package_info
def get_update_information_core(information):
"""
Called when the 'Update' button in the GUI is pressed
This call collects additional information about the packages which can be updated
Eg:
* Downtime for Arakoons
* Downtime for StorageDrivers
* Prerequisites that haven't been met
* Services which will be stopped during update
* Services which will be restarted after update
"""
# Verify arakoon info
arakoon_ovs_info = {'down': False,
'name': None,
'internal': False}
arakoon_cacc_info = {'down': False,
'name': None,
'internal': False}
arakoon_voldrv_info = {'down': False,
'name': None,
'internal': False}
for cluster in ['cacc', 'ovsdb', 'voldrv']:
cluster_name = ArakoonClusterConfig.get_cluster_name(cluster)
if cluster_name is None:
continue
if cluster == 'cacc':
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name, filesystem=True, ip=System.get_my_storagerouter().ip)
else:
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if arakoon_metadata['internal'] is True:
config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=(cluster == 'cacc'))
config.load_config(System.get_my_storagerouter().ip if cluster == 'cacc' else None)
if cluster == 'ovsdb':
arakoon_ovs_info['down'] = len(config.nodes) < 3
arakoon_ovs_info['name'] = arakoon_metadata['cluster_name']
arakoon_ovs_info['internal'] = True
elif cluster == 'voldrv':
arakoon_voldrv_info['down'] = len(config.nodes) < 3
arakoon_voldrv_info['name'] = arakoon_metadata['cluster_name']
arakoon_voldrv_info['internal'] = True
else:
arakoon_cacc_info['name'] = arakoon_metadata['cluster_name']
arakoon_cacc_info['internal'] = True
# Verify StorageRouter downtime
prerequisites = []
all_storagerouters = StorageRouterList.get_storagerouters()
for storagerouter in all_storagerouters:
try:
SSHClient(endpoint=storagerouter, username='******')
except UnableToConnectException:
prerequisites.append(['node_down', storagerouter.name])
for key in ['framework', 'storagedriver']:
if key not in information:
information[key] = {'packages': {},
'downtime': [],
'prerequisites': prerequisites,
'services_stop_start': set(),
'services_post_update': set()}
for storagerouter in all_storagerouters:
if key not in storagerouter.package_information:
continue
# Retrieve ALBA proxy issues
alba_services = []
alba_downtime = []
for service in storagerouter.services:
if service.type.name != ServiceType.SERVICE_TYPES.ALBA_PROXY or service.alba_proxy is None:
continue
alba_services.append(service.name)
alba_downtime.append(['proxy', service.alba_proxy.storagedriver.vpool.name])
# Retrieve StorageDriver issues
storagedriver_downtime = []
storagedriver_services = []
for sd in storagerouter.storagedrivers:
# Order of services is important, first we want to stop all volume-drivers, then DTLs
storagedriver_services.append('ovs-volumedriver_{0}'.format(sd.vpool.name))
for sd in storagerouter.storagedrivers:
storagedriver_services.append('ovs-dtl_{0}'.format(sd.vpool.name))
if len(sd.vdisks_guids) > 0:
storagedriver_downtime.append(['voldrv', sd.vpool.name])
# Retrieve the actual update information
for package_name, package_info in storagerouter.package_information[key].iteritems():
if package_name not in UpdateController.all_core_packages:
continue # Only gather information for the core packages
information[key]['services_post_update'].update(package_info.pop('services_to_restart'))
if package_name not in information[key]['packages']:
information[key]['packages'][package_name] = {}
information[key]['packages'][package_name].update(package_info)
if package_name == 'openvstorage':
if ['gui', None] not in information[key]['downtime']:
information[key]['downtime'].append(['gui', None])
if ['api', None] not in information[key]['downtime']:
information[key]['downtime'].append(['api', None])
information[key]['services_stop_start'].update({'watcher-framework', 'memcached'})
elif package_name == 'alba':
for down in alba_downtime:
if down not in information[key]['downtime']:
information[key]['downtime'].append(down)
information[key]['services_post_update'].update(alba_services)
elif package_name == 'volumedriver-no-dedup-base':
for down in storagedriver_downtime:
if down not in information[key]['downtime']:
information[key]['downtime'].append(down)
information[key]['services_post_update'].update(storagedriver_services)
elif package_name == 'volumedriver-no-dedup-server':
for down in storagedriver_downtime:
if down not in information[key]['downtime']:
information[key]['downtime'].append(down)
information[key]['services_post_update'].update(storagedriver_services)
elif package_name == 'arakoon':
if key == 'framework':
framework_arakoons = set()
if arakoon_ovs_info['internal'] is True:
framework_arakoons.add('ovs-arakoon-{0}'.format(arakoon_ovs_info['name']))
if arakoon_cacc_info['internal'] is True:
framework_arakoons.add('ovs-arakoon-{0}'.format(arakoon_cacc_info['name']))
information[key]['services_post_update'].update(framework_arakoons)
if arakoon_ovs_info['down'] is True and ['ovsdb', None] not in information[key]['downtime']:
information[key]['downtime'].append(['ovsdb', None])
elif arakoon_voldrv_info['internal'] is True:
information[key]['services_post_update'].update({'ovs-arakoon-{0}'.format(arakoon_voldrv_info['name'])})
if arakoon_voldrv_info['down'] is True and ['voldrv', None] not in information[key]['downtime']:
information[key]['downtime'].append(['voldrv', None])
return information
def get_update_information_alba_plugin(information):
"""
Called when the 'Update' button in the GUI is pressed
This call collects additional information about the packages which can be updated
Eg:
* Downtime for Arakoons
* Downtime for StorageDrivers
* Prerequisites that haven't been met
* Services which will be stopped during update
* Services which will be restarted after update
"""
# Verify arakoon info
arakoon_ovs_info = {'down': False,
'name': None,
'internal': False}
arakoon_cacc_info = {'down': False,
'name': None,
'internal': False}
for cluster in ['cacc', 'ovsdb']:
cluster_name = ArakoonClusterConfig.get_cluster_name(cluster)
if cluster_name is None:
continue
if cluster == 'cacc':
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name, filesystem=True, ip=System.get_my_storagerouter().ip)
else:
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if arakoon_metadata['internal'] is True:
config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=(cluster == 'cacc'))
config.load_config(System.get_my_storagerouter().ip if cluster == 'cacc' else None)
if cluster == 'ovsdb':
arakoon_ovs_info['down'] = len(config.nodes) < 3
arakoon_ovs_info['name'] = arakoon_metadata['cluster_name']
arakoon_ovs_info['internal'] = True
else:
arakoon_cacc_info['name'] = arakoon_metadata['cluster_name']
arakoon_cacc_info['internal'] = True
# Verify StorageRouter downtime
fwk_prerequisites = []
all_storagerouters = StorageRouterList.get_storagerouters()
for storagerouter in all_storagerouters:
try:
SSHClient(endpoint=storagerouter, username='******')
except UnableToConnectException:
fwk_prerequisites.append(['node_down', storagerouter.name])
# Verify ALBA node responsiveness
alba_prerequisites = []
for alba_node in AlbaNodeList.get_albanodes():
try:
alba_node.client.get_metadata()
except Exception:
alba_prerequisites.append(['alba_node_unresponsive', alba_node.ip])
for key in ['framework', 'alba']:
if key not in information:
information[key] = {'packages': {},
'downtime': [],
'prerequisites': fwk_prerequisites if key == 'framework' else alba_prerequisites,
'services_stop_start': set(),
'services_post_update': set()}
for storagerouter in StorageRouterList.get_storagerouters():
if key not in storagerouter.package_information:
continue
# Retrieve Arakoon issues
arakoon_downtime = []
arakoon_services = []
for service in storagerouter.services:
if service.type.name not in [ServiceType.SERVICE_TYPES.ALBA_MGR, ServiceType.SERVICE_TYPES.NS_MGR]:
continue
if service.type.name == ServiceType.SERVICE_TYPES.ALBA_MGR:
cluster_name = AlbaController.get_abm_cluster_name(alba_backend=service.abm_service.alba_backend)
else:
cluster_name = AlbaController.get_nsm_cluster_name(alba_backend=service.nsm_service.alba_backend, number=service.nsm_service.number)
if Configuration.exists('/ovs/arakoon/{0}/config'.format(cluster_name), raw=True) is False:
continue
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if arakoon_metadata['internal'] is True:
arakoon_services.append('ovs-{0}'.format(service.name))
config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=False)
config.load_config()
if len(config.nodes) < 3:
if service.type.name == ServiceType.SERVICE_TYPES.NS_MGR:
arakoon_downtime.append(['backend', service.nsm_service.alba_backend.name])
else:
arakoon_downtime.append(['backend', service.abm_service.alba_backend.name])
for package_name, package_info in storagerouter.package_information[key].iteritems():
if package_name not in AlbaUpdateController.alba_plugin_packages:
continue # Only gather information for the core packages
information[key]['services_post_update'].update(package_info.pop('services_to_restart'))
if package_name not in information[key]['packages']:
information[key]['packages'][package_name] = {}
information[key]['packages'][package_name].update(package_info)
if package_name == 'openvstorage-backend':
if ['gui', None] not in information[key]['downtime']:
information[key]['downtime'].append(['gui', None])
if ['api', None] not in information[key]['downtime']:
information[key]['downtime'].append(['api', None])
information[key]['services_stop_start'].update({'watcher-framework', 'memcached'})
elif package_name == 'alba':
for down in arakoon_downtime:
if down not in information[key]['downtime']:
information[key]['downtime'].append(down)
information[key]['services_post_update'].update(arakoon_services)
elif package_name == 'arakoon':
if key == 'framework':
framework_arakoons = set()
if arakoon_ovs_info['internal'] is True:
framework_arakoons.add('ovs-arakoon-{0}'.format(arakoon_ovs_info['name']))
if arakoon_cacc_info['internal'] is True:
framework_arakoons.add('ovs-arakoon-{0}'.format(arakoon_cacc_info['name']))
information[key]['services_post_update'].update(framework_arakoons)
if arakoon_ovs_info['down'] is True and ['ovsdb', None] not in information[key]['downtime']:
information[key]['downtime'].append(['ovsdb', None])
else:
for down in arakoon_downtime:
if down not in information[key]['downtime']:
information[key]['downtime'].append(down)
information[key]['services_post_update'].update(arakoon_services)
for alba_node in AlbaNodeList.get_albanodes():
for package_name, package_info in alba_node.package_information.get(key, {}).iteritems():
if package_name not in AlbaUpdateController.sdm_packages:
continue # Only gather information for the SDM packages
information[key]['services_post_update'].update(package_info.pop('services_to_restart'))
if package_name not in information[key]['packages']:
information[key]['packages'][package_name] = {}
information[key]['packages'][package_name].update(package_info)
return information
