def package_install_core(client, package_info, components):
"""
Update the core packages
:param client: Client on which to execute update the packages
:type client: SSHClient
:param package_info: Information about the packages (installed, candidate)
:type package_info: dict
:param components: Components which have been selected for update
:type components: list
:return: None
"""
if 'framework' not in components and 'storagedriver' not in components:
return
packages_to_install = {}
for pkg_name, pkg_info in package_info.iteritems():
if pkg_name in UpdateController.all_core_packages:
packages_to_install[pkg_name] = pkg_info
if not packages_to_install:
return
UpdateController._logger.debug('{0}: Executing hook {1}'.format(client.ip, inspect.currentframe().f_code.co_name))
for pkg_name, pkg_info in packages_to_install.iteritems():
UpdateController._logger.debug('{0}: Updating core package {1} ({2} --> {3})'.format(client.ip, pkg_name, pkg_info['installed'], pkg_info['candidate']))
PackageManager.install(package_name=pkg_name, client=client)
UpdateController._logger.debug('{0}: Updated core package {1}'.format(client.ip, pkg_name))
UpdateController._logger.debug('{0}: Executed hook {1}'.format(client.ip, inspect.currentframe().f_code.co_name))
def get_heartbeat_data(self):
"""
Returns heartbeat data
"""
data = {'cid': EtcdConfiguration.get('/ovs/framework/cluster_id'),
'nid': System.get_my_machine_id(),
'metadata': {},
'errors': []}
try:
# Versions
data['metadata']['versions'] = PackageManager.get_versions()
except Exception, ex:
data['errors'].append(str(ex))
def get_heartbeat_data():
"""
Returns heartbeat data
"""
data = {'cid': Configuration.get('ovs.support.cid'),
'nid': Configuration.get('ovs.support.nid'),
'metadata': {},
'errors': []}
try:
# Versions
data['metadata']['versions'] = PackageManager.get_versions()
except Exception, ex:
data['errors'].append(str(ex))
def get_heartbeat_data(self):
"""
Returns heartbeat data
"""
data = {
'cid': EtcdConfiguration.get('/ovs/framework/cluster_id'),
'nid': System.get_my_machine_id(),
'metadata': {},
'errors': []
}
try:
# Versions
data['metadata']['versions'] = PackageManager.get_versions()
except Exception, ex:
data['errors'].append(str(ex))
def update_framework():
"""
Update the framework
:return: None
"""
filemutex = file_mutex('system_update', wait=2)
upgrade_file = '/etc/ready_for_upgrade'
upgrade_ongoing_check_file = '/etc/upgrade_ongoing'
ssh_clients = []
try:
filemutex.acquire()
UpdateController._log_message('+++ Starting framework update +++')
from ovs.dal.lists.storagerouterlist import StorageRouterList
UpdateController._log_message('Generating SSH client connections for each storage router')
upgrade_file = '/etc/ready_for_upgrade'
upgrade_ongoing_check_file = '/etc/upgrade_ongoing'
storage_routers = StorageRouterList.get_storagerouters()
ssh_clients = []
master_ips = []
extra_ips = []
for sr in storage_routers:
ssh_clients.append(SSHClient(sr.ip, username='******'))
if sr.node_type == 'MASTER':
master_ips.append(sr.ip)
elif sr.node_type == 'EXTRA':
extra_ips.append(sr.ip)
this_client = [client for client in ssh_clients if client.is_local is True][0]
# Create locks
UpdateController._log_message('Creating lock files', client_ip=this_client.ip)
for client in ssh_clients:
client.run('touch {0}'.format(upgrade_file)) # Prevents manual install or upgrade individual packages
client.run('touch {0}'.format(upgrade_ongoing_check_file)) # Prevents clicking x times on 'Update' btn
# Check requirements
packages_to_update = set()
all_services_to_restart = []
for client in ssh_clients:
for function in Toolbox.fetch_hooks('update', 'metadata'):
UpdateController._log_message('Executing function {0}'.format(function.__name__),
client_ip=client.ip)
output = function(client)
for key, value in output.iteritems():
if key != 'framework':
continue
for package_info in value:
packages_to_update.update(package_info['packages'])
all_services_to_restart += package_info['services']
services_to_restart = []
for service in all_services_to_restart:
if service not in services_to_restart:
services_to_restart.append(service) # Filter out duplicates maintaining the order of services (eg: watcher-framework before memcached)
UpdateController._log_message('Services which will be restarted --> {0}'.format(', '.join(services_to_restart)))
UpdateController._log_message('Packages which will be installed --> {0}'.format(', '.join(packages_to_update)))
# Stop services
if UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='stop') is False:
UpdateController._log_message('Stopping all services on every node failed, cannot continue',
client_ip=this_client.ip, severity='warning')
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
# Start services again if a service could not be stopped
UpdateController._log_message('Attempting to start the services again', client_ip=this_client.ip)
UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message('Failed to stop all required services, aborting update',
client_ip=this_client.ip, severity='error')
return
# Update packages
failed_clients = []
for client in ssh_clients:
PackageManager.update(client=client)
try:
UpdateController._log_message('Installing latest packages', client.ip)
for package in packages_to_update:
UpdateController._log_message('Installing {0}'.format(package), client.ip)
PackageManager.install(package_name=package,
client=client,
force=True)
UpdateController._log_message('Installed {0}'.format(package), client.ip)
client.file_delete(upgrade_file)
except subprocess.CalledProcessError as cpe:
UpdateController._log_message('Upgrade failed with error: {0}'.format(cpe.output), client.ip,
'error')
failed_clients.append(client)
break
if failed_clients:
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('Error occurred. Attempting to start all services again',
client_ip=this_client.ip, severity='error')
UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message('Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'.format('\n - '.join([client.ip for client in failed_clients]), this_client.ip),
this_client.ip,
'error')
return
# Migrate code
for client in ssh_clients:
try:
UpdateController._log_message('Started code migration', client.ip)
try:
with remote(client.ip, [Migrator]) as rem:
rem.Migrator.migrate(master_ips, extra_ips)
except EOFError as eof:
UpdateController._log_message('EOFError during code migration, retrying {0}'.format(eof), client.ip, 'warning')
with remote(client.ip, [Migrator]) as rem:
rem.Migrator.migrate(master_ips, extra_ips)
UpdateController._log_message('Finished code migration', client.ip)
except Exception as ex:
UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('Code migration failed with error: {0}'.format(ex), client.ip, 'error')
return
# Start services
UpdateController._log_message('Starting services', client_ip=this_client.ip)
model_services = []
if 'arakoon-ovsdb' in services_to_restart:
model_services.append('arakoon-ovsdb')
services_to_restart.remove('arakoon-ovsdb')
if 'memcached' in services_to_restart:
model_services.append('memcached')
services_to_restart.remove('memcached')
UpdateController._change_services_state(services=model_services,
ssh_clients=ssh_clients,
action='start')
# Migrate model
UpdateController._log_message('Started model migration', client_ip=this_client.ip)
try:
from ovs.dal.helpers import Migration
with remote(ssh_clients[0].ip, [Migration]) as rem:
rem.Migration.migrate()
UpdateController._log_message('Finished model migration', client_ip=this_client.ip)
except Exception as ex:
UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('An unexpected error occurred: {0}'.format(ex), client_ip=this_client.ip,
severity='error')
return
# Post upgrade actions
UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip)
for client in ssh_clients:
with remote(client.ip, [Toolbox, SSHClient]) as rem:
for function in rem.Toolbox.fetch_hooks('update', 'postupgrade'):
UpdateController._log_message('Executing action {0}'.format(function.__name__),
client_ip=client.ip)
try:
function(rem.SSHClient(client.ip, username='******'))
UpdateController._log_message('Executing action {0} completed'.format(function.__name__),
client_ip=client.ip)
except Exception as ex:
UpdateController._log_message('Post upgrade action failed with error: {0}'.format(ex),
client.ip, 'error')
# Start watcher and restart support-agent
UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._change_services_state(services=['support-agent'],
ssh_clients=ssh_clients,
action='restart')
UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('+++ Finished updating +++')
except RuntimeError as rte:
UpdateController._log_message('Error during framework update: {0}'.format(rte), severity='error')
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
except NoLockAvailableException:
UpdateController._log_message('Another framework update is currently in progress!')
except Exception as ex:
UpdateController._log_message('Error during framework update: {0}'.format(ex), severity='error')
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
finally:
filemutex.release()
def update_volumedriver():
"""
Update the volumedriver
:return: None
"""
filemutex = file_mutex('system_update', wait=2)
upgrade_file = '/etc/ready_for_upgrade'
upgrade_ongoing_check_file = '/etc/upgrade_ongoing'
ssh_clients = []
try:
filemutex.acquire()
UpdateController._log_message('+++ Starting volumedriver update +++')
from ovs.dal.lists.storagerouterlist import StorageRouterList
UpdateController._log_message('Generating SSH client connections for each storage router')
storage_routers = StorageRouterList.get_storagerouters()
ssh_clients = [SSHClient(storage_router.ip, 'root') for storage_router in storage_routers]
this_client = [client for client in ssh_clients if client.is_local is True][0]
# Commence update !!!!!!!
# 0. Create locks
UpdateController._log_message('Creating lock files', client_ip=this_client.ip)
for client in ssh_clients:
client.run('touch {0}'.format(upgrade_file)) # Prevents manual install or upgrade individual packages
client.run('touch {0}'.format(upgrade_ongoing_check_file)) # Prevents clicking x times on 'Update' btn
# 1. Check requirements
packages_to_update = set()
all_services_to_restart = []
for client in ssh_clients:
for function in Toolbox.fetch_hooks('update', 'metadata'):
UpdateController._log_message('Executing function {0}'.format(function.__name__),
client_ip=client.ip)
output = function(client)
for key, value in output.iteritems():
if key != 'volumedriver':
continue
for package_info in value:
packages_to_update.update(package_info['packages'])
all_services_to_restart += package_info['services']
services_to_restart = []
for service in all_services_to_restart:
if service not in services_to_restart:
services_to_restart.append(service) # Filter out duplicates keeping the order of services (eg: watcher-framework before memcached)
UpdateController._log_message('Services which will be restarted --> {0}'.format(', '.join(services_to_restart)))
UpdateController._log_message('Packages which will be installed --> {0}'.format(', '.join(packages_to_update)))
# 1. Stop services
if UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='stop') is False:
UpdateController._log_message('Stopping all services on every node failed, cannot continue',
client_ip=this_client.ip, severity='warning')
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('Attempting to start the services again', client_ip=this_client.ip)
UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message('Failed to stop all required services, update aborted',
client_ip=this_client.ip, severity='error')
return
# 2. Update packages
failed_clients = []
for client in ssh_clients:
PackageManager.update(client=client)
try:
for package_name in packages_to_update:
UpdateController._log_message('Installing {0}'.format(package_name), client.ip)
PackageManager.install(package_name=package_name,
client=client,
force=True)
UpdateController._log_message('Installed {0}'.format(package_name), client.ip)
client.file_delete(upgrade_file)
except subprocess.CalledProcessError as cpe:
UpdateController._log_message('Upgrade failed with error: {0}'.format(cpe.output), client.ip,
'error')
failed_clients.append(client)
break
if failed_clients:
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('Error occurred. Attempting to start all services again',
client_ip=this_client.ip, severity='error')
UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message('Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'.format('\n - '.join([client.ip for client in failed_clients]), this_client.ip),
this_client.ip,
'error')
return
# 3. Post upgrade actions
UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip)
for client in ssh_clients:
for function in Toolbox.fetch_hooks('update', 'postupgrade'):
UpdateController._log_message('Executing action: {0}'.format(function.__name__), client_ip=client.ip)
try:
function(client)
except Exception as ex:
UpdateController._log_message('Post upgrade action failed with error: {0}'.format(ex),
client.ip, 'error')
# 4. Start services
UpdateController._log_message('Starting services', client_ip=this_client.ip)
UpdateController._change_services_state(services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message('+++ Finished updating +++')
except RuntimeError as rte:
UpdateController._log_message('Error during volumedriver update: {0}'.format(rte), severity='error')
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
except NoLockAvailableException:
UpdateController._log_message('Another volumedriver update is currently in progress!')
except Exception as ex:
UpdateController._log_message('Error during volumedriver update: {0}'.format(ex), severity='error')
UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients)
finally:
filemutex.release()
def get_package_information_core(client, package_info):
"""
Called by GenericController.refresh_package_information() every hour
Retrieve information about the currently installed versions of the core packages
Retrieve information about the versions to which each package can potentially be updated
If installed version is different from candidate version --> store this information in model
Additionally check the services with a 'run' file
Verify whether the running version is up-to-date with the candidate version
If different --> store this information in the model
Result: Every package with updates or which requires services to be restarted is stored in the model
:param client: Client on which to collect the version information
:type client: SSHClient
:param package_info: Dictionary passed in by the thread calling this function
:type package_info: dict
:return: Package information
:rtype: dict
"""
try:
if client.username != 'root':
raise RuntimeError('Only the "root" user can retrieve the package information')
binaries = PackageManager.get_binary_versions(client=client, package_names=UpdateController.core_packages_with_binaries)
installed = PackageManager.get_installed_versions(client=client, package_names=UpdateController.all_core_packages)
candidate = PackageManager.get_candidate_versions(client=client, package_names=UpdateController.all_core_packages)
if set(installed.keys()) != set(UpdateController.all_core_packages) or set(candidate.keys()) != set(UpdateController.all_core_packages):
raise RuntimeError('Failed to retrieve the installed and candidate versions for packages: {0}'.format(', '.join(UpdateController.all_core_packages)))
# Retrieve Arakoon information
framework_arakoons = []
storagedriver_arakoons = []
for cluster, arakoon_list in {'cacc': framework_arakoons,
'ovsdb': framework_arakoons,
'voldrv': storagedriver_arakoons}.iteritems():
cluster_name = ArakoonClusterConfig.get_cluster_name(cluster)
if cluster_name is None:
continue
if cluster == 'cacc':
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name, filesystem=True, ip=client.ip)
else:
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if arakoon_metadata['internal'] is True:
arakoon_list.append(ArakoonInstaller.get_service_name_for_cluster(cluster_name=arakoon_metadata['cluster_name']))
storagerouter = StorageRouterList.get_by_ip(client.ip)
alba_proxies = []
for service in storagerouter.services:
if service.type.name == ServiceType.SERVICE_TYPES.ALBA_PROXY:
alba_proxies.append(service.name)
storagedriver_services = []
for sd in storagerouter.storagedrivers:
storagedriver_services.append('ovs-dtl_{0}'.format(sd.vpool.name))
storagedriver_services.append('ovs-volumedriver_{0}'.format(sd.vpool.name))
default_entry = {'candidate': None,
'installed': None,
'services_to_restart': []}
# component: package_name: services_with_run_file
for component, info in {'framework': {'arakoon': framework_arakoons,
'openvstorage': []},
'storagedriver': {'alba': alba_proxies,
'arakoon': storagedriver_arakoons,
'volumedriver-no-dedup-base': [],
'volumedriver-no-dedup-server': storagedriver_services}}.iteritems():
component_info = {}
for package, services in info.iteritems():
for service in services:
service = ExtensionToolbox.remove_prefix(service, 'ovs-')
version_file = '/opt/OpenvStorage/run/{0}.version'.format(service)
if not client.file_exists(version_file):
UpdateController._logger.warning('{0}: Failed to find a version file in /opt/OpenvStorage/run for service {1}'.format(client.ip, service))
continue
package_name = package
running_versions = client.file_read(version_file).strip()
for version in running_versions.split(';'):
version = version.strip()
running_version = None
if '=' in version:
package_name = version.split('=')[0]
running_version = version.split('=')[1]
elif version:
running_version = version
if package_name not in UpdateController.all_core_packages:
raise ValueError('Unknown package dependency found in {0}'.format(version_file))
if package_name not in binaries:
raise RuntimeError('Binary version for package {0} was not retrieved'.format(package_name))
if running_version is not None and running_version != binaries[package_name]:
if package_name not in component_info:
component_info[package_name] = copy.deepcopy(default_entry)
component_info[package_name]['installed'] = running_version
component_info[package_name]['candidate'] = binaries[package_name]
component_info[package_name]['services_to_restart'].append('ovs-{0}'.format(service))
if installed[package] != candidate[package] and package not in component_info:
component_info[package] = copy.deepcopy(default_entry)
component_info[package]['installed'] = installed[package]
component_info[package]['candidate'] = candidate[package]
if component_info:
if component not in package_info[client.ip]:
package_info[client.ip][component] = {}
package_info[client.ip][component].update(component_info)
except Exception as ex:
if 'errors' not in package_info[client.ip]:
package_info[client.ip]['errors'] = []
package_info[client.ip]['errors'].append(ex)
return package_info
def update_framework():
"""
Update the framework
:return: None
"""
file_mutex = FileMutex('system_update', wait=2)
upgrade_file = '/etc/ready_for_upgrade'
upgrade_ongoing_check_file = '/etc/upgrade_ongoing'
ssh_clients = []
try:
file_mutex.acquire()
UpdateController._log_message('+++ Starting framework update +++')
from ovs.dal.lists.storagerouterlist import StorageRouterList
UpdateController._log_message(
'Generating SSH client connections for each storage router')
upgrade_file = '/etc/ready_for_upgrade'
upgrade_ongoing_check_file = '/etc/upgrade_ongoing'
storage_routers = StorageRouterList.get_storagerouters()
ssh_clients = []
master_ips = []
extra_ips = []
for sr in storage_routers:
ssh_clients.append(SSHClient(sr.ip, username='******'))
if sr.node_type == 'MASTER':
master_ips.append(sr.ip)
elif sr.node_type == 'EXTRA':
extra_ips.append(sr.ip)
this_client = [
client for client in ssh_clients if client.is_local is True
][0]
# Create locks
UpdateController._log_message('Creating lock files',
client_ip=this_client.ip)
for client in ssh_clients:
client.run(
'touch {0}'.format(upgrade_file)
) # Prevents manual install or upgrade individual packages
client.run('touch {0}'.format(upgrade_ongoing_check_file)
) # Prevents clicking x times on 'Update' btn
# Check requirements
packages_to_update = set()
all_services_to_restart = []
for client in ssh_clients:
for function in Toolbox.fetch_hooks('update', 'metadata'):
UpdateController._log_message(
'Executing function {0}'.format(function.__name__),
client_ip=client.ip)
output = function(client)
for key, value in output.iteritems():
if key != 'framework':
continue
for package_info in value:
packages_to_update.update(package_info['packages'])
all_services_to_restart += package_info['services']
services_to_restart = []
for service in all_services_to_restart:
if service not in services_to_restart:
services_to_restart.append(
service
) # Filter out duplicates maintaining the order of services (eg: watcher-framework before memcached)
UpdateController._log_message(
'Services which will be restarted --> {0}'.format(
', '.join(services_to_restart)))
UpdateController._log_message(
'Packages which will be installed --> {0}'.format(
', '.join(packages_to_update)))
# Stop services
if UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='stop') is False:
UpdateController._log_message(
'Stopping all services on every node failed, cannot continue',
client_ip=this_client.ip,
severity='warning')
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
# Start services again if a service could not be stopped
UpdateController._log_message(
'Attempting to start the services again',
client_ip=this_client.ip)
UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message(
'Failed to stop all required services, aborting update',
client_ip=this_client.ip,
severity='error')
return
# Update packages
failed_clients = []
for client in ssh_clients:
PackageManager.update(client=client)
try:
UpdateController._log_message('Installing latest packages',
client.ip)
for package in packages_to_update:
UpdateController._log_message(
'Installing {0}'.format(package), client.ip)
PackageManager.install(package_name=package,
client=client,
force=True)
UpdateController._log_message(
'Installed {0}'.format(package), client.ip)
client.file_delete(upgrade_file)
except subprocess.CalledProcessError as cpe:
UpdateController._log_message(
'Upgrade failed with error: {0}'.format(cpe.output),
client.ip, 'error')
failed_clients.append(client)
break
if failed_clients:
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message(
'Error occurred. Attempting to start all services again',
client_ip=this_client.ip,
severity='error')
UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message(
'Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'
.format('\n - '.join([
client.ip for client in failed_clients
])), this_client.ip, 'error')
return
# Migrate code
for client in ssh_clients:
try:
UpdateController._log_message('Started code migration',
client.ip)
try:
with Remote(client.ip, [Migrator]) as remote:
remote.Migrator.migrate(master_ips, extra_ips)
except EOFError as eof:
UpdateController._log_message(
'EOFError during code migration, retrying {0}'.
format(eof), client.ip, 'warning')
with Remote(client.ip, [Migrator]) as remote:
remote.Migrator.migrate(master_ips, extra_ips)
UpdateController._log_message('Finished code migration',
client.ip)
except Exception as ex:
UpdateController._remove_lock_files(
[upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message(
'Code migration failed with error: {0}'.format(ex),
client.ip, 'error')
return
# Start services
UpdateController._log_message('Starting services',
client_ip=this_client.ip)
model_services = []
if 'arakoon-ovsdb' in services_to_restart:
model_services.append('arakoon-ovsdb')
services_to_restart.remove('arakoon-ovsdb')
if 'memcached' in services_to_restart:
model_services.append('memcached')
services_to_restart.remove('memcached')
UpdateController._change_services_state(services=model_services,
ssh_clients=ssh_clients,
action='start')
# Migrate model
UpdateController._log_message('Started model migration',
client_ip=this_client.ip)
try:
from ovs.dal.helpers import Migration
Migration.migrate()
UpdateController._log_message('Finished model migration',
client_ip=this_client.ip)
except Exception as ex:
UpdateController._remove_lock_files(
[upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message(
'An unexpected error occurred: {0}'.format(ex),
client_ip=this_client.ip,
severity='error')
return
# Post upgrade actions
UpdateController._log_message('Executing post upgrade actions',
client_ip=this_client.ip)
for client in ssh_clients:
with Remote(client.ip, [Toolbox, SSHClient]) as remote:
for function in remote.Toolbox.fetch_hooks(
'update', 'postupgrade'):
UpdateController._log_message(
'Executing action {0}'.format(function.__name__),
client_ip=client.ip)
try:
function(
remote.SSHClient(client.ip, username='******'))
UpdateController._log_message(
'Executing action {0} completed'.format(
function.__name__),
client_ip=client.ip)
except Exception as ex:
UpdateController._log_message(
'Post upgrade action failed with error: {0}'.
format(ex), client.ip, 'error')
# Start watcher and restart support-agent
UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._change_services_state(services=['support-agent'],
ssh_clients=ssh_clients,
action='restart')
UpdateController._remove_lock_files([upgrade_ongoing_check_file],
ssh_clients)
UpdateController._log_message('+++ Finished updating +++')
except RuntimeError as rte:
if 'Could not acquire lock' in rte.message:
UpdateController._log_message(
'Another framework update is currently in progress!')
else:
UpdateController._log_message(
'Error during framework update: {0}'.format(rte),
severity='error')
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
except Exception as ex:
UpdateController._log_message(
'Error during framework update: {0}'.format(ex),
severity='error')
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
finally:
file_mutex.release()
def update_volumedriver():
"""
Update the volumedriver
:return: None
"""
file_mutex = FileMutex('system_update', wait=2)
upgrade_file = '/etc/ready_for_upgrade'
upgrade_ongoing_check_file = '/etc/upgrade_ongoing'
ssh_clients = []
try:
file_mutex.acquire()
UpdateController._log_message(
'+++ Starting volumedriver update +++')
from ovs.dal.lists.storagerouterlist import StorageRouterList
UpdateController._log_message(
'Generating SSH client connections for each storage router')
storage_routers = StorageRouterList.get_storagerouters()
ssh_clients = [
SSHClient(storage_router.ip, 'root')
for storage_router in storage_routers
]
this_client = [
client for client in ssh_clients if client.is_local is True
][0]
# Commence update !!!!!!!
# 0. Create locks
UpdateController._log_message('Creating lock files',
client_ip=this_client.ip)
for client in ssh_clients:
client.run(
'touch {0}'.format(upgrade_file)
) # Prevents manual install or upgrade individual packages
client.run('touch {0}'.format(upgrade_ongoing_check_file)
) # Prevents clicking x times on 'Update' btn
# 1. Check requirements
packages_to_update = set()
all_services_to_restart = []
for client in ssh_clients:
for function in Toolbox.fetch_hooks('update', 'metadata'):
UpdateController._log_message(
'Executing function {0}'.format(function.__name__),
client_ip=client.ip)
output = function(client)
for key, value in output.iteritems():
if key != 'volumedriver':
continue
for package_info in value:
packages_to_update.update(package_info['packages'])
all_services_to_restart += package_info['services']
services_to_restart = []
for service in all_services_to_restart:
if service not in services_to_restart:
services_to_restart.append(
service
) # Filter out duplicates keeping the order of services (eg: watcher-framework before memcached)
UpdateController._log_message(
'Services which will be restarted --> {0}'.format(
', '.join(services_to_restart)))
UpdateController._log_message(
'Packages which will be installed --> {0}'.format(
', '.join(packages_to_update)))
# 1. Stop services
if UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='stop') is False:
UpdateController._log_message(
'Stopping all services on every node failed, cannot continue',
client_ip=this_client.ip,
severity='warning')
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message(
'Attempting to start the services again',
client_ip=this_client.ip)
UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message(
'Failed to stop all required services, update aborted',
client_ip=this_client.ip,
severity='error')
return
# 2. Update packages
failed_clients = []
for client in ssh_clients:
PackageManager.update(client=client)
try:
for package_name in packages_to_update:
UpdateController._log_message(
'Installing {0}'.format(package_name), client.ip)
PackageManager.install(package_name=package_name,
client=client,
force=True)
UpdateController._log_message(
'Installed {0}'.format(package_name), client.ip)
client.file_delete(upgrade_file)
except subprocess.CalledProcessError as cpe:
UpdateController._log_message(
'Upgrade failed with error: {0}'.format(cpe.output),
client.ip, 'error')
failed_clients.append(client)
break
if failed_clients:
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
UpdateController._log_message(
'Error occurred. Attempting to start all services again',
client_ip=this_client.ip,
severity='error')
UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._log_message(
'Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'
.format('\n - '.join([
client.ip for client in failed_clients
])), this_client.ip, 'error')
return
# 3. Post upgrade actions
UpdateController._log_message('Executing post upgrade actions',
client_ip=this_client.ip)
for client in ssh_clients:
for function in Toolbox.fetch_hooks('update', 'postupgrade'):
UpdateController._log_message(
'Executing action: {0}'.format(function.__name__),
client_ip=client.ip)
try:
function(client)
except Exception as ex:
UpdateController._log_message(
'Post upgrade action failed with error: {0}'.
format(ex), client.ip, 'error')
# 4. Start services
UpdateController._log_message('Starting services',
client_ip=this_client.ip)
UpdateController._change_services_state(
services=services_to_restart,
ssh_clients=ssh_clients,
action='start')
UpdateController._remove_lock_files([upgrade_ongoing_check_file],
ssh_clients)
UpdateController._log_message('+++ Finished updating +++')
except RuntimeError as rte:
if 'Could not acquire lock' in rte.message:
UpdateController._log_message(
'Another volumedriver update is currently in progress!')
else:
UpdateController._log_message(
'Error during volumedriver update: {0}'.format(rte),
severity='error')
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
except Exception as ex:
UpdateController._log_message(
'Error during volumedriver update: {0}'.format(ex),
severity='error')
UpdateController._remove_lock_files(
[upgrade_file, upgrade_ongoing_check_file], ssh_clients)
finally:
file_mutex.release()
