def get(pmachine): """ Returns the appropriate hypervisor client class for a given PMachine """ hvtype = pmachine.hvtype ip = pmachine.ip username = pmachine.username password = pmachine.password key = '{0}_{1}'.format(ip, username) if key not in Factory.hypervisors: mutex = FileMutex('hypervisor_{0}'.format(key)) try: mutex.acquire(30) if key not in Factory.hypervisors: if hvtype == 'VMWARE': from hypervisors.vmware import VMware hypervisor = VMware(ip, username, password) elif hvtype == 'KVM': from hypervisors.kvm import KVM hypervisor = KVM(ip, username, password) else: raise NotImplementedError('Hypervisor {0} is not yet supported'.format(hvtype)) Factory.hypervisors[key] = hypervisor finally: mutex.release() return Factory.hypervisors[key]
def get_mgmtcenter(pmachine=None, mgmt_center=None): """ @param pmachine: pmachine hybrid from DAL @param mgmt_center: mgmtcenter hybrid from DAL Returns the appropriate sdk client for the management center of the node """ if not ((pmachine is None) ^ (mgmt_center is None)): raise ValueError('Either a pMachine or a Management center should be passed') if pmachine is not None: mgmt_center = pmachine.mgmtcenter if mgmt_center is None: return None mgmtcenter_type = mgmt_center.type ip = mgmt_center.ip username = mgmt_center.username password = mgmt_center.password key = '{0}_{1}'.format(ip, username) if key not in Factory.mgmtcenters: mutex = FileMutex('mgmtcenter_{0}'.format(key)) try: mutex.acquire(30) if key not in Factory.mgmtcenters: if mgmtcenter_type == 'VCENTER': from mgmtcenters.vcenter import VCenter mgmtcenter = VCenter(ip, username, password) elif mgmtcenter_type == 'OPENSTACK': from mgmtcenters.openstack import OpenStack mgmtcenter = OpenStack(ip, username, password) else: raise NotImplementedError('Management center for {0} is not yet supported'.format(mgmtcenter_type)) Factory.mgmtcenters[key] = mgmtcenter finally: mutex.release() return Factory.mgmtcenters[key]
def new_function(*args, **kw): """ Executes the decorated function in a locked context """ filemutex = FileMutex('messaging') try: filemutex.acquire(wait=5) mutex = VolatileMutex('messaging') try: mutex.acquire(wait=5) return f(*args, **kw) finally: mutex.release() finally: filemutex.release()
def get(pmachine): """ Returns the appropriate hypervisor client class for a given PMachine """ hvtype = pmachine.hvtype ip = pmachine.ip username = pmachine.username password = pmachine.password key = '{0}_{1}'.format(ip, username) if key not in Factory.hypervisors: mutex = FileMutex('hypervisor_{0}'.format(key)) try: mutex.acquire(30) if key not in Factory.hypervisors: if hvtype == 'VMWARE': from hypervisors.vmware import VMware hypervisor = VMware(ip, username, password) elif hvtype == 'KVM': from hypervisors.kvm import KVM hypervisor = KVM(ip, username, password) else: raise NotImplementedError( 'Hypervisor {0} is not yet supported'.format( hvtype)) Factory.hypervisors[key] = hypervisor finally: mutex.release() return Factory.hypervisors[key]
def get_mgmtcenter(pmachine=None, mgmt_center=None): """ @param pmachine: pmachine hybrid from DAL @param mgmt_center: mgmtcenter hybrid from DAL Returns the appropriate sdk client for the management center of the node Implemented only for VMWare / vCenter Server TODO: KVM """ if not ((pmachine is None) ^ (mgmt_center is None)): raise ValueError( 'Either a pMachine or a Management center should be passed') if pmachine is not None: mgmt_center = pmachine.mgmtcenter if mgmt_center is None: return None mgmtcenter_type = mgmt_center.type ip = mgmt_center.ip username = mgmt_center.username password = mgmt_center.password key = '{0}_{1}'.format(ip, username) if key not in Factory.mgmtcenters: mutex = FileMutex('mgmtcenter_{0}'.format(key)) try: mutex.acquire(30) if key not in Factory.mgmtcenters: if mgmtcenter_type == 'VCENTER': from mgmtcenters.vcenter import VCenter mgmtcenter = VCenter(ip, username, password) elif mgmtcenter_type == 'OPENSTACK': from mgmtcenters.openstack import OpenStack mgmtcenter = OpenStack(ip, username, password) else: raise NotImplementedError( 'Management center for {0} is not yet supported'. format(mgmtcenter_type)) Factory.mgmtcenters[key] = mgmtcenter finally: mutex.release() return Factory.mgmtcenters[key]
def update_framework(): """ Update the framework :return: None """ file_mutex = FileMutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: file_mutex.acquire() UpdateController._log_message('+++ Starting framework update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message('Generating SSH client connections for each storage router') upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [] master_ips = [] extra_ips = [] for sr in storage_routers: ssh_clients.append(SSHClient(sr.ip, username='******')) if sr.node_type == 'MASTER': master_ips.append(sr.ip) elif sr.node_type == 'EXTRA': extra_ips.append(sr.ip) this_client = [client for client in ssh_clients if client.is_local is True][0] # Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run('touch {0}'.format(upgrade_file)) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file)) # Prevents clicking x times on 'Update' btn # Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message('Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'framework': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append(service) # Filter out duplicates maintaining the order of services (eg: watcher-framework before memcached) UpdateController._log_message('Services which will be restarted --> {0}'.format(', '.join(services_to_restart))) UpdateController._log_message('Packages which will be installed --> {0}'.format(', '.join(packages_to_update))) # Stop services if UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message('Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) # Start services again if a service could not be stopped UpdateController._log_message('Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to stop all required services, aborting update', client_ip=this_client.ip, severity='error') return # Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: UpdateController._log_message('Installing latest packages', client.ip) for package in packages_to_update: UpdateController._log_message('Installing {0}'.format(package), client.ip) PackageManager.install(package_name=package, client=client, force=True) UpdateController._log_message('Installed {0}'.format(package), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message('Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'.format('\n - '.join([client.ip for client in failed_clients])), this_client.ip, 'error') return # Migrate code for client in ssh_clients: try: UpdateController._log_message('Started code migration', client.ip) try: with Remote(client.ip, [Migrator]) as remote: remote.Migrator.migrate(master_ips, extra_ips) except EOFError as eof: UpdateController._log_message('EOFError during code migration, retrying {0}'.format(eof), client.ip, 'warning') with Remote(client.ip, [Migrator]) as remote: remote.Migrator.migrate(master_ips, extra_ips) UpdateController._log_message('Finished code migration', client.ip) except Exception as ex: UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Code migration failed with error: {0}'.format(ex), client.ip, 'error') return # Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) model_services = [] if 'arakoon-ovsdb' in services_to_restart: model_services.append('arakoon-ovsdb') services_to_restart.remove('arakoon-ovsdb') if 'memcached' in services_to_restart: model_services.append('memcached') services_to_restart.remove('memcached') UpdateController._change_services_state(services=model_services, ssh_clients=ssh_clients, action='start') # Migrate model UpdateController._log_message('Started model migration', client_ip=this_client.ip) try: from ovs.dal.helpers import Migration Migration.migrate() UpdateController._log_message('Finished model migration', client_ip=this_client.ip) except Exception as ex: UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('An unexpected error occurred: {0}'.format(ex), client_ip=this_client.ip, severity='error') return # Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: with Remote(client.ip, [Toolbox, SSHClient]) as remote: for function in remote.Toolbox.fetch_hooks('update', 'postupgrade'): UpdateController._log_message('Executing action {0}'.format(function.__name__), client_ip=client.ip) try: function(remote.SSHClient(client.ip, username='******')) UpdateController._log_message('Executing action {0} completed'.format(function.__name__), client_ip=client.ip) except Exception as ex: UpdateController._log_message('Post upgrade action failed with error: {0}'.format(ex), client.ip, 'error') # Start watcher and restart support-agent UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._change_services_state(services=['support-agent'], ssh_clients=ssh_clients, action='restart') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: UpdateController._log_message('Error during framework update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) except NoLockAvailableException: UpdateController._log_message('Another framework update is currently in progress!') except Exception as ex: UpdateController._log_message('Error during framework update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: file_mutex.release()
def update_volumedriver(): """ Update the volumedriver :return: None """ file_mutex = FileMutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: file_mutex.acquire() UpdateController._log_message('+++ Starting volumedriver update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message('Generating SSH client connections for each storage router') storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [SSHClient(storage_router.ip, 'root') for storage_router in storage_routers] this_client = [client for client in ssh_clients if client.is_local is True][0] # Commence update !!!!!!! # 0. Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run('touch {0}'.format(upgrade_file)) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file)) # Prevents clicking x times on 'Update' btn # 1. Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message('Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'volumedriver': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append(service) # Filter out duplicates keeping the order of services (eg: watcher-framework before memcached) UpdateController._log_message('Services which will be restarted --> {0}'.format(', '.join(services_to_restart))) UpdateController._log_message('Packages which will be installed --> {0}'.format(', '.join(packages_to_update))) # 1. Stop services if UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message('Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to stop all required services, update aborted', client_ip=this_client.ip, severity='error') return # 2. Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: for package_name in packages_to_update: UpdateController._log_message('Installing {0}'.format(package_name), client.ip) PackageManager.install(package_name=package_name, client=client, force=True) UpdateController._log_message('Installed {0}'.format(package_name), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message('Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'.format('\n - '.join([client.ip for client in failed_clients])), this_client.ip, 'error') return # 3. Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'postupgrade'): UpdateController._log_message('Executing action: {0}'.format(function.__name__), client_ip=client.ip) try: function(client) except Exception as ex: UpdateController._log_message('Post upgrade action failed with error: {0}'.format(ex), client.ip, 'error') # 4. Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: UpdateController._log_message('Error during volumedriver update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) except NoLockAvailableException: UpdateController._log_message('Another volumedriver update is currently in progress!') except Exception as ex: UpdateController._log_message('Error during volumedriver update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: file_mutex.release()
def update_framework(): """ Update the framework :return: None """ file_mutex = FileMutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: file_mutex.acquire() UpdateController._log_message('+++ Starting framework update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message( 'Generating SSH client connections for each storage router') upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [] master_ips = [] extra_ips = [] for sr in storage_routers: ssh_clients.append(SSHClient(sr.ip, username='******')) if sr.node_type == 'MASTER': master_ips.append(sr.ip) elif sr.node_type == 'EXTRA': extra_ips.append(sr.ip) this_client = [ client for client in ssh_clients if client.is_local is True ][0] # Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run( 'touch {0}'.format(upgrade_file) ) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file) ) # Prevents clicking x times on 'Update' btn # Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message( 'Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'framework': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append( service ) # Filter out duplicates maintaining the order of services (eg: watcher-framework before memcached) UpdateController._log_message( 'Services which will be restarted --> {0}'.format( ', '.join(services_to_restart))) UpdateController._log_message( 'Packages which will be installed --> {0}'.format( ', '.join(packages_to_update))) # Stop services if UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message( 'Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) # Start services again if a service could not be stopped UpdateController._log_message( 'Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to stop all required services, aborting update', client_ip=this_client.ip, severity='error') return # Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: UpdateController._log_message('Installing latest packages', client.ip) for package in packages_to_update: UpdateController._log_message( 'Installing {0}'.format(package), client.ip) PackageManager.install(package_name=package, client=client, force=True) UpdateController._log_message( 'Installed {0}'.format(package), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message( 'Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information' .format('\n - '.join([ client.ip for client in failed_clients ])), this_client.ip, 'error') return # Migrate code for client in ssh_clients: try: UpdateController._log_message('Started code migration', client.ip) try: with Remote(client.ip, [Migrator]) as remote: remote.Migrator.migrate(master_ips, extra_ips) except EOFError as eof: UpdateController._log_message( 'EOFError during code migration, retrying {0}'. format(eof), client.ip, 'warning') with Remote(client.ip, [Migrator]) as remote: remote.Migrator.migrate(master_ips, extra_ips) UpdateController._log_message('Finished code migration', client.ip) except Exception as ex: UpdateController._remove_lock_files( [upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Code migration failed with error: {0}'.format(ex), client.ip, 'error') return # Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) model_services = [] if 'arakoon-ovsdb' in services_to_restart: model_services.append('arakoon-ovsdb') services_to_restart.remove('arakoon-ovsdb') if 'memcached' in services_to_restart: model_services.append('memcached') services_to_restart.remove('memcached') UpdateController._change_services_state(services=model_services, ssh_clients=ssh_clients, action='start') # Migrate model UpdateController._log_message('Started model migration', client_ip=this_client.ip) try: from ovs.dal.helpers import Migration Migration.migrate() UpdateController._log_message('Finished model migration', client_ip=this_client.ip) except Exception as ex: UpdateController._remove_lock_files( [upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'An unexpected error occurred: {0}'.format(ex), client_ip=this_client.ip, severity='error') return # Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: with Remote(client.ip, [Toolbox, SSHClient]) as remote: for function in remote.Toolbox.fetch_hooks( 'update', 'postupgrade'): UpdateController._log_message( 'Executing action {0}'.format(function.__name__), client_ip=client.ip) try: function( remote.SSHClient(client.ip, username='******')) UpdateController._log_message( 'Executing action {0} completed'.format( function.__name__), client_ip=client.ip) except Exception as ex: UpdateController._log_message( 'Post upgrade action failed with error: {0}'. format(ex), client.ip, 'error') # Start watcher and restart support-agent UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._change_services_state(services=['support-agent'], ssh_clients=ssh_clients, action='restart') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: if 'Could not acquire lock' in rte.message: UpdateController._log_message( 'Another framework update is currently in progress!') else: UpdateController._log_message( 'Error during framework update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) except Exception as ex: UpdateController._log_message( 'Error during framework update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: file_mutex.release()
def update_volumedriver(): """ Update the volumedriver :return: None """ file_mutex = FileMutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: file_mutex.acquire() UpdateController._log_message( '+++ Starting volumedriver update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message( 'Generating SSH client connections for each storage router') storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [ SSHClient(storage_router.ip, 'root') for storage_router in storage_routers ] this_client = [ client for client in ssh_clients if client.is_local is True ][0] # Commence update !!!!!!! # 0. Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run( 'touch {0}'.format(upgrade_file) ) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file) ) # Prevents clicking x times on 'Update' btn # 1. Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message( 'Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'volumedriver': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append( service ) # Filter out duplicates keeping the order of services (eg: watcher-framework before memcached) UpdateController._log_message( 'Services which will be restarted --> {0}'.format( ', '.join(services_to_restart))) UpdateController._log_message( 'Packages which will be installed --> {0}'.format( ', '.join(packages_to_update))) # 1. Stop services if UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message( 'Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to stop all required services, update aborted', client_ip=this_client.ip, severity='error') return # 2. Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: for package_name in packages_to_update: UpdateController._log_message( 'Installing {0}'.format(package_name), client.ip) PackageManager.install(package_name=package_name, client=client, force=True) UpdateController._log_message( 'Installed {0}'.format(package_name), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message( 'Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information' .format('\n - '.join([ client.ip for client in failed_clients ])), this_client.ip, 'error') return # 3. Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'postupgrade'): UpdateController._log_message( 'Executing action: {0}'.format(function.__name__), client_ip=client.ip) try: function(client) except Exception as ex: UpdateController._log_message( 'Post upgrade action failed with error: {0}'. format(ex), client.ip, 'error') # 4. Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: if 'Could not acquire lock' in rte.message: UpdateController._log_message( 'Another volumedriver update is currently in progress!') else: UpdateController._log_message( 'Error during volumedriver update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) except Exception as ex: UpdateController._log_message( 'Error during volumedriver update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: file_mutex.release()