def package_install_core(client, package_info, components): """ Update the core packages :param client: Client on which to execute update the packages :type client: SSHClient :param package_info: Information about the packages (installed, candidate) :type package_info: dict :param components: Components which have been selected for update :type components: list :return: None """ if 'framework' not in components and 'storagedriver' not in components: return packages_to_install = {} for pkg_name, pkg_info in package_info.iteritems(): if pkg_name in UpdateController.all_core_packages: packages_to_install[pkg_name] = pkg_info if not packages_to_install: return UpdateController._logger.debug('{0}: Executing hook {1}'.format(client.ip, inspect.currentframe().f_code.co_name)) for pkg_name, pkg_info in packages_to_install.iteritems(): UpdateController._logger.debug('{0}: Updating core package {1} ({2} --> {3})'.format(client.ip, pkg_name, pkg_info['installed'], pkg_info['candidate'])) PackageManager.install(package_name=pkg_name, client=client) UpdateController._logger.debug('{0}: Updated core package {1}'.format(client.ip, pkg_name)) UpdateController._logger.debug('{0}: Executed hook {1}'.format(client.ip, inspect.currentframe().f_code.co_name))
def update_framework(): """ Update the framework :return: None """ filemutex = file_mutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: filemutex.acquire() UpdateController._log_message('+++ Starting framework update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message('Generating SSH client connections for each storage router') upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [] master_ips = [] extra_ips = [] for sr in storage_routers: ssh_clients.append(SSHClient(sr.ip, username='******')) if sr.node_type == 'MASTER': master_ips.append(sr.ip) elif sr.node_type == 'EXTRA': extra_ips.append(sr.ip) this_client = [client for client in ssh_clients if client.is_local is True][0] # Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run('touch {0}'.format(upgrade_file)) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file)) # Prevents clicking x times on 'Update' btn # Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message('Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'framework': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append(service) # Filter out duplicates maintaining the order of services (eg: watcher-framework before memcached) UpdateController._log_message('Services which will be restarted --> {0}'.format(', '.join(services_to_restart))) UpdateController._log_message('Packages which will be installed --> {0}'.format(', '.join(packages_to_update))) # Stop services if UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message('Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) # Start services again if a service could not be stopped UpdateController._log_message('Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to stop all required services, aborting update', client_ip=this_client.ip, severity='error') return # Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: UpdateController._log_message('Installing latest packages', client.ip) for package in packages_to_update: UpdateController._log_message('Installing {0}'.format(package), client.ip) PackageManager.install(package_name=package, client=client, force=True) UpdateController._log_message('Installed {0}'.format(package), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message('Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'.format('\n - '.join([client.ip for client in failed_clients]), this_client.ip), this_client.ip, 'error') return # Migrate code for client in ssh_clients: try: UpdateController._log_message('Started code migration', client.ip) try: with remote(client.ip, [Migrator]) as rem: rem.Migrator.migrate(master_ips, extra_ips) except EOFError as eof: UpdateController._log_message('EOFError during code migration, retrying {0}'.format(eof), client.ip, 'warning') with remote(client.ip, [Migrator]) as rem: rem.Migrator.migrate(master_ips, extra_ips) UpdateController._log_message('Finished code migration', client.ip) except Exception as ex: UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Code migration failed with error: {0}'.format(ex), client.ip, 'error') return # Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) model_services = [] if 'arakoon-ovsdb' in services_to_restart: model_services.append('arakoon-ovsdb') services_to_restart.remove('arakoon-ovsdb') if 'memcached' in services_to_restart: model_services.append('memcached') services_to_restart.remove('memcached') UpdateController._change_services_state(services=model_services, ssh_clients=ssh_clients, action='start') # Migrate model UpdateController._log_message('Started model migration', client_ip=this_client.ip) try: from ovs.dal.helpers import Migration with remote(ssh_clients[0].ip, [Migration]) as rem: rem.Migration.migrate() UpdateController._log_message('Finished model migration', client_ip=this_client.ip) except Exception as ex: UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('An unexpected error occurred: {0}'.format(ex), client_ip=this_client.ip, severity='error') return # Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: with remote(client.ip, [Toolbox, SSHClient]) as rem: for function in rem.Toolbox.fetch_hooks('update', 'postupgrade'): UpdateController._log_message('Executing action {0}'.format(function.__name__), client_ip=client.ip) try: function(rem.SSHClient(client.ip, username='******')) UpdateController._log_message('Executing action {0} completed'.format(function.__name__), client_ip=client.ip) except Exception as ex: UpdateController._log_message('Post upgrade action failed with error: {0}'.format(ex), client.ip, 'error') # Start watcher and restart support-agent UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._change_services_state(services=['support-agent'], ssh_clients=ssh_clients, action='restart') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: UpdateController._log_message('Error during framework update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) except NoLockAvailableException: UpdateController._log_message('Another framework update is currently in progress!') except Exception as ex: UpdateController._log_message('Error during framework update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: filemutex.release()
def update_volumedriver(): """ Update the volumedriver :return: None """ filemutex = file_mutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: filemutex.acquire() UpdateController._log_message('+++ Starting volumedriver update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message('Generating SSH client connections for each storage router') storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [SSHClient(storage_router.ip, 'root') for storage_router in storage_routers] this_client = [client for client in ssh_clients if client.is_local is True][0] # Commence update !!!!!!! # 0. Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run('touch {0}'.format(upgrade_file)) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file)) # Prevents clicking x times on 'Update' btn # 1. Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message('Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'volumedriver': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append(service) # Filter out duplicates keeping the order of services (eg: watcher-framework before memcached) UpdateController._log_message('Services which will be restarted --> {0}'.format(', '.join(services_to_restart))) UpdateController._log_message('Packages which will be installed --> {0}'.format(', '.join(packages_to_update))) # 1. Stop services if UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message('Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to stop all required services, update aborted', client_ip=this_client.ip, severity='error') return # 2. Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: for package_name in packages_to_update: UpdateController._log_message('Installing {0}'.format(package_name), client.ip) PackageManager.install(package_name=package_name, client=client, force=True) UpdateController._log_message('Installed {0}'.format(package_name), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message('Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message('Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information'.format('\n - '.join([client.ip for client in failed_clients]), this_client.ip), this_client.ip, 'error') return # 3. Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'postupgrade'): UpdateController._log_message('Executing action: {0}'.format(function.__name__), client_ip=client.ip) try: function(client) except Exception as ex: UpdateController._log_message('Post upgrade action failed with error: {0}'.format(ex), client.ip, 'error') # 4. Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) UpdateController._change_services_state(services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: UpdateController._log_message('Error during volumedriver update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) except NoLockAvailableException: UpdateController._log_message('Another volumedriver update is currently in progress!') except Exception as ex: UpdateController._log_message('Error during volumedriver update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files([upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: filemutex.release()
def update_framework(): """ Update the framework :return: None """ file_mutex = FileMutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: file_mutex.acquire() UpdateController._log_message('+++ Starting framework update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message( 'Generating SSH client connections for each storage router') upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [] master_ips = [] extra_ips = [] for sr in storage_routers: ssh_clients.append(SSHClient(sr.ip, username='******')) if sr.node_type == 'MASTER': master_ips.append(sr.ip) elif sr.node_type == 'EXTRA': extra_ips.append(sr.ip) this_client = [ client for client in ssh_clients if client.is_local is True ][0] # Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run( 'touch {0}'.format(upgrade_file) ) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file) ) # Prevents clicking x times on 'Update' btn # Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message( 'Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'framework': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append( service ) # Filter out duplicates maintaining the order of services (eg: watcher-framework before memcached) UpdateController._log_message( 'Services which will be restarted --> {0}'.format( ', '.join(services_to_restart))) UpdateController._log_message( 'Packages which will be installed --> {0}'.format( ', '.join(packages_to_update))) # Stop services if UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message( 'Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) # Start services again if a service could not be stopped UpdateController._log_message( 'Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to stop all required services, aborting update', client_ip=this_client.ip, severity='error') return # Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: UpdateController._log_message('Installing latest packages', client.ip) for package in packages_to_update: UpdateController._log_message( 'Installing {0}'.format(package), client.ip) PackageManager.install(package_name=package, client=client, force=True) UpdateController._log_message( 'Installed {0}'.format(package), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message( 'Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information' .format('\n - '.join([ client.ip for client in failed_clients ])), this_client.ip, 'error') return # Migrate code for client in ssh_clients: try: UpdateController._log_message('Started code migration', client.ip) try: with Remote(client.ip, [Migrator]) as remote: remote.Migrator.migrate(master_ips, extra_ips) except EOFError as eof: UpdateController._log_message( 'EOFError during code migration, retrying {0}'. format(eof), client.ip, 'warning') with Remote(client.ip, [Migrator]) as remote: remote.Migrator.migrate(master_ips, extra_ips) UpdateController._log_message('Finished code migration', client.ip) except Exception as ex: UpdateController._remove_lock_files( [upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Code migration failed with error: {0}'.format(ex), client.ip, 'error') return # Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) model_services = [] if 'arakoon-ovsdb' in services_to_restart: model_services.append('arakoon-ovsdb') services_to_restart.remove('arakoon-ovsdb') if 'memcached' in services_to_restart: model_services.append('memcached') services_to_restart.remove('memcached') UpdateController._change_services_state(services=model_services, ssh_clients=ssh_clients, action='start') # Migrate model UpdateController._log_message('Started model migration', client_ip=this_client.ip) try: from ovs.dal.helpers import Migration Migration.migrate() UpdateController._log_message('Finished model migration', client_ip=this_client.ip) except Exception as ex: UpdateController._remove_lock_files( [upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'An unexpected error occurred: {0}'.format(ex), client_ip=this_client.ip, severity='error') return # Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: with Remote(client.ip, [Toolbox, SSHClient]) as remote: for function in remote.Toolbox.fetch_hooks( 'update', 'postupgrade'): UpdateController._log_message( 'Executing action {0}'.format(function.__name__), client_ip=client.ip) try: function( remote.SSHClient(client.ip, username='******')) UpdateController._log_message( 'Executing action {0} completed'.format( function.__name__), client_ip=client.ip) except Exception as ex: UpdateController._log_message( 'Post upgrade action failed with error: {0}'. format(ex), client.ip, 'error') # Start watcher and restart support-agent UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._change_services_state(services=['support-agent'], ssh_clients=ssh_clients, action='restart') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: if 'Could not acquire lock' in rte.message: UpdateController._log_message( 'Another framework update is currently in progress!') else: UpdateController._log_message( 'Error during framework update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) except Exception as ex: UpdateController._log_message( 'Error during framework update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: file_mutex.release()
def update_volumedriver(): """ Update the volumedriver :return: None """ file_mutex = FileMutex('system_update', wait=2) upgrade_file = '/etc/ready_for_upgrade' upgrade_ongoing_check_file = '/etc/upgrade_ongoing' ssh_clients = [] try: file_mutex.acquire() UpdateController._log_message( '+++ Starting volumedriver update +++') from ovs.dal.lists.storagerouterlist import StorageRouterList UpdateController._log_message( 'Generating SSH client connections for each storage router') storage_routers = StorageRouterList.get_storagerouters() ssh_clients = [ SSHClient(storage_router.ip, 'root') for storage_router in storage_routers ] this_client = [ client for client in ssh_clients if client.is_local is True ][0] # Commence update !!!!!!! # 0. Create locks UpdateController._log_message('Creating lock files', client_ip=this_client.ip) for client in ssh_clients: client.run( 'touch {0}'.format(upgrade_file) ) # Prevents manual install or upgrade individual packages client.run('touch {0}'.format(upgrade_ongoing_check_file) ) # Prevents clicking x times on 'Update' btn # 1. Check requirements packages_to_update = set() all_services_to_restart = [] for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'metadata'): UpdateController._log_message( 'Executing function {0}'.format(function.__name__), client_ip=client.ip) output = function(client) for key, value in output.iteritems(): if key != 'volumedriver': continue for package_info in value: packages_to_update.update(package_info['packages']) all_services_to_restart += package_info['services'] services_to_restart = [] for service in all_services_to_restart: if service not in services_to_restart: services_to_restart.append( service ) # Filter out duplicates keeping the order of services (eg: watcher-framework before memcached) UpdateController._log_message( 'Services which will be restarted --> {0}'.format( ', '.join(services_to_restart))) UpdateController._log_message( 'Packages which will be installed --> {0}'.format( ', '.join(packages_to_update))) # 1. Stop services if UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='stop') is False: UpdateController._log_message( 'Stopping all services on every node failed, cannot continue', client_ip=this_client.ip, severity='warning') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Attempting to start the services again', client_ip=this_client.ip) UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to stop all required services, update aborted', client_ip=this_client.ip, severity='error') return # 2. Update packages failed_clients = [] for client in ssh_clients: PackageManager.update(client=client) try: for package_name in packages_to_update: UpdateController._log_message( 'Installing {0}'.format(package_name), client.ip) PackageManager.install(package_name=package_name, client=client, force=True) UpdateController._log_message( 'Installed {0}'.format(package_name), client.ip) client.file_delete(upgrade_file) except subprocess.CalledProcessError as cpe: UpdateController._log_message( 'Upgrade failed with error: {0}'.format(cpe.output), client.ip, 'error') failed_clients.append(client) break if failed_clients: UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message( 'Error occurred. Attempting to start all services again', client_ip=this_client.ip, severity='error') UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._log_message( 'Failed to upgrade following nodes:\n - {0}\nPlease check /var/log/ovs/lib.log on {1} for more information' .format('\n - '.join([ client.ip for client in failed_clients ])), this_client.ip, 'error') return # 3. Post upgrade actions UpdateController._log_message('Executing post upgrade actions', client_ip=this_client.ip) for client in ssh_clients: for function in Toolbox.fetch_hooks('update', 'postupgrade'): UpdateController._log_message( 'Executing action: {0}'.format(function.__name__), client_ip=client.ip) try: function(client) except Exception as ex: UpdateController._log_message( 'Post upgrade action failed with error: {0}'. format(ex), client.ip, 'error') # 4. Start services UpdateController._log_message('Starting services', client_ip=this_client.ip) UpdateController._change_services_state( services=services_to_restart, ssh_clients=ssh_clients, action='start') UpdateController._remove_lock_files([upgrade_ongoing_check_file], ssh_clients) UpdateController._log_message('+++ Finished updating +++') except RuntimeError as rte: if 'Could not acquire lock' in rte.message: UpdateController._log_message( 'Another volumedriver update is currently in progress!') else: UpdateController._log_message( 'Error during volumedriver update: {0}'.format(rte), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) except Exception as ex: UpdateController._log_message( 'Error during volumedriver update: {0}'.format(ex), severity='error') UpdateController._remove_lock_files( [upgrade_file, upgrade_ongoing_check_file], ssh_clients) finally: file_mutex.release()