def change_service_state(client, name, state, logger=None): """ Starts/stops/restarts a service :param client: SSHClient on which to connect and change service state :param name: Name of the service :param state: State to put the service in :param logger: LogHandler Object """ action = None status, _ = ServiceManager.get_service_status(name, client=client) if status is False and state in ['start', 'restart']: if logger is not None: logger.debug(' {0:<15} - Starting service {1}'.format(client.ip, name)) ServiceManager.start_service(name, client=client) action = 'started' elif status is True and state == 'stop': if logger is not None: logger.debug(' {0:<15} - Stopping service {1}'.format(client.ip, name)) ServiceManager.stop_service(name, client=client) action = 'stopped' elif status is True and state == 'restart': if logger is not None: logger.debug(' {0:<15} - Restarting service {1}'.format(client.ip, name)) ServiceManager.restart_service(name, client=client) action = 'restarted' if action is None: print ' [{0}] {1} already {2}'.format(client.ip, name, 'running' if status is True else 'halted') else: logger.debug(' {0:<15} - Service {1} {2}'.format(client.ip, name, action)) print ' [{0}] {1} {2}'.format(client.ip, name, action)
def start(cluster_name, client): """ Starts an arakoon cluster """ if ServiceManager.get_service_status( 'arakoon-{0}'.format(cluster_name), client=client) is False: ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def start_service(name, client): """ Start a service :param name: Name of the service :param client: SSHClient object :return: None """ ServiceManager.start_service(name, client)
def start(cluster_name, client): """ Starts an etcd cluster :param client: Client on which to start the service :param cluster_name: The name of the cluster service to start """ if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True: ServiceManager.start_service('etcd-{0}'.format(cluster_name), client=client)
def start(cluster_name, client): """ Starts an arakoon cluster :param client: Client on which to start the service :param cluster_name: The name of the cluster service to start """ if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True and \ ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client) is False: ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def _enable_openstack_events_consumer(self): """ Enable service ovs-openstack-events-consumer """ from ovs.extensions.services.service import ServiceManager service_name = 'ovs-openstack-events-consumer' if not ServiceManager.has_service(service_name, self.client): ServiceManager.add_service(service_name, self.client) ServiceManager.enable_service(service_name, self.client) ServiceManager.start_service(service_name, self.client)
def start(cluster_name, client): """ Starts an etcd cluster :param client: Client on which to start the service :param cluster_name: The name of the cluster service to start """ if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True and \ ServiceManager.get_service_status('etcd-{0}'.format(cluster_name), client=client) is False: ServiceManager.start_service('etcd-{0}'.format(cluster_name), client=client)
def _roll_out_dtl_services(vpool, storagerouters): """ Deploy and start the DTL service on all storagerouters :param storagerouters: StorageRouters to deploy and start a DTL service on :return: None """ service_name = 'dtl_{0}'.format(vpool.name) for sr in storagerouters.values(): client = SSHClient(sr, 'root') ServiceManager.add_service(name=service_name, client=client) ServiceManager.start_service(name=service_name, client=client)
def start(cluster_name, client): """ Starts an arakoon cluster :param cluster_name: The name of the cluster service to start :type cluster_name: str :param client: Client on which to start the service :type client: SSHClient :return: None """ service_name = ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name) if ServiceManager.has_service(name=service_name, client=client) is True: ServiceManager.start_service(name=service_name, client=client)
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed from ovs.dal.lists.storagerouterlist import StorageRouterList clients = [] try: for storagerouter in StorageRouterList.get_storagerouters(): clients.append(SSHClient(storagerouter, username='******')) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') for client in clients: for service_name in ['watcher-framework', 'memcached']: ServiceManager.stop_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status( service_name, client=client) is False: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError( 'Could not stop service: {0}'.format(service_name)) for client in clients: for service_name in ['memcached', 'watcher-framework']: ServiceManager.start_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status( service_name, client=client) is True: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError( 'Could not start service: {0}'.format( service_name)) from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') for function in functions: function(ip=ip)
def start(cluster_name, client): """ Starts an arakoon cluster :param cluster_name: The name of the cluster service to start :type cluster_name: str :param client: Client on which to start the service :type client: SSHClient :return: None """ if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True: ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed from ovs.dal.lists.storagerouterlist import StorageRouterList clients = [] try: for storagerouter in StorageRouterList.get_storagerouters(): clients.append(SSHClient(storagerouter, username='******')) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') for client in clients: for service_name in ['watcher-framework', 'memcached']: ServiceManager.stop_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status(service_name, client=client) is False: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError('Could not stop service: {0}'.format(service_name)) for client in clients: for service_name in ['memcached', 'watcher-framework']: ServiceManager.start_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status(service_name, client=client) is True: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError('Could not start service: {0}'.format(service_name)) from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') for function in functions: function(ip=ip)
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed print 'Installing plugin into Open vStorage' from ovs.dal.lists.storagerouterlist import StorageRouterList clients = {} masters = StorageRouterList.get_masters() slaves = StorageRouterList.get_slaves() try: for sr in masters + slaves: clients[sr] = SSHClient(sr, username='******') except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') memcached = 'memcached' watcher = 'watcher-framework' for sr in masters + slaves: if ServiceManager.has_service(watcher, clients[sr]): print '- Stopping watcher on {0} ({1})'.format( sr.name, sr.ip) ServiceManager.stop_service(watcher, clients[sr]) for sr in masters: print '- Restarting memcached on {0} ({1})'.format( sr.name, sr.ip) ServiceManager.restart_service(memcached, clients[sr]) for sr in masters + slaves: if ServiceManager.has_service(watcher, clients[sr]): print '- Starting watcher on {0} ({1})'.format( sr.name, sr.ip) ServiceManager.start_service(watcher, clients[sr]) print '- Execute model migrations' from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') if len(functions) > 0: print '- Execute post installation scripts' for function in functions: function(ip=ip) print 'Installing plugin into Open vStorage: Completed'
def change_service_state(client, name, state, logger=None): """ Starts/stops/restarts a service :param client: SSHClient on which to connect and change service state :param name: Name of the service :param state: State to put the service in :param logger: LogHandler Object """ action = None # Enable service before changing the state status = ServiceManager.is_enabled(name, client=client) if status is False: if logger is not None: logger.debug(' {0:<15} - Enabling service {1}'.format( client.ip, name)) ServiceManager.enable_service(name, client=client) status = ServiceManager.get_service_status(name, client=client) if status is False and state in ['start', 'restart']: if logger is not None: logger.debug(' {0:<15} - Starting service {1}'.format( client.ip, name)) ServiceManager.start_service(name, client=client) action = 'started' elif status is True and state == 'stop': if logger is not None: logger.debug(' {0:<15} - Stopping service {1}'.format( client.ip, name)) ServiceManager.stop_service(name, client=client) action = 'stopped' elif status is True and state == 'restart': if logger is not None: logger.debug(' {0:<15} - Restarting service {1}'.format( client.ip, name)) ServiceManager.restart_service(name, client=client) action = 'restarted' if action is None: print ' [{0}] {1} already {2}'.format( client.ip, name, 'running' if status is True else 'halted') else: logger.debug(' {0:<15} - Service {1} {2}'.format( client.ip, name, action)) print ' [{0}] {1} {2}'.format(client.ip, name, action)
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed print 'Installing plugin into Open vStorage' from ovs.dal.lists.storagerouterlist import StorageRouterList clients = {} masters = StorageRouterList.get_masters() slaves = StorageRouterList.get_slaves() try: for sr in masters + slaves: clients[sr] = SSHClient(sr, username='******') except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') memcached = 'memcached' watcher = 'watcher-framework' for sr in masters + slaves: if ServiceManager.has_service(watcher, clients[sr]): print '- Stopping watcher on {0} ({1})'.format(sr.name, sr.ip) ServiceManager.stop_service(watcher, clients[sr]) for sr in masters: print '- Restarting memcached on {0} ({1})'.format(sr.name, sr.ip) ServiceManager.restart_service(memcached, clients[sr]) for sr in masters + slaves: if ServiceManager.has_service(watcher, clients[sr]): print '- Starting watcher on {0} ({1})'.format(sr.name, sr.ip) ServiceManager.start_service(watcher, clients[sr]) print '- Execute model migrations' from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') if len(functions) > 0: print '- Execute post installation scripts' for function in functions: function(ip=ip) print 'Installing plugin into Open vStorage: Completed'
def execute_scrub_work(queue, vpool, scrub_info, error_messages): """ Executes scrub work for a given vDisk queue and vPool, based on scrub_info :param queue: a Queue with vDisk guids that need to be scrubbed (they should only be member of a single vPool) :type queue: Queue :param vpool: the vPool object of the vDisks :type vpool: VPool :param scrub_info: A dict containing scrub information: `scrub_path` with the path where to scrub and `storage_router` with the StorageRouter that needs to do the work :type scrub_info: dict :param error_messages: A list of error messages to be filled :type error_messages: list :return: a list of error messages :rtype: list """ def _verify_mds_config(current_vdisk): current_vdisk.invalidate_dynamics('info') vdisk_configs = current_vdisk.info['metadata_backend_config'] if len(vdisk_configs) == 0: raise RuntimeError('Could not load MDS configuration') return vdisk_configs client = None lock_time = 5 * 60 storagerouter = scrub_info['storage_router'] scrub_directory = '{0}/scrub_work_{1}_{2}'.format( scrub_info['scrub_path'], vpool.name, storagerouter.name) scrub_config_key = 'ovs/vpools/{0}/proxies/scrub/scrub_config_{1}'.format( vpool.guid, storagerouter.guid) backend_config_key = 'ovs/vpools/{0}/proxies/scrub/backend_config_{1}'.format( vpool.guid, storagerouter.guid) alba_proxy_service = 'ovs-albaproxy_{0}_{1}_scrub'.format( vpool.name, storagerouter.name) # Deploy a proxy try: with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time): ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - Deploying ALBA proxy {2}' .format(vpool.name, storagerouter.name, alba_proxy_service)) client = SSHClient(storagerouter, 'root') client.dir_create(scrub_directory) client.dir_chmod( scrub_directory, 0777 ) # Celery task executed by 'ovs' user and should be able to write in it if ServiceManager.has_service( name=alba_proxy_service, client=client ) is True and ServiceManager.get_service_status( name=alba_proxy_service, client=client) is True: ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - Re-using existing proxy service {2}' .format(vpool.name, storagerouter.name, alba_proxy_service)) scrub_config = Configuration.get(scrub_config_key) else: machine_id = System.get_my_machine_id(client) port_range = Configuration.get( '/ovs/framework/hosts/{0}/ports|storagedriver'.format( machine_id)) port = System.get_free_ports(selected_range=port_range, nr=1, client=client)[0] # Scrub config # {u'albamgr_cfg_url': u'arakoon://config/ovs/vpools/71e2f717-f270-4a41-bbb0-d4c8c084d43e/proxies/64759516-3471-4321-b912-fb424568fc5b/config/abm?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini', # u'fragment_cache': [u'none'], # u'ips': [u'127.0.0.1'], # u'log_level': u'info', # u'manifest_cache_size': 17179869184, # u'port': 0, # u'transport': u'tcp'} # Backend config # {u'alba_connection_host': u'10.100.193.155', # u'alba_connection_port': 26204, # u'alba_connection_preset': u'preset', # u'alba_connection_timeout': 15, # u'alba_connection_transport': u'TCP', # u'backend_interface_retries_on_error': 5, # u'backend_interface_retry_backoff_multiplier': 2.0, # u'backend_interface_retry_interval_secs': 1, # u'backend_type': u'ALBA'} scrub_config = Configuration.get( 'ovs/vpools/{0}/proxies/scrub/generic_scrub'.format( vpool.guid)) scrub_config['port'] = port scrub_config['transport'] = 'tcp' Configuration.set(scrub_config_key, json.dumps(scrub_config, indent=4), raw=True) params = { 'VPOOL_NAME': vpool.name, 'LOG_SINK': LogHandler.get_sink_path('alba_proxy'), 'CONFIG_PATH': Configuration.get_configuration_path(scrub_config_key) } ServiceManager.add_service(name='ovs-albaproxy', params=params, client=client, target_name=alba_proxy_service) ServiceManager.start_service(name=alba_proxy_service, client=client) ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - Deployed ALBA proxy {2}' .format(vpool.name, storagerouter.name, alba_proxy_service)) backend_config = Configuration.get( 'ovs/vpools/{0}/hosts/{1}/config'.format( vpool.guid, vpool.storagedrivers[0].storagedriver_id ))['backend_connection_manager'] backend_config['alba_connection_host'] = '127.0.0.1' backend_config['alba_connection_port'] = scrub_config['port'] Configuration.set( backend_config_key, json.dumps({"backend_connection_manager": backend_config}, indent=4), raw=True) except Exception: message = 'Scrubber - vPool {0} - StorageRouter {1} - An error occurred deploying ALBA proxy {2}'.format( vpool.name, storagerouter.name, alba_proxy_service) error_messages.append(message) ScheduledTaskController._logger.exception(message) if client is not None and ServiceManager.has_service( name=alba_proxy_service, client=client) is True: if ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True: ServiceManager.stop_service(name=alba_proxy_service, client=client) ServiceManager.remove_service(name=alba_proxy_service, client=client) if Configuration.exists(scrub_config_key): Configuration.delete(scrub_config_key) try: # Empty the queue with vDisks to scrub with remote(storagerouter.ip, [VDisk]) as rem: while True: vdisk = None vdisk_guid = queue.get(False) try: # Check MDS master is local. Trigger MDS handover if necessary vdisk = rem.VDisk(vdisk_guid) ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Started scrubbing at location {3}' .format(vpool.name, storagerouter.name, vdisk.name, scrub_directory)) configs = _verify_mds_config(current_vdisk=vdisk) storagedriver = StorageDriverList.get_by_storagedriver_id( vdisk.storagedriver_id) if configs[0].get( 'ip') != storagedriver.storagerouter.ip: ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - MDS master is not local, trigger handover' .format(vpool.name, storagerouter.name, vdisk.name)) MDSServiceController.ensure_safety( VDisk(vdisk_guid) ) # Do not use a remote VDisk instance here configs = _verify_mds_config(current_vdisk=vdisk) if configs[0].get( 'ip') != storagedriver.storagerouter.ip: ScheduledTaskController._logger.warning( 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Skipping because master MDS still not local' .format(vpool.name, storagerouter.name, vdisk.name)) continue # Do the actual scrubbing with vdisk.storagedriver_client.make_locked_client( str(vdisk.volume_id)) as locked_client: ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Retrieve and apply scrub work' .format(vpool.name, storagerouter.name, vdisk.name)) work_units = locked_client.get_scrubbing_workunits( ) for work_unit in work_units: res = locked_client.scrub( work_unit=work_unit, scratch_dir=scrub_directory, log_sinks=[ LogHandler.get_sink_path( 'scrubber', allow_override=True) ], backend_config=Configuration. get_configuration_path(backend_config_key)) locked_client.apply_scrubbing_result( scrubbing_work_result=res) if work_units: ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - {3} work units successfully applied' .format(vpool.name, storagerouter.name, vdisk.name, len(work_units))) else: ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - No scrubbing required' .format(vpool.name, storagerouter.name, vdisk.name)) except Exception: if vdisk is None: message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk with guid {2} could not be found'.format( vpool.name, storagerouter.name, vdisk_guid) else: message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Scrubbing failed'.format( vpool.name, storagerouter.name, vdisk.name) error_messages.append(message) ScheduledTaskController._logger.exception(message) except Empty: # Raised when all items have been fetched from the queue ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - Queue completely processed' .format(vpool.name, storagerouter.name)) except Exception: message = 'Scrubber - vPool {0} - StorageRouter {1} - Scrubbing failed'.format( vpool.name, storagerouter.name) error_messages.append(message) ScheduledTaskController._logger.exception(message) # Delete the proxy again try: with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time): ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2}' .format(vpool.name, storagerouter.name, alba_proxy_service)) client = SSHClient(storagerouter, 'root') client.dir_delete(scrub_directory) if ServiceManager.has_service(alba_proxy_service, client=client): ServiceManager.stop_service(alba_proxy_service, client=client) ServiceManager.remove_service(alba_proxy_service, client=client) if Configuration.exists(scrub_config_key): Configuration.delete(scrub_config_key) ScheduledTaskController._logger.info( 'Scrubber - vPool {0} - StorageRouter {1} - Removed service {2}' .format(vpool.name, storagerouter.name, alba_proxy_service)) except Exception: message = 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2} failed'.format( vpool.name, storagerouter.name, alba_proxy_service) error_messages.append(message) ScheduledTaskController._logger.exception(message)
def configure_host(self, ip): if self._is_devstack is False and self._is_openstack is False or self._cinder_installed is False or self._nova_installed is False: self._logger.warning('Configure host: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed') return # 1. Get Driver code self._logger.info('*** Configuring host with IP {0} ***'.format(ip)) self._logger.info(' Copy driver code') remote_driver = "/opt/OpenvStorage/config/templates/cinder-volume-driver/{0}/openvstorage.py".format(self._stack_version) remote_version = '0.0.0' existing_version = '0.0.0' try: from cinder.volume.drivers import openvstorage if hasattr(openvstorage, 'OVSVolumeDriver'): existing_version = getattr(openvstorage.OVSVolumeDriver, 'VERSION', '0.0.0') except ImportError: pass for line in self.client.file_read(remote_driver).splitlines(): if 'VERSION = ' in line: remote_version = line.split('VERSION = ')[-1].strip().replace("'", "").replace('"', "") break nova_base_path = self._get_base_path('nova') cinder_base_path = self._get_base_path('cinder') if self._is_devstack is True: local_driver = '{0}/volume/drivers/openvstorage.py'.format(cinder_base_path) else: local_driver = '{0}/cinder/volume/drivers/openvstorage.py'.format(self._driver_location) if remote_version > existing_version: self._logger.debug('Updating existing driver using {0} from version {1} to version {2}'.format(remote_driver, existing_version, remote_version)) self.client.run('cp -f {0} {1}'.format(remote_driver, local_driver)) else: self._logger.debug('Using driver {0} version {1}'.format(local_driver, existing_version)) # 2. Configure users and groups self._logger.info(' Add users to group ovs') users = ['libvirt-qemu', 'stack'] if self._is_devstack is True else self._openstack_users for user in users: self.client.run('usermod -a -G ovs {0}'.format(user)) # 3. Apply patches self._logger.info(' Applying patches') if self._stack_version in ('liberty', 'mitaka', 'newton'): try: import os_brick cinder_brick_initiator_file = "{0}/initiator/connector.py".format(os.path.dirname(os_brick.__file__)) except ImportError: cinder_brick_initiator_file = '' if self._is_devstack is True: nova_volume_file = '{0}/virt/libvirt/volume/volume.py'.format(nova_base_path) else: nova_volume_file = '{0}/nova/virt/libvirt/volume/volume.py'.format(self._driver_location) else: cinder_brick_initiator_file = '{0}/cinder/brick/initiator/connector.py'.format(self._driver_location) if self._is_devstack is True: nova_volume_file = '{0}/virt/libvirt/volume.py'.format(nova_base_path) else: nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format(self._driver_location) if self._is_devstack is True: nova_driver_file = '{0}/virt/libvirt/driver.py'.format(nova_base_path) else: nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format(self._driver_location) self._logger.info(' Patching file {0}'.format(nova_volume_file)) file_contents = self.client.file_read(nova_volume_file) if 'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):' not in file_contents: file_contents += ''' class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver): def __init__(self, connection): super(LibvirtFileVolumeDriver, self).__init__(connection, is_block_dev=False) def get_config(self, connection_info, disk_info): conf = super(LibvirtFileVolumeDriver, self).get_config(connection_info, disk_info) conf.source_type = 'file' conf.source_path = connection_info['data']['device_path'] return conf ''' self.client.file_write(nova_volume_file, file_contents) self._logger.info(' Patching file {0}'.format(nova_driver_file)) file_contents = self.client.file_read(nova_driver_file) if self._stack_version in ('liberty', 'mitaka'): check_line = 'local=nova.virt.libvirt.volume.volume.LibvirtVolumeDriver' new_line = 'file=nova.virt.libvirt.volume.volume.LibvirtFileVolumeDriver' else: check_line = 'local=nova.virt.libvirt.volume.LibvirtVolumeDriver' new_line = 'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver' if new_line not in file_contents: for line in file_contents.splitlines(): if check_line in line: stripped_line = line.rstrip() whitespaces = len(stripped_line) - len(stripped_line.lstrip()) new_line = "{0}'{1}',\n".format(' ' * whitespaces, new_line) fc = file_contents[:file_contents.index(line)] + new_line + file_contents[file_contents.index(line):] self.client.file_write(nova_driver_file, "".join(fc)) break if os.path.exists(cinder_brick_initiator_file): # fix brick/upload to glance self._logger.info(' Patching file {0}'.format(cinder_brick_initiator_file)) if self._stack_version in ('liberty', 'mitaka', 'newton'): self.client.run("""sed -i 's/elif protocol == LOCAL:/elif protocol in [LOCAL, "FILE"]:/g' {0}""".format(cinder_brick_initiator_file)) else: self.client.run("""sed -i 's/elif protocol == "LOCAL":/elif protocol in ["LOCAL", "FILE"]:/g' {0}""".format(cinder_brick_initiator_file)) # 4. Configure messaging driver self._logger.info(' - Configure messaging driver') nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging' cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging' with remote(ip, [RawConfigParser, open], 'root') as rem: for config_file, driver in {self._NOVA_CONF: nova_messaging_driver, self._CINDER_CONF: cinder_messaging_driver}.iteritems(): changed = False cfg = rem.RawConfigParser() cfg.read([config_file]) if cfg.has_option("DEFAULT", "notification_driver"): if cfg.get("DEFAULT", "notification_driver") != driver: changed = True cfg.set("DEFAULT", "notification_driver", driver) else: changed = True cfg.set("DEFAULT", "notification_driver", driver) if cfg.has_option("DEFAULT", "notification_topics"): notification_topics = cfg.get("DEFAULT", "notification_topics").split(",") if "notifications" not in notification_topics: notification_topics.append("notifications") changed = True cfg.set("DEFAULT", "notification_topics", ",".join(notification_topics)) else: changed = True cfg.set("DEFAULT", "notification_topics", "notifications") if config_file == self._NOVA_CONF: for param, value in {'notify_on_any_change': 'True', 'notify_on_state_change': 'vm_and_task_state'}.iteritems(): if not cfg.has_option("DEFAULT", param): changed = True cfg.set("DEFAULT", param, value) if changed is True: with rem.open(config_file, "w") as fp: cfg.write(fp) # 5. Enable events consumer self._logger.info(' - Enabling events consumer service') service_name = 'openstack-events-consumer' if not ServiceManager.has_service(service_name, self.client): ServiceManager.add_service(service_name, self.client) ServiceManager.enable_service(service_name, self.client) ServiceManager.start_service(service_name, self.client)
def start(cluster_name, client): """ Starts an arakoon cluster """ if ServiceManager.get_service_status("arakoon-{0}".format(cluster_name), client=client) is False: ServiceManager.start_service("arakoon-{0}".format(cluster_name), client=client)
def execute_scrub_work(queue, vpool, scrub_info, error_messages): """ Executes scrub work for a given vDisk queue and vPool, based on scrub_info :param queue: a Queue with vDisk guids that need to be scrubbed (they should only be member of a single vPool) :type queue: Queue :param vpool: the vPool object of the vDisks :type vpool: VPool :param scrub_info: A dict containing scrub information: `scrub_path` with the path where to scrub and `storage_router` with the StorageRouter that needs to do the work :type scrub_info: dict :param error_messages: A list of error messages to be filled :type error_messages: list :return: a list of error messages :rtype: list """ def _verify_mds_config(current_vdisk): current_vdisk.invalidate_dynamics('info') vdisk_configs = current_vdisk.info['metadata_backend_config'] if len(vdisk_configs) == 0: raise RuntimeError('Could not load MDS configuration') return vdisk_configs client = None lock_time = 5 * 60 storagerouter = scrub_info['storage_router'] scrub_directory = '{0}/scrub_work_{1}_{2}'.format(scrub_info['scrub_path'], vpool.name, storagerouter.name) scrub_config_key = 'ovs/vpools/{0}/proxies/scrub/scrub_config_{1}'.format(vpool.guid, storagerouter.guid) backend_config_key = 'ovs/vpools/{0}/proxies/scrub/backend_config_{1}'.format(vpool.guid, storagerouter.guid) alba_proxy_service = 'ovs-albaproxy_{0}_{1}_scrub'.format(vpool.name, storagerouter.name) # Deploy a proxy try: with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time): ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Deploying ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)) client = SSHClient(storagerouter, 'root') client.dir_create(scrub_directory) client.dir_chmod(scrub_directory, 0777) # Celery task executed by 'ovs' user and should be able to write in it if ServiceManager.has_service(name=alba_proxy_service, client=client) is True and ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True: ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Re-using existing proxy service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)) scrub_config = Configuration.get(scrub_config_key) else: machine_id = System.get_my_machine_id(client) port_range = Configuration.get('/ovs/framework/hosts/{0}/ports|storagedriver'.format(machine_id)) port = System.get_free_ports(selected_range=port_range, nr=1, client=client)[0] # Scrub config # {u'albamgr_cfg_url': u'arakoon://config/ovs/vpools/71e2f717-f270-4a41-bbb0-d4c8c084d43e/proxies/64759516-3471-4321-b912-fb424568fc5b/config/abm?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini', # u'fragment_cache': [u'none'], # u'ips': [u'127.0.0.1'], # u'log_level': u'info', # u'manifest_cache_size': 17179869184, # u'port': 0, # u'transport': u'tcp'} # Backend config # {u'alba_connection_host': u'10.100.193.155', # u'alba_connection_port': 26204, # u'alba_connection_preset': u'preset', # u'alba_connection_timeout': 15, # u'alba_connection_transport': u'TCP', # u'backend_interface_retries_on_error': 5, # u'backend_interface_retry_backoff_multiplier': 2.0, # u'backend_interface_retry_interval_secs': 1, # u'backend_type': u'ALBA'} scrub_config = Configuration.get('ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid)) scrub_config['port'] = port scrub_config['transport'] = 'tcp' Configuration.set(scrub_config_key, json.dumps(scrub_config, indent=4), raw=True) params = {'VPOOL_NAME': vpool.name, 'LOG_SINK': LogHandler.get_sink_path('alba_proxy'), 'CONFIG_PATH': Configuration.get_configuration_path(scrub_config_key)} ServiceManager.add_service(name='ovs-albaproxy', params=params, client=client, target_name=alba_proxy_service) ServiceManager.start_service(name=alba_proxy_service, client=client) ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Deployed ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)) backend_config = Configuration.get('ovs/vpools/{0}/hosts/{1}/config'.format(vpool.guid, vpool.storagedrivers[0].storagedriver_id))['backend_connection_manager'] backend_config['alba_connection_host'] = '127.0.0.1' backend_config['alba_connection_port'] = scrub_config['port'] Configuration.set(backend_config_key, json.dumps({"backend_connection_manager": backend_config}, indent=4), raw=True) except Exception: message = 'Scrubber - vPool {0} - StorageRouter {1} - An error occurred deploying ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service) error_messages.append(message) ScheduledTaskController._logger.exception(message) if client is not None and ServiceManager.has_service(name=alba_proxy_service, client=client) is True: if ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True: ServiceManager.stop_service(name=alba_proxy_service, client=client) ServiceManager.remove_service(name=alba_proxy_service, client=client) if Configuration.exists(scrub_config_key): Configuration.delete(scrub_config_key) try: # Empty the queue with vDisks to scrub with remote(storagerouter.ip, [VDisk]) as rem: while True: vdisk = None vdisk_guid = queue.get(False) try: # Check MDS master is local. Trigger MDS handover if necessary vdisk = rem.VDisk(vdisk_guid) ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Started scrubbing at location {3}'.format(vpool.name, storagerouter.name, vdisk.name, scrub_directory)) configs = _verify_mds_config(current_vdisk=vdisk) storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id) if configs[0].get('ip') != storagedriver.storagerouter.ip: ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - MDS master is not local, trigger handover'.format(vpool.name, storagerouter.name, vdisk.name)) MDSServiceController.ensure_safety(VDisk(vdisk_guid)) # Do not use a remote VDisk instance here configs = _verify_mds_config(current_vdisk=vdisk) if configs[0].get('ip') != storagedriver.storagerouter.ip: ScheduledTaskController._logger.warning('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Skipping because master MDS still not local'.format(vpool.name, storagerouter.name, vdisk.name)) continue # Do the actual scrubbing with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client: ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Retrieve and apply scrub work'.format(vpool.name, storagerouter.name, vdisk.name)) work_units = locked_client.get_scrubbing_workunits() for work_unit in work_units: res = locked_client.scrub(work_unit=work_unit, scratch_dir=scrub_directory, log_sinks=[LogHandler.get_sink_path('scrubber', allow_override=True)], backend_config=Configuration.get_configuration_path(backend_config_key)) locked_client.apply_scrubbing_result(scrubbing_work_result=res) if work_units: ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - {3} work units successfully applied'.format(vpool.name, storagerouter.name, vdisk.name, len(work_units))) else: ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - No scrubbing required'.format(vpool.name, storagerouter.name, vdisk.name)) except Exception: if vdisk is None: message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk with guid {2} could not be found'.format(vpool.name, storagerouter.name, vdisk_guid) else: message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Scrubbing failed'.format(vpool.name, storagerouter.name, vdisk.name) error_messages.append(message) ScheduledTaskController._logger.exception(message) except Empty: # Raised when all items have been fetched from the queue ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Queue completely processed'.format(vpool.name, storagerouter.name)) except Exception: message = 'Scrubber - vPool {0} - StorageRouter {1} - Scrubbing failed'.format(vpool.name, storagerouter.name) error_messages.append(message) ScheduledTaskController._logger.exception(message) # Delete the proxy again try: with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time): ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Removing service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)) client = SSHClient(storagerouter, 'root') client.dir_delete(scrub_directory) if ServiceManager.has_service(alba_proxy_service, client=client): ServiceManager.stop_service(alba_proxy_service, client=client) ServiceManager.remove_service(alba_proxy_service, client=client) if Configuration.exists(scrub_config_key): Configuration.delete(scrub_config_key) ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Removed service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)) except Exception: message = 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2} failed'.format(vpool.name, storagerouter.name, alba_proxy_service) error_messages.append(message) ScheduledTaskController._logger.exception(message)
def configure_host(self, ip): if self._is_devstack is False and self._is_openstack is False or self._cinder_installed is False or self._nova_installed is False: self._logger.warning( 'Configure host: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed' ) return # 1. Get Driver code self._logger.info('*** Configuring host with IP {0} ***'.format(ip)) self._logger.info(' Copy driver code') remote_driver = "/opt/OpenvStorage/config/templates/cinder-volume-driver/{0}/openvstorage.py".format( self._stack_version) remote_version = '0.0.0' existing_version = '0.0.0' try: from cinder.volume.drivers import openvstorage if hasattr(openvstorage, 'OVSVolumeDriver'): existing_version = getattr(openvstorage.OVSVolumeDriver, 'VERSION', '0.0.0') except ImportError: pass for line in self.client.file_read(remote_driver).splitlines(): if 'VERSION = ' in line: remote_version = line.split('VERSION = ')[-1].strip().replace( "'", "").replace('"', "") break nova_base_path = self._get_base_path('nova') cinder_base_path = self._get_base_path('cinder') if self._is_devstack is True: local_driver = '{0}/volume/drivers/openvstorage.py'.format( cinder_base_path) else: local_driver = '{0}/cinder/volume/drivers/openvstorage.py'.format( self._driver_location) if remote_version > existing_version: self._logger.debug( 'Updating existing driver using {0} from version {1} to version {2}' .format(remote_driver, existing_version, remote_version)) self.client.run('cp -f {0} {1}'.format(remote_driver, local_driver)) else: self._logger.debug('Using driver {0} version {1}'.format( local_driver, existing_version)) # 2. Configure users and groups self._logger.info(' Add users to group ovs') users = ['libvirt-qemu', 'stack' ] if self._is_devstack is True else self._openstack_users for user in users: self.client.run('usermod -a -G ovs {0}'.format(user)) # 3. Apply patches self._logger.info(' Applying patches') if self._stack_version in ('liberty', 'mitaka', 'newton'): try: import os_brick cinder_brick_initiator_file = "{0}/initiator/connector.py".format( os.path.dirname(os_brick.__file__)) except ImportError: cinder_brick_initiator_file = '' if self._is_devstack is True: nova_volume_file = '{0}/virt/libvirt/volume/volume.py'.format( nova_base_path) else: nova_volume_file = '{0}/nova/virt/libvirt/volume/volume.py'.format( self._driver_location) else: cinder_brick_initiator_file = '{0}/cinder/brick/initiator/connector.py'.format( self._driver_location) if self._is_devstack is True: nova_volume_file = '{0}/virt/libvirt/volume.py'.format( nova_base_path) else: nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format( self._driver_location) if self._is_devstack is True: nova_driver_file = '{0}/virt/libvirt/driver.py'.format( nova_base_path) else: nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format( self._driver_location) self._logger.info(' Patching file {0}'.format(nova_volume_file)) file_contents = self.client.file_read(nova_volume_file) if 'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):' not in file_contents: file_contents += ''' class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver): def __init__(self, connection): super(LibvirtFileVolumeDriver, self).__init__(connection, is_block_dev=False) def get_config(self, connection_info, disk_info): conf = super(LibvirtFileVolumeDriver, self).get_config(connection_info, disk_info) conf.source_type = 'file' conf.source_path = connection_info['data']['device_path'] return conf ''' self.client.file_write(nova_volume_file, file_contents) self._logger.info(' Patching file {0}'.format(nova_driver_file)) file_contents = self.client.file_read(nova_driver_file) if self._stack_version in ('liberty', 'mitaka'): check_line = 'local=nova.virt.libvirt.volume.volume.LibvirtVolumeDriver' new_line = 'file=nova.virt.libvirt.volume.volume.LibvirtFileVolumeDriver' else: check_line = 'local=nova.virt.libvirt.volume.LibvirtVolumeDriver' new_line = 'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver' if new_line not in file_contents: for line in file_contents.splitlines(): if check_line in line: stripped_line = line.rstrip() whitespaces = len(stripped_line) - len( stripped_line.lstrip()) new_line = "{0}'{1}',\n".format(' ' * whitespaces, new_line) fc = file_contents[:file_contents. index(line)] + new_line + file_contents[ file_contents.index(line):] self.client.file_write(nova_driver_file, "".join(fc)) break if os.path.exists(cinder_brick_initiator_file): # fix brick/upload to glance self._logger.info( ' Patching file {0}'.format(cinder_brick_initiator_file)) if self._stack_version in ('liberty', 'mitaka', 'newton'): self.client.run( """sed -i 's/elif protocol == LOCAL:/elif protocol in [LOCAL, "FILE"]:/g' {0}""" .format(cinder_brick_initiator_file)) else: self.client.run( """sed -i 's/elif protocol == "LOCAL":/elif protocol in ["LOCAL", "FILE"]:/g' {0}""" .format(cinder_brick_initiator_file)) # 4. Configure messaging driver self._logger.info(' - Configure messaging driver') nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging' cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging' with remote(ip, [RawConfigParser, open], 'root') as rem: for config_file, driver in { self._NOVA_CONF: nova_messaging_driver, self._CINDER_CONF: cinder_messaging_driver }.iteritems(): changed = False cfg = rem.RawConfigParser() cfg.read([config_file]) if cfg.has_option("DEFAULT", "notification_driver"): if cfg.get("DEFAULT", "notification_driver") != driver: changed = True cfg.set("DEFAULT", "notification_driver", driver) else: changed = True cfg.set("DEFAULT", "notification_driver", driver) if cfg.has_option("DEFAULT", "notification_topics"): notification_topics = cfg.get( "DEFAULT", "notification_topics").split(",") if "notifications" not in notification_topics: notification_topics.append("notifications") changed = True cfg.set("DEFAULT", "notification_topics", ",".join(notification_topics)) else: changed = True cfg.set("DEFAULT", "notification_topics", "notifications") if config_file == self._NOVA_CONF: for param, value in { 'notify_on_any_change': 'True', 'notify_on_state_change': 'vm_and_task_state' }.iteritems(): if not cfg.has_option("DEFAULT", param): changed = True cfg.set("DEFAULT", param, value) if changed is True: with rem.open(config_file, "w") as fp: cfg.write(fp) # 5. Enable events consumer self._logger.info(' - Enabling events consumer service') service_name = 'openstack-events-consumer' if not ServiceManager.has_service(service_name, self.client): ServiceManager.add_service(service_name, self.client) ServiceManager.enable_service(service_name, self.client) ServiceManager.start_service(service_name, self.client)