def restart_framework_and_memcache_services(clients, logger, offline_node_ips=None):
"""
Restart framework and Memcached services
:param clients: Clients on which to restart these services
:type clients: dict
:param logger: Logger object used for logging
:type logger: ovs.log.log_handler.LogHandler
:param offline_node_ips: IP addresses of offline nodes in the cluster
:type offline_node_ips: list
:return: None
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
master_ips = [sr.ip for sr in StorageRouterList.get_masters()]
slave_ips = [sr.ip for sr in StorageRouterList.get_slaves()]
if offline_node_ips is None:
offline_node_ips = []
memcached = 'memcached'
watcher = 'watcher-framework'
support_agent = 'support-agent'
for ip in master_ips + slave_ips:
if ip not in offline_node_ips:
if ServiceManager.has_service(watcher, clients[ip]):
Toolbox.change_service_state(clients[ip], watcher, 'stop', logger)
for ip in master_ips:
if ip not in offline_node_ips:
Toolbox.change_service_state(clients[ip], memcached, 'restart', logger)
for ip in master_ips + slave_ips:
if ip not in offline_node_ips:
if ServiceManager.has_service(watcher, clients[ip]):
Toolbox.change_service_state(clients[ip], watcher, 'start', logger)
if ServiceManager.has_service(support_agent, clients[ip]):
Toolbox.change_service_state(clients[ip], support_agent, 'restart', logger)
VolatileFactory.store = None
def __init__(self, cinder_client):
self.client = SSHClient('127.0.0.1', username='******')
self.cinder_client = cinder_client
self._NOVA_CONF = '/etc/nova/nova.conf'
self._CINDER_CONF = '/etc/cinder/cinder.conf'
self._is_openstack = ServiceManager.has_service(OSManager.get_openstack_cinder_service_name(), self.client)
self._nova_installed = self.client.file_exists(self._NOVA_CONF)
self._cinder_installed = self.client.file_exists(self._CINDER_CONF)
self._driver_location = OSManager.get_openstack_package_base_path()
self._openstack_users = OSManager.get_openstack_users()
self._devstack_driver = '/opt/stack/cinder/cinder/volume/drivers/openvstorage.py'
try:
self._is_devstack = 'stack' in str(self.client.run('ps aux | grep SCREEN | grep stack | grep -v grep || true'))
except SystemExit: # ssh client raises system exit 1
self._is_devstack = False
except Exception:
self._is_devstack = False
try:
from cinder import version
version_string = version.version_string()
if version_string.startswith('2015.2') or version_string.startswith('7.0'):
self._stack_version = 'liberty'
elif version_string.startswith('2015.1'):
self._stack_version = 'kilo'
elif version_string.startswith('2014.2'):
self._stack_version = 'juno'
else:
raise ValueError('Unsupported cinder version: {0}'.format(version_string))
except Exception as ex:
raise ValueError('Cannot determine cinder version: {0}'.format(ex))
def add_services(client, node_type, logger):
"""
Add the services required by the OVS cluster
:param client: Client on which to add the services
:type client: ovs.extensions.generic.sshclient.SSHClient
:param node_type: Type of node ('master' or 'extra')
:type node_type: str
:param logger: Logger object used for logging
:type logger: ovs.log.log_handler.LogHandler
:return: None
"""
Toolbox.log(logger=logger, messages='Adding services')
services = {}
worker_queue = System.get_my_machine_id(client=client)
if node_type == 'master':
worker_queue += ',ovs_masters'
services.update({'memcached': {'MEMCACHE_NODE_IP': client.ip, 'WORKER_QUEUE': worker_queue},
'rabbitmq-server': {'MEMCACHE_NODE_IP': client.ip, 'WORKER_QUEUE': worker_queue},
'scheduled-tasks': {},
'webapp-api': {},
'volumerouter-consumer': {}})
services.update({'workers': {'WORKER_QUEUE': worker_queue},
'watcher-framework': {}})
for service_name, params in services.iteritems():
if not ServiceManager.has_service(service_name, client):
Toolbox.log(logger=logger, messages='Adding service {0}'.format(service_name))
ServiceManager.add_service(name=service_name, params=params, client=client)
def stop(cluster_name, client):
"""
Stops an arakoon service
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('arakoon-{0}'.format(cluster_name), client=client)
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:param master_ip: IP of the master node
:param offline_node_ips: IPs of nodes which are offline
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(
cluster_ip,
username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name('Arakoon')
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv':
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
print '* Shrink StorageDriver cluster'
ArakoonInstaller.shrink_cluster(cluster_ip, 'voldrv',
offline_node_ips)
if client is not None and ServiceManager.has_service(
current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name,
client=client)
ServiceManager.remove_service(current_service.name,
client=client)
ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver()
def remove_services(client, node_type, logger):
"""
Remove all services managed by OVS
:param client: Client on which to remove the services
:type client: ovs.extensions.generic.sshclient.SSHClient
:param node_type: Type of node, can be 'master' or 'extra'
:type node_type: str
:param logger: Logger object used for logging
:type logger: ovs.log.log_handler.LogHandler
:return: None
"""
Toolbox.log(logger=logger, messages="Removing services")
stop_only = ["rabbitmq-server", "memcached"]
services = ["workers", "support-agent", "watcher-framework"]
if node_type == "master":
services += ["scheduled-tasks", "webapp-api", "volumerouter-consumer"]
if Toolbox.is_service_internally_managed(service="rabbitmq") is True:
services.append("rabbitmq-server")
if Toolbox.is_service_internally_managed(service="memcached") is True:
services.append("memcached")
for service in services:
if ServiceManager.has_service(service, client=client):
Toolbox.log(
logger=logger,
messages="{0} service {1}".format("Removing" if service not in stop_only else "Stopping", service),
)
ServiceManager.stop_service(service, client=client)
if service not in stop_only:
ServiceManager.remove_service(service, client=client)
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:param master_ip: IP of the master node
:param offline_node_ips: IPs of nodes which are offline
"""
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(cluster_ip, username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name('Arakoon')
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv':
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
print '* Shrink StorageDriver cluster'
ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv', offline_node_ips)
if client is not None and ServiceManager.has_service(current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name, client=client)
ServiceManager.remove_service(current_service.name, client=client)
ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(offline_node_ips)
def _setup_proxy(initial_cluster, slave_client, cluster_name, force=False):
base_name = 'ovs-etcd-proxy'
target_name = 'ovs-etcd-{0}'.format(cluster_name)
if force is False and ServiceManager.has_service(target_name, slave_client) and \
ServiceManager.get_service_status(target_name, slave_client) is True:
logger.info('Service {0} already configured and running'.format(target_name))
return
EtcdInstaller.stop(cluster_name, slave_client)
data_dir = EtcdInstaller.DATA_DIR.format(EtcdInstaller.DB_DIR, cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(EtcdInstaller.DB_DIR, cluster_name)
abs_paths = [data_dir, wal_dir]
slave_client.dir_delete(abs_paths)
slave_client.dir_create(data_dir)
slave_client.dir_chmod(data_dir, 0755, recursive=True)
slave_client.dir_chown(data_dir, 'ovs', 'ovs', recursive=True)
ServiceManager.add_service(base_name, slave_client,
params={'CLUSTER': cluster_name,
'DATA_DIR': data_dir,
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1'),
'INITIAL_CLUSTER': initial_cluster},
target_name=target_name)
EtcdInstaller.start(cluster_name, slave_client)
EtcdInstaller.wait_for_cluster(cluster_name, slave_client)
def on_demote(cluster_ip, master_ip):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:param master_ip: IP of the master node
"""
client = SSHClient(cluster_ip, username='******')
servicetype = ServiceTypeList.get_by_name('Arakoon')
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv':
if service.storagerouter.ip == cluster_ip:
current_service = service
else:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
print '* Shrink StorageDriver cluster'
ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv')
if ServiceManager.has_service(current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name, client=client)
ServiceManager.remove_service(current_service.name, client=client)
ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
current_service.delete()
for storagerouter in StorageRouterList.get_storagerouters():
ArakoonInstaller.deploy_to_slave(master_ip, storagerouter.ip, 'voldrv')
StorageDriverController._configure_arakoon_to_volumedriver()
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework',
SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
print 'Installing plugin into Open vStorage'
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = {}
masters = StorageRouterList.get_masters()
slaves = StorageRouterList.get_slaves()
try:
for sr in masters + slaves:
clients[sr] = SSHClient(sr, username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
memcached = 'memcached'
watcher = 'watcher-framework'
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Stopping watcher on {0} ({1})'.format(
sr.name, sr.ip)
ServiceManager.stop_service(watcher, clients[sr])
for sr in masters:
print '- Restarting memcached on {0} ({1})'.format(
sr.name, sr.ip)
ServiceManager.restart_service(memcached, clients[sr])
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Starting watcher on {0} ({1})'.format(
sr.name, sr.ip)
ServiceManager.start_service(watcher, clients[sr])
print '- Execute model migrations'
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
if len(functions) > 0:
print '- Execute post installation scripts'
for function in functions:
function(ip=ip)
print 'Installing plugin into Open vStorage: Completed'
def has_service(name, client):
"""
Validate if the node has the service configured
:param name: Name of the service
:param client: SSHClient object
:return: True if service is configured
"""
return ServiceManager.has_service(name, client)
def remove(cluster_name, client):
"""
Removes an etcd service
:param client: Client on which to remove the service
:param cluster_name: The name of the cluster service to remove
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True:
ServiceManager.remove_service('etcd-{0}'.format(cluster_name), client=client)
def start(cluster_name, client):
"""
Starts an arakoon cluster
:param client: Client on which to start the service
:param cluster_name: The name of the cluster service to start
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def stop(cluster_name, client):
"""
Stops an arakoon service
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('arakoon-{0}'.format(cluster_name),
client=client)
def remove(cluster_name, client):
"""
Removes an arakoon service
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name),
client=client) is True:
ServiceManager.remove_service('arakoon-{0}'.format(cluster_name),
client=client)
def stop(cluster_name, client):
"""
Stops an etcd service
:param client: Client on which to stop the service
:param cluster_name: The name of the cluster service to stop
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('etcd-{0}'.format(cluster_name), client=client)
def stop(cluster_name, client):
"""
Stops an arakoon service
"""
if (
ServiceManager.has_service("arakoon-{0}".format(cluster_name), client=client) is True
and ServiceManager.get_service_status("arakoon-{0}".format(cluster_name), client=client) is True
):
ServiceManager.stop_service("arakoon-{0}".format(cluster_name), client=client)
def start(cluster_name, client):
"""
Starts an etcd cluster
:param client: Client on which to start the service
:param cluster_name: The name of the cluster service to start
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('etcd-{0}'.format(cluster_name), client=client) is False:
ServiceManager.start_service('etcd-{0}'.format(cluster_name), client=client)
def is_running(cluster_name, client):
"""
Checks if arakoon service is running
:param client: Client on which to stop the service
:param cluster_name: The name of the cluster service to stop
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client):
return ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client)
return False
def is_devstack_installed():
"""
Check if OpenStack or DevStack is installed
:return: True if installed
"""
client = SSHClient('127.0.0.1', username='******')
is_openstack = ServiceManager.has_service(OSManager.get_openstack_cinder_service_name(), client)
is_devstack = 'stack' in str(client.run('ps aux | grep SCREEN | grep stack | grep -v grep || true'), allow_insecure=True)
return is_openstack or is_devstack
def stop(cluster_name, client):
"""
Stops an arakoon service
:param client: Client on which to stop the service
:param cluster_name: The name of the cluster service to stop
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('arakoon-{0}'.format(cluster_name), client=client)
def stop(cluster_name, client):
"""
Stops an etcd service
:param client: Client on which to stop the service
:param cluster_name: The name of the cluster service to stop
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('etcd-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('etcd-{0}'.format(cluster_name),
client=client)
def _enable_openstack_events_consumer(self):
"""
Enable service ovs-openstack-events-consumer
"""
from ovs.extensions.services.service import ServiceManager
service_name = 'ovs-openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
def start(cluster_name, client):
"""
Starts an arakoon cluster
:param client: Client on which to start the service
:param cluster_name: The name of the cluster service to start
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client) is False:
ServiceManager.start_service('arakoon-{0}'.format(cluster_name),
client=client)
def remove(cluster_name, client):
"""
Removes an etcd service
:param client: Client on which to remove the service
:param cluster_name: The name of the cluster service to remove
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name),
client=client) is True:
ServiceManager.remove_service('etcd-{0}'.format(cluster_name),
client=client)
def _enable_openstack_events_consumer(self):
"""
Enable service ovs-openstack-events-consumer
"""
from ovs.extensions.services.service import ServiceManager
service_name = 'ovs-openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
print 'Installing plugin into Open vStorage'
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = {}
masters = StorageRouterList.get_masters()
slaves = StorageRouterList.get_slaves()
try:
for sr in masters + slaves:
clients[sr] = SSHClient(sr, username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
memcached = 'memcached'
watcher = 'watcher-framework'
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Stopping watcher on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.stop_service(watcher, clients[sr])
for sr in masters:
print '- Restarting memcached on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.restart_service(memcached, clients[sr])
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Starting watcher on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.start_service(watcher, clients[sr])
print '- Execute model migrations'
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
if len(functions) > 0:
print '- Execute post installation scripts'
for function in functions:
function(ip=ip)
print 'Installing plugin into Open vStorage: Completed'
def start(cluster_name, client):
"""
Starts an etcd cluster
:param cluster_name: The name of the cluster service to start
:type cluster_name: str
:param client: Client on which to start the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True:
ServiceManager.start_service('etcd-{0}'.format(cluster_name), client=client)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework',
SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = []
try:
for storagerouter in StorageRouterList.get_storagerouters():
clients.append(SSHClient(storagerouter, username='******'))
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
for client in clients:
for service_name in ['watcher-framework', 'memcached']:
ServiceManager.stop_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(
service_name, client=client) is False:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError(
'Could not stop service: {0}'.format(service_name))
for client in clients:
for service_name in ['memcached', 'watcher-framework']:
ServiceManager.start_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(
service_name, client=client) is True:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError(
'Could not start service: {0}'.format(
service_name))
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
for function in functions:
function(ip=ip)
def stop(cluster_name, client):
"""
Stops an arakoon service
:param cluster_name: The name of the cluster service to stop
:type cluster_name: str
:param client: Client on which to stop the service
:type client: SSHClient
:return: None
"""
service_name = ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name)
if ServiceManager.has_service(name=service_name, client=client) is True:
ServiceManager.stop_service(name=service_name, client=client)
def is_running(cluster_name, client):
"""
Checks if arakoon service is running
:param cluster_name: The name of the cluster service to check
:type cluster_name: str
:param client: Client on which to check the service
:type client: SSHClient
:return: None
"""
service_name = ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name)
if ServiceManager.has_service(name=service_name, client=client):
return ServiceManager.get_service_status(name=service_name, client=client)[0]
return False
def stop(cluster_name, client):
"""
Stops an arakoon service
:param cluster_name: The name of the cluster service to stop
:type cluster_name: str
:param client: Client on which to stop the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('arakoon-{0}'.format(cluster_name), client=client)
def remove(cluster_name, client):
"""
Removes an arakoon service
:param cluster_name: The name of the cluster service to remove
:type cluster_name: str
:param client: Client on which to remove the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.remove_service('arakoon-{0}'.format(cluster_name), client=client)
def _restart_openstack_services(self):
"""
Restart services on openstack
"""
services = OSManager.get_openstack_services()
for service_name in services:
if ServiceManager.has_service(service_name, self.client):
try:
ServiceManager.restart_service(service_name, self.client)
except SystemExit as sex:
logger.debug('Failed to restart service {0}. {1}'.format(service_name, sex))
time.sleep(3)
return self._is_cinder_running()
def start(cluster_name, client):
"""
Starts an etcd cluster
:param cluster_name: The name of the cluster service to start
:type cluster_name: str
:param client: Client on which to start the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True:
ServiceManager.start_service('etcd-{0}'.format(cluster_name), client=client)
def stop(cluster_name, client):
"""
Stops an arakoon service
:param cluster_name: The name of the cluster service to stop
:type cluster_name: str
:param client: Client on which to stop the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.stop_service('arakoon-{0}'.format(cluster_name), client=client)
def remove(cluster_name, client):
"""
Removes an arakoon service
:param cluster_name: The name of the cluster service to remove
:type cluster_name: str
:param client: Client on which to remove the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.remove_service('arakoon-{0}'.format(cluster_name), client=client)
def create_cluster(cluster_name, ip, server_port=DEFAULT_SERVER_PORT, client_port=DEFAULT_CLIENT_PORT):
"""
Creates a cluster
:param cluster_name: Name of the cluster
:type cluster_name: str
:param ip: IP address of the first node of the new cluster
:type ip: str
:param server_port: Port to be used by server
:type server_port: int
:param client_port: Port to be used by client
:type client_port: int
:return: None
"""
EtcdInstaller._logger.debug('Creating cluster "{0}" on {1}'.format(cluster_name, ip))
client = SSHClient(ip, username='******')
target_name = 'ovs-etcd-{0}'.format(cluster_name)
if ServiceManager.has_service(target_name, client) and ServiceManager.get_service_status(target_name, client) is True:
EtcdInstaller._logger.info('Service {0} already configured and running'.format(target_name))
return
node_name = System.get_my_machine_id(client)
data_dir = EtcdInstaller.DATA_DIR.format(cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(cluster_name)
abs_paths = [data_dir, wal_dir]
client.dir_delete(abs_paths)
client.dir_create(abs_paths)
client.dir_chmod(abs_paths, 0755, recursive=True)
client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
base_name = 'ovs-etcd'
ServiceManager.add_service(base_name, client,
params={'CLUSTER': cluster_name,
'NODE_ID': node_name,
'DATA_DIR': data_dir,
'WAL_DIR': wal_dir,
'SERVER_URL': EtcdInstaller.SERVER_URL.format(ip, server_port),
'CLIENT_URL': EtcdInstaller.CLIENT_URL.format(ip, client_port),
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1', client_port),
'INITIAL_CLUSTER': '{0}={1}'.format(node_name, EtcdInstaller.SERVER_URL.format(ip, server_port)),
'INITIAL_STATE': 'new',
'INITIAL_PEERS': '-initial-advertise-peer-urls {0}'.format(EtcdInstaller.SERVER_URL.format(ip, server_port))},
target_name=target_name)
EtcdInstaller.start(cluster_name, client)
EtcdInstaller.wait_for_cluster(cluster_name, client, client_port=client_port)
EtcdInstaller._logger.debug('Creating cluster "{0}" on {1} completed'.format(cluster_name, ip))
def create_cluster(cluster_name, ip, server_port=DEFAULT_SERVER_PORT, client_port=DEFAULT_CLIENT_PORT):
"""
Creates a cluster
:param cluster_name: Name of the cluster
:type cluster_name: str
:param ip: IP address of the first node of the new cluster
:type ip: str
:param server_port: Port to be used by server
:type server_port: int
:param client_port: Port to be used by client
:type client_port: int
:return: None
"""
EtcdInstaller._logger.debug('Creating cluster "{0}" on {1}'.format(cluster_name, ip))
client = SSHClient(ip, username='******')
target_name = 'ovs-etcd-{0}'.format(cluster_name)
if ServiceManager.has_service(target_name, client) and ServiceManager.get_service_status(target_name, client) is True:
EtcdInstaller._logger.info('Service {0} already configured and running'.format(target_name))
return
node_name = System.get_my_machine_id(client)
data_dir = EtcdInstaller.DATA_DIR.format(cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(cluster_name)
abs_paths = [data_dir, wal_dir]
client.dir_delete(abs_paths)
client.dir_create(abs_paths)
client.dir_chmod(abs_paths, 0755, recursive=True)
client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
base_name = 'ovs-etcd'
ServiceManager.add_service(base_name, client,
params={'CLUSTER': cluster_name,
'NODE_ID': node_name,
'DATA_DIR': data_dir,
'WAL_DIR': wal_dir,
'SERVER_URL': EtcdInstaller.SERVER_URL.format(ip, server_port),
'CLIENT_URL': EtcdInstaller.CLIENT_URL.format(ip, client_port),
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1', client_port),
'INITIAL_CLUSTER': '{0}={1}'.format(node_name, EtcdInstaller.SERVER_URL.format(ip, server_port)),
'INITIAL_STATE': 'new',
'INITIAL_PEERS': '-initial-advertise-peer-urls {0}'.format(EtcdInstaller.SERVER_URL.format(ip, server_port))},
target_name=target_name)
EtcdInstaller.start(cluster_name, client)
EtcdInstaller.wait_for_cluster(cluster_name, client, client_port=client_port)
EtcdInstaller._logger.debug('Creating cluster "{0}" on {1} completed'.format(cluster_name, ip))
def is_running(cluster_name, client):
"""
Checks if arakoon service is running
:param cluster_name: The name of the cluster service to check
:type cluster_name: str
:param client: Client on which to check the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client):
return ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client)[0]
return False
def _restart_openstack_services(self):
"""
Restart services on openstack
"""
services = OSManager.get_openstack_services()
for service_name in services:
if ServiceManager.has_service(service_name, self.client):
try:
ServiceManager.restart_service(service_name, self.client)
except SystemExit as sex:
logger.debug('Failed to restart service {0}. {1}'.format(
service_name, sex))
time.sleep(3)
return self._is_cinder_running()
def is_running(cluster_name, client):
"""
Checks if arakoon service is running
:param cluster_name: The name of the cluster service to check
:type cluster_name: str
:param client: Client on which to check the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client):
return ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client)
return False
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:type cluster_ip: str
:param master_ip: IP of the master node
:type master_ip: str
:param offline_node_ips: IPs of nodes which are offline
:type offline_node_ips: list
:return: None
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(
cluster_ip,
username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
StorageDriverController._logger.debug(
'* Shrink StorageDriver cluster')
cluster_name = str(
EtcdConfiguration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
cluster_name=cluster_name,
offline_nodes=offline_node_ips)
if client is not None and ServiceManager.has_service(
current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name,
client=client)
ServiceManager.remove_service(current_service.name,
client=client)
ArakoonInstaller.restart_cluster_remove(cluster_name,
remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
def remove(cluster_name, client, delay_unregistration=False):
"""
Removes an arakoon service
:param cluster_name: The name of the cluster service to remove
:type cluster_name: str
:param client: Client on which to remove the service
:type client: SSHClient
:param delay_unregistration: Un-register the service right away or not
:type delay_unregistration: bool
:return: None
"""
service_name = ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name)
if ServiceManager.has_service(name=service_name, client=client) is True:
ServiceManager.remove_service(name=service_name, client=client, delay_unregistration=delay_unregistration)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = []
try:
for storagerouter in StorageRouterList.get_storagerouters():
clients.append(SSHClient(storagerouter, username='******'))
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
for client in clients:
for service_name in ['watcher-framework', 'memcached']:
ServiceManager.stop_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(service_name, client=client) is False:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError('Could not stop service: {0}'.format(service_name))
for client in clients:
for service_name in ['memcached', 'watcher-framework']:
ServiceManager.start_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(service_name, client=client) is True:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError('Could not start service: {0}'.format(service_name))
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
for function in functions:
function(ip=ip)
def check_rabbitmq_and_enable_ha_mode(client, logger):
"""
Verify RabbitMQ is running properly and enable HA mode
:param client: Client on which to check RabbitMQ
:type client: ovs.extensions.generic.sshclient.SSHClient
:param logger: Logger object used for logging
:type logger: ovs.log.log_handler.LogHandler
:return: None
"""
if not ServiceManager.has_service('rabbitmq-server', client):
raise RuntimeError('Service rabbitmq-server has not been added on node {0}'.format(client.ip))
rabbitmq_running, same_process = ServiceManager.is_rabbitmq_running(client=client)
if rabbitmq_running is False or same_process is False:
Toolbox.change_service_state(client, 'rabbitmq-server', 'restart', logger)
time.sleep(5)
client.run(['rabbitmqctl', 'set_policy', 'ha-all', '^(volumerouter|ovs_.*)$', '{"ha-mode":"all"}'])
def on_remove(cluster_ip, complete_removal):
"""
Handles the StorageDriver removal part of a node
:param cluster_ip: IP of the node which is being removed from the cluster
:type cluster_ip: str
:param complete_removal: Unused for StorageDriver, used for AlbaController
:type complete_removal: bool
:return: None
"""
_ = complete_removal
service_name = 'watcher-volumedriver'
try:
client = SSHClient(endpoint=cluster_ip, username='******')
if ServiceManager.has_service(name=service_name, client=client):
ServiceManager.stop_service(name=service_name, client=client)
ServiceManager.remove_service(name=service_name, client=client)
except UnableToConnectException:
pass
def __init__(self, cinder_client):
self._logger = LogHandler.get('extensions', name='openstack_mgmt')
self.client = SSHClient('127.0.0.1', username='******')
self.cinder_client = cinder_client
self._NOVA_CONF = '/etc/nova/nova.conf'
self._CINDER_CONF = '/etc/cinder/cinder.conf'
self._is_openstack = ServiceManager.has_service(
OSManager.get_openstack_cinder_service_name(), self.client)
self._nova_installed = self.client.file_exists(self._NOVA_CONF)
self._cinder_installed = self.client.file_exists(self._CINDER_CONF)
self._driver_location = OSManager.get_openstack_package_base_path()
self._openstack_users = OSManager.get_openstack_users()
self._devstack_driver = '/opt/stack/cinder/cinder/volume/drivers/openvstorage.py'
try:
self._is_devstack = 'stack' in str(
self.client.run(
'ps aux | grep SCREEN | grep stack | grep -v grep || true')
)
except SystemExit: # ssh client raises system exit 1
self._is_devstack = False
except Exception:
self._is_devstack = False
try:
from cinder import version
version_string = version.version_string()
if version_string.startswith('9.0'):
self._stack_version = 'newton'
elif version_string.startswith('8.0'):
self._stack_version = 'mitaka'
elif version_string.startswith(
'2015.2') or version_string.startswith('7.0'):
self._stack_version = 'liberty'
elif version_string.startswith('2015.1'):
self._stack_version = 'kilo'
elif version_string.startswith('2014.2'):
self._stack_version = 'juno'
else:
raise ValueError(
'Unsupported cinder version: {0}'.format(version_string))
except Exception as ex:
raise ValueError('Cannot determine cinder version: {0}'.format(ex))
def _setup_proxy(initial_cluster,
slave_client,
cluster_name,
force=False,
client_port=DEFAULT_CLIENT_PORT):
base_name = 'ovs-etcd-proxy'
target_name = 'ovs-etcd-{0}'.format(cluster_name)
if force is False and ServiceManager.has_service(
target_name,
slave_client) and ServiceManager.get_service_status(
target_name, slave_client)[0] is True:
EtcdInstaller._logger.info(
'Service {0} already configured and running'.format(
target_name))
return
EtcdInstaller.stop(cluster_name, slave_client)
data_dir = EtcdInstaller.DATA_DIR.format(cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(cluster_name)
abs_paths = [data_dir, wal_dir]
slave_client.dir_delete(abs_paths)
slave_client.dir_create(data_dir)
slave_client.dir_chmod(data_dir, 0755, recursive=True)
slave_client.dir_chown(data_dir, 'ovs', 'ovs', recursive=True)
ServiceManager.add_service(base_name,
slave_client,
params={
'CLUSTER':
cluster_name,
'DATA_DIR':
data_dir,
'LOCAL_CLIENT_URL':
EtcdInstaller.CLIENT_URL.format(
'127.0.0.1', client_port),
'INITIAL_CLUSTER':
initial_cluster
},
target_name=target_name)
EtcdInstaller.start(cluster_name, slave_client)
EtcdInstaller.wait_for_cluster(cluster_name,
slave_client,
client_port=client_port)
def _change_services_state(services, ssh_clients, action):
"""
Stop/start services on SSH clients
If action is start, we ignore errors and try to start other services on other nodes
"""
if action == 'start':
services.reverse(
) # Start services again in reverse order of stopping
for service_name in services:
for ssh_client in ssh_clients:
description = 'stopping' if action == 'stop' else 'starting' if action == 'start' else 'restarting'
try:
if ServiceManager.has_service(service_name,
client=ssh_client):
UpdateController._log_message(
'{0} service {1}'.format(description.capitalize(),
service_name),
ssh_client.ip)
Toolbox.change_service_state(
client=ssh_client,
name=service_name,
state=action,
logger=UpdateController._logger)
UpdateController._log_message(
'{0} service {1}'.format(
'Stopped' if action == 'stop' else 'Started'
if action == 'start' else 'Restarted',
service_name), ssh_client.ip)
except Exception as exc:
UpdateController._log_message(
'Something went wrong {0} service {1}: {2}'.format(
description, service_name, exc),
ssh_client.ip,
severity='warning')
if action == 'stop':
return False
return True
def execute_scrub_work(queue, vpool, scrub_info, error_messages):
"""
Executes scrub work for a given vDisk queue and vPool, based on scrub_info
:param queue: a Queue with vDisk guids that need to be scrubbed (they should only be member of a single vPool)
:type queue: Queue
:param vpool: the vPool object of the vDisks
:type vpool: VPool
:param scrub_info: A dict containing scrub information: `scrub_path` with the path where to scrub and `storage_router` with the StorageRouter
that needs to do the work
:type scrub_info: dict
:param error_messages: A list of error messages to be filled
:type error_messages: list
:return: a list of error messages
:rtype: list
"""
def _verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics('info')
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
client = None
lock_time = 5 * 60
storagerouter = scrub_info['storage_router']
scrub_directory = '{0}/scrub_work_{1}_{2}'.format(
scrub_info['scrub_path'], vpool.name, storagerouter.name)
scrub_config_key = 'ovs/vpools/{0}/proxies/scrub/scrub_config_{1}'.format(
vpool.guid, storagerouter.guid)
backend_config_key = 'ovs/vpools/{0}/proxies/scrub/backend_config_{1}'.format(
vpool.guid, storagerouter.guid)
alba_proxy_service = 'ovs-albaproxy_{0}_{1}_scrub'.format(
vpool.name, storagerouter.name)
# Deploy a proxy
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Deploying ALBA proxy {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_create(scrub_directory)
client.dir_chmod(
scrub_directory, 0777
) # Celery task executed by 'ovs' user and should be able to write in it
if ServiceManager.has_service(
name=alba_proxy_service, client=client
) is True and ServiceManager.get_service_status(
name=alba_proxy_service, client=client) is True:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Re-using existing proxy service {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
scrub_config = Configuration.get(scrub_config_key)
else:
machine_id = System.get_my_machine_id(client)
port_range = Configuration.get(
'/ovs/framework/hosts/{0}/ports|storagedriver'.format(
machine_id))
port = System.get_free_ports(selected_range=port_range,
nr=1,
client=client)[0]
# Scrub config
# {u'albamgr_cfg_url': u'arakoon://config/ovs/vpools/71e2f717-f270-4a41-bbb0-d4c8c084d43e/proxies/64759516-3471-4321-b912-fb424568fc5b/config/abm?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini',
# u'fragment_cache': [u'none'],
# u'ips': [u'127.0.0.1'],
# u'log_level': u'info',
# u'manifest_cache_size': 17179869184,
# u'port': 0,
# u'transport': u'tcp'}
# Backend config
# {u'alba_connection_host': u'10.100.193.155',
# u'alba_connection_port': 26204,
# u'alba_connection_preset': u'preset',
# u'alba_connection_timeout': 15,
# u'alba_connection_transport': u'TCP',
# u'backend_interface_retries_on_error': 5,
# u'backend_interface_retry_backoff_multiplier': 2.0,
# u'backend_interface_retry_interval_secs': 1,
# u'backend_type': u'ALBA'}
scrub_config = Configuration.get(
'ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid))
scrub_config['port'] = port
scrub_config['transport'] = 'tcp'
Configuration.set(scrub_config_key,
json.dumps(scrub_config, indent=4),
raw=True)
params = {
'VPOOL_NAME':
vpool.name,
'LOG_SINK':
LogHandler.get_sink_path('alba_proxy'),
'CONFIG_PATH':
Configuration.get_configuration_path(scrub_config_key)
}
ServiceManager.add_service(name='ovs-albaproxy',
params=params,
client=client,
target_name=alba_proxy_service)
ServiceManager.start_service(name=alba_proxy_service,
client=client)
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Deployed ALBA proxy {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
backend_config = Configuration.get(
'ovs/vpools/{0}/hosts/{1}/config'.format(
vpool.guid, vpool.storagedrivers[0].storagedriver_id
))['backend_connection_manager']
backend_config['alba_connection_host'] = '127.0.0.1'
backend_config['alba_connection_port'] = scrub_config['port']
Configuration.set(
backend_config_key,
json.dumps({"backend_connection_manager": backend_config},
indent=4),
raw=True)
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - An error occurred deploying ALBA proxy {2}'.format(
vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
if client is not None and ServiceManager.has_service(
name=alba_proxy_service, client=client) is True:
if ServiceManager.get_service_status(name=alba_proxy_service,
client=client) is True:
ServiceManager.stop_service(name=alba_proxy_service,
client=client)
ServiceManager.remove_service(name=alba_proxy_service,
client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
try:
# Empty the queue with vDisks to scrub
with remote(storagerouter.ip, [VDisk]) as rem:
while True:
vdisk = None
vdisk_guid = queue.get(False)
try:
# Check MDS master is local. Trigger MDS handover if necessary
vdisk = rem.VDisk(vdisk_guid)
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Started scrubbing at location {3}'
.format(vpool.name, storagerouter.name, vdisk.name,
scrub_directory))
configs = _verify_mds_config(current_vdisk=vdisk)
storagedriver = StorageDriverList.get_by_storagedriver_id(
vdisk.storagedriver_id)
if configs[0].get(
'ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - MDS master is not local, trigger handover'
.format(vpool.name, storagerouter.name,
vdisk.name))
MDSServiceController.ensure_safety(
VDisk(vdisk_guid)
) # Do not use a remote VDisk instance here
configs = _verify_mds_config(current_vdisk=vdisk)
if configs[0].get(
'ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.warning(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Skipping because master MDS still not local'
.format(vpool.name, storagerouter.name,
vdisk.name))
continue
# Do the actual scrubbing
with vdisk.storagedriver_client.make_locked_client(
str(vdisk.volume_id)) as locked_client:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Retrieve and apply scrub work'
.format(vpool.name, storagerouter.name,
vdisk.name))
work_units = locked_client.get_scrubbing_workunits(
)
for work_unit in work_units:
res = locked_client.scrub(
work_unit=work_unit,
scratch_dir=scrub_directory,
log_sinks=[
LogHandler.get_sink_path(
'scrubber', allow_override=True)
],
backend_config=Configuration.
get_configuration_path(backend_config_key))
locked_client.apply_scrubbing_result(
scrubbing_work_result=res)
if work_units:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - {3} work units successfully applied'
.format(vpool.name, storagerouter.name,
vdisk.name, len(work_units)))
else:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - No scrubbing required'
.format(vpool.name, storagerouter.name,
vdisk.name))
except Exception:
if vdisk is None:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk with guid {2} could not be found'.format(
vpool.name, storagerouter.name, vdisk_guid)
else:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Scrubbing failed'.format(
vpool.name, storagerouter.name, vdisk.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
except Empty: # Raised when all items have been fetched from the queue
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Queue completely processed'
.format(vpool.name, storagerouter.name))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Scrubbing failed'.format(
vpool.name, storagerouter.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
# Delete the proxy again
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_delete(scrub_directory)
if ServiceManager.has_service(alba_proxy_service,
client=client):
ServiceManager.stop_service(alba_proxy_service,
client=client)
ServiceManager.remove_service(alba_proxy_service,
client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Removed service {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2} failed'.format(
vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
def configure_host(self, ip):
if self._is_devstack is False and self._is_openstack is False or self._cinder_installed is False or self._nova_installed is False:
self._logger.warning(
'Configure host: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed'
)
return
# 1. Get Driver code
self._logger.info('*** Configuring host with IP {0} ***'.format(ip))
self._logger.info(' Copy driver code')
remote_driver = "/opt/OpenvStorage/config/templates/cinder-volume-driver/{0}/openvstorage.py".format(
self._stack_version)
remote_version = '0.0.0'
existing_version = '0.0.0'
try:
from cinder.volume.drivers import openvstorage
if hasattr(openvstorage, 'OVSVolumeDriver'):
existing_version = getattr(openvstorage.OVSVolumeDriver,
'VERSION', '0.0.0')
except ImportError:
pass
for line in self.client.file_read(remote_driver).splitlines():
if 'VERSION = ' in line:
remote_version = line.split('VERSION = ')[-1].strip().replace(
"'", "").replace('"', "")
break
nova_base_path = self._get_base_path('nova')
cinder_base_path = self._get_base_path('cinder')
if self._is_devstack is True:
local_driver = '{0}/volume/drivers/openvstorage.py'.format(
cinder_base_path)
else:
local_driver = '{0}/cinder/volume/drivers/openvstorage.py'.format(
self._driver_location)
if remote_version > existing_version:
self._logger.debug(
'Updating existing driver using {0} from version {1} to version {2}'
.format(remote_driver, existing_version, remote_version))
self.client.run('cp -f {0} {1}'.format(remote_driver,
local_driver))
else:
self._logger.debug('Using driver {0} version {1}'.format(
local_driver, existing_version))
# 2. Configure users and groups
self._logger.info(' Add users to group ovs')
users = ['libvirt-qemu', 'stack'
] if self._is_devstack is True else self._openstack_users
for user in users:
self.client.run('usermod -a -G ovs {0}'.format(user))
# 3. Apply patches
self._logger.info(' Applying patches')
if self._stack_version in ('liberty', 'mitaka', 'newton'):
try:
import os_brick
cinder_brick_initiator_file = "{0}/initiator/connector.py".format(
os.path.dirname(os_brick.__file__))
except ImportError:
cinder_brick_initiator_file = ''
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume/volume.py'.format(
nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume/volume.py'.format(
self._driver_location)
else:
cinder_brick_initiator_file = '{0}/cinder/brick/initiator/connector.py'.format(
self._driver_location)
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume.py'.format(
nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format(
self._driver_location)
if self._is_devstack is True:
nova_driver_file = '{0}/virt/libvirt/driver.py'.format(
nova_base_path)
else:
nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format(
self._driver_location)
self._logger.info(' Patching file {0}'.format(nova_volume_file))
file_contents = self.client.file_read(nova_volume_file)
if 'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):' not in file_contents:
file_contents += '''
class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):
def __init__(self, connection):
super(LibvirtFileVolumeDriver,
self).__init__(connection, is_block_dev=False)
def get_config(self, connection_info, disk_info):
conf = super(LibvirtFileVolumeDriver,
self).get_config(connection_info, disk_info)
conf.source_type = 'file'
conf.source_path = connection_info['data']['device_path']
return conf
'''
self.client.file_write(nova_volume_file, file_contents)
self._logger.info(' Patching file {0}'.format(nova_driver_file))
file_contents = self.client.file_read(nova_driver_file)
if self._stack_version in ('liberty', 'mitaka'):
check_line = 'local=nova.virt.libvirt.volume.volume.LibvirtVolumeDriver'
new_line = 'file=nova.virt.libvirt.volume.volume.LibvirtFileVolumeDriver'
else:
check_line = 'local=nova.virt.libvirt.volume.LibvirtVolumeDriver'
new_line = 'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver'
if new_line not in file_contents:
for line in file_contents.splitlines():
if check_line in line:
stripped_line = line.rstrip()
whitespaces = len(stripped_line) - len(
stripped_line.lstrip())
new_line = "{0}'{1}',\n".format(' ' * whitespaces,
new_line)
fc = file_contents[:file_contents.
index(line)] + new_line + file_contents[
file_contents.index(line):]
self.client.file_write(nova_driver_file, "".join(fc))
break
if os.path.exists(cinder_brick_initiator_file):
# fix brick/upload to glance
self._logger.info(
' Patching file {0}'.format(cinder_brick_initiator_file))
if self._stack_version in ('liberty', 'mitaka', 'newton'):
self.client.run(
"""sed -i 's/elif protocol == LOCAL:/elif protocol in [LOCAL, "FILE"]:/g' {0}"""
.format(cinder_brick_initiator_file))
else:
self.client.run(
"""sed -i 's/elif protocol == "LOCAL":/elif protocol in ["LOCAL", "FILE"]:/g' {0}"""
.format(cinder_brick_initiator_file))
# 4. Configure messaging driver
self._logger.info(' - Configure messaging driver')
nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
with remote(ip, [RawConfigParser, open], 'root') as rem:
for config_file, driver in {
self._NOVA_CONF: nova_messaging_driver,
self._CINDER_CONF: cinder_messaging_driver
}.iteritems():
changed = False
cfg = rem.RawConfigParser()
cfg.read([config_file])
if cfg.has_option("DEFAULT", "notification_driver"):
if cfg.get("DEFAULT", "notification_driver") != driver:
changed = True
cfg.set("DEFAULT", "notification_driver", driver)
else:
changed = True
cfg.set("DEFAULT", "notification_driver", driver)
if cfg.has_option("DEFAULT", "notification_topics"):
notification_topics = cfg.get(
"DEFAULT", "notification_topics").split(",")
if "notifications" not in notification_topics:
notification_topics.append("notifications")
changed = True
cfg.set("DEFAULT", "notification_topics",
",".join(notification_topics))
else:
changed = True
cfg.set("DEFAULT", "notification_topics", "notifications")
if config_file == self._NOVA_CONF:
for param, value in {
'notify_on_any_change': 'True',
'notify_on_state_change': 'vm_and_task_state'
}.iteritems():
if not cfg.has_option("DEFAULT", param):
changed = True
cfg.set("DEFAULT", param, value)
if changed is True:
with rem.open(config_file, "w") as fp:
cfg.write(fp)
# 5. Enable events consumer
self._logger.info(' - Enabling events consumer service')
service_name = 'openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
def unconfigure_host(self, ip):
if self._is_devstack is False and self._is_openstack is False or self._cinder_installed is False or self._nova_installed is False:
self._logger.warning(
'Unconfigure host: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed'
)
return
# 1. Remove driver code
self._logger.info('*** Unconfiguring host with IP {0} ***'.format(ip))
self._logger.info(' Removing driver code')
if self._is_devstack is True:
self.client.file_delete(self._devstack_driver)
else:
self.client.file_delete(
'{0}/cinder/volume/drivers/openvstorage.py'.format(
self._driver_location))
# 2. Removing users from group
self._logger.info(' Removing users from group ovs')
for user in ['libvirt-qemu', 'stack'
] if self._is_devstack is True else self._openstack_users:
self.client.run('deluser {0} ovs'.format(user))
# 3. Revert patches
self._logger.info(' Reverting patches')
nova_base_path = self._get_base_path('nova')
cinder_base_path = self._get_base_path('cinder')
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume.py'.format(
nova_base_path)
nova_driver_file = '{0}/virt/libvirt/driver.py'.format(
nova_base_path)
cinder_brick_initiator_file = '{0}/brick/initiator/connector.py'.format(
cinder_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format(
self._driver_location)
nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format(
self._driver_location)
cinder_brick_initiator_file = '{0}/cinder/brick/initiator/connector.py'.format(
self._driver_location)
self._logger.info(
' Reverting patched file: {0}'.format(nova_volume_file))
new_contents = []
skip_class = False
for line in self.client.file_read(nova_volume_file).splitlines():
if line.startswith(
'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):'):
skip_class = True
continue
if line.startswith('class'):
skip_class = False
if skip_class is False:
new_contents.append(line)
self.client.file_write(nova_volume_file, "".join(new_contents))
self._logger.info(
' Reverting patched file: {0}'.format(nova_driver_file))
new_contents = []
for line in self.client.file_read(nova_driver_file).splitlines():
stripped_line = line.strip()
if stripped_line.startswith(
"'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver'"):
continue
new_contents.append(line)
self.client.file_write(nova_driver_file, "".join(new_contents))
if os.path.exists(cinder_brick_initiator_file):
self._logger.info(' Reverting patched file: {0}'.format(
cinder_brick_initiator_file))
self.client.run(
"""sed -i 's/elif protocol in ["LOCAL", "FILE"]:/elif protocol == "LOCAL":/g' {0}"""
.format(cinder_brick_initiator_file))
# 4. Unconfigure messaging driver
self._logger.info(' Unconfiguring messaging driver')
nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
with remote(ip, [RawConfigParser, open], 'root') as rem:
for config_file, driver in {
self._NOVA_CONF: nova_messaging_driver,
self._CINDER_CONF: cinder_messaging_driver
}.iteritems():
cfg = rem.RawConfigParser()
cfg.read([config_file])
if cfg.has_option("DEFAULT", "notification_driver"):
cfg.remove_option("DEFAULT", "notification_driver")
if cfg.has_option("DEFAULT", "notification_topics"):
notification_topics = cfg.get(
"DEFAULT", "notification_topics").split(",")
if "notifications" in notification_topics:
notification_topics.remove("notifications")
cfg.set("DEFAULT", "notification_topics",
",".join(notification_topics))
if config_file == self._NOVA_CONF:
for param, value in {
'notify_on_any_change': 'True',
'notify_on_state_change': 'vm_and_task_state'
}.iteritems():
if cfg.has_option("DEFAULT", param):
cfg.remove_option("DEFAULT", param)
with rem.open(config_file, "w") as fp:
cfg.write(fp)
# 5. Disable events consumer
self._logger.info(' Disabling events consumer')
service_name = 'ovs-openstack-events-consumer'
if ServiceManager.has_service(service_name, self.client):
ServiceManager.stop_service(service_name, self.client)
ServiceManager.disable_service(service_name, self.client)
ServiceManager.remove_service(service_name, self.client)
def _restart_processes(self):
"""
Restart the cinder process that uses the OVS volume driver
- also restarts nova api and compute services
"""
def stop_screen_process(process_name):
out = self.client.run(
'''su stack -c 'screen -S {0} -p {1} -Q select 1>/dev/null; echo $?' '''
.format(screen_name, process_name))
process_screen_exists = out == '0'
if process_screen_exists:
self.client.run(
'''su stack -c 'screen -S {0} -p {1} -X stuff \n' '''.
format(screen_name, process_name))
self.client.run(
'''su stack -c 'screen -S {0} -p {1} -X kill' '''.format(
screen_name, process_name))
return process_screen_exists
def start_screen_process(process_name, commands):
logfile = '{0}/{1}.log.{2}'.format(
logdir, process_name,
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d-%H%M%S'))
self._logger.debug(
self.client.run(
'''su stack -c 'touch {0}' '''.format(logfile)))
self._logger.debug(
self.client.run(
'''su stack -c 'screen -S {0} -X screen -t {1}' '''.format(
screen_name, process_name)))
self._logger.debug(
self.client.run(
'''su stack -c 'screen -S {0} -p {1} -X logfile {2}' '''.
format(screen_name, process_name, logfile)))
self._logger.debug(
self.client.run(
'''su stack -c 'screen -S {0} -p {1} -X log on' '''.format(
screen_name, process_name)))
time.sleep(1)
self._logger.debug(
self.client.run('rm {0}/{1}.log || true'.format(
logdir, process_name)))
self._logger.debug(
self.client.run('ln -sf {0} {1}/{2}.log'.format(
logfile, logdir, process_name)))
for command in commands:
cmd = '''su stack -c 'screen -S {0} -p {1} -X stuff "{2}\012"' '''.format(
screen_name, process_name, command)
self._logger.debug(cmd)
self._logger.debug(self.client.run(cmd))
logdir = '/opt/stack/logs'
screen_name = 'stack'
if self._is_devstack is True:
try:
c_vol_screen_exists = stop_screen_process('c-vol')
n_cpu_screen_exists = stop_screen_process('n-cpu')
n_api_screen_exists = stop_screen_process('n-api')
c_api_screen_exists = stop_screen_process('c-api')
self.client.run('''su stack -c 'mkdir -p /opt/stack/logs' ''')
if c_vol_screen_exists:
start_screen_process('c-vol', [
"export PYTHONPATH=\"${PYTHONPATH}:/opt/OpenvStorage\" ",
"newgrp ovs", "newgrp stack", "umask 0002",
"/usr/local/bin/cinder-volume --config-file /etc/cinder/cinder.conf & echo \$! >/opt/stack/status/stack/c-vol.pid; fg || echo c-vol failed to start | tee \"/opt/stack/status/stack/c-vol.failure\" "
])
time.sleep(3)
if n_cpu_screen_exists:
start_screen_process('n-cpu', [
"newgrp ovs", "newgrp stack",
"sg libvirtd /usr/local/bin/nova-compute --config-file /etc/nova/nova.conf & echo $! >/opt/stack/status/stack/n-cpu.pid; fg || echo n-cpu failed to start | tee \"/opt/stack/status/stack/n-cpu.failure\" "
])
time.sleep(3)
if n_api_screen_exists:
start_screen_process('n-api', [
"export PYTHONPATH=\"${PYTHONPATH}:/opt/OpenvStorage\" ",
"/usr/local/bin/nova-api & echo $! >/opt/stack/status/stack/n-api.pid; fg || echo n-api failed to start | tee \"/opt/stack/status/stack/n-api.failure\" "
])
time.sleep(3)
if c_api_screen_exists:
start_screen_process('c-api', [
"/usr/local/bin/cinder-api --config-file /etc/cinder/cinder.conf & echo $! >/opt/stack/status/stack/c-api.pid; fg || echo c-api failed to start | tee \"/opt/stack/status/stack/c-api.failure\" "
])
time.sleep(3)
except SystemExit as se: # failed command or non-zero exit codes raise SystemExit
raise RuntimeError(str(se))
else:
for service_name in OSManager.get_openstack_services():
if ServiceManager.has_service(service_name, self.client):
try:
ServiceManager.restart_service(service_name,
self.client)
except SystemExit as sex:
self._logger.debug(
'Failed to restart service {0}. {1}'.format(
service_name, sex))
time.sleep(3)
def is_host_configured(self, ip):
if (
self._is_devstack is False and self._is_openstack is False
) or self._cinder_installed is False or self._nova_installed is False:
self._logger.warning(
'Host configured: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed'
)
return False
# 1. Check driver code
if self._is_devstack is True:
if not self.client.file_exists(filename=self._devstack_driver):
self._logger.info(' File "{0}" does not exist'.format(
self._devstack_driver))
return False
else:
if not self.client.file_exists(
filename='{0}/cinder/volume/drivers/openvstorage.py'.
format(self._driver_location)):
self._logger.info(
' File "{0}/cinder/volume/drivers/openvstorage.py" does not exist'
.format(self._driver_location))
return False
# 2. Check configured users
ovs_id = self.client.run('id -u ovs')
if not ovs_id:
self._logger.info('Failed to determine the OVS user group ID')
return False
users = ['libvirt-qemu', 'stack'
] if self._is_devstack is True else self._openstack_users
for user in users:
if '{0}(ovs)'.format(ovs_id) not in self.client.run(
'id -a {0}'.format(user)):
self._logger.info(
'User "{0}" is not part of the OVS user group')
return False
# 3. Check patches
nova_base_path = self._get_base_path('nova')
cinder_base_path = self._get_base_path('cinder')
if self._stack_version in ('liberty', 'mitaka', 'newton'):
try:
import os_brick
cinder_brick_initiator_file = "{0}/initiator/connector.py".format(
os.path.dirname(os_brick.__file__))
except ImportError:
cinder_brick_initiator_file = ''
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume/volume.py'.format(
nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume/volume.py'.format(
self._driver_location)
else:
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume.py'.format(
nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format(
self._driver_location)
cinder_brick_initiator_file = '{0}/brick/initiator/connector.py'.format(
cinder_base_path)
if self._is_devstack is True:
nova_driver_file = '{0}/virt/libvirt/driver.py'.format(
nova_base_path)
else:
nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format(
self._driver_location)
file_contents = self.client.file_read(nova_volume_file)
if 'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):' not in file_contents:
self._logger.info('File "{0}" is not configured properly'.format(
nova_volume_file))
return False
if self._stack_version in ('liberty', 'mitaka'):
check_line = 'file=nova.virt.libvirt.volume.volume.LibvirtFileVolumeDriver'
else:
check_line = 'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver'
file_contents = self.client.file_read(nova_driver_file)
if check_line not in file_contents:
self._logger.info('File "{0}" is not configured properly'.format(
nova_driver_file))
return False
if os.path.exists(cinder_brick_initiator_file):
file_contents = self.client.file_read(cinder_brick_initiator_file)
if self._stack_version in ('liberty', 'mitaka', 'newton'):
if 'elif protocol in [LOCAL, "FILE"]:' not in file_contents:
self._logger.info(
'File "{0}" is not configured properly'.format(
cinder_brick_initiator_file))
return False
else:
if 'elif protocol in ["LOCAL", "FILE"]:' not in file_contents:
self._logger.info(
'File "{0}" is not configured properly'.format(
cinder_brick_initiator_file))
return False
# 4. Check messaging driver configuration
nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
host_configured = True
with remote(ip, [RawConfigParser], 'root') as rem:
for config_file, driver in {
self._NOVA_CONF: nova_messaging_driver,
self._CINDER_CONF: cinder_messaging_driver
}.iteritems():
cfg = rem.RawConfigParser()
cfg.read([config_file])
host_configured &= cfg.get("DEFAULT",
"notification_driver") == driver
host_configured &= "notifications" in cfg.get(
"DEFAULT", "notification_topics")
if config_file == self._NOVA_CONF:
host_configured &= cfg.get(
"DEFAULT", "notify_on_any_change") == "True"
host_configured &= cfg.get(
"DEFAULT",
"notify_on_state_change") == "vm_and_task_state"
if host_configured is False:
self._logger.info(
'Nova and/or Cinder configuration files are not configured properly'
)
return host_configured
# 5. Check events consumer service
service_name = 'ovs-openstack-events-consumer'
if not (ServiceManager.has_service(service_name, self.client)
and ServiceManager.get_service_status(service_name,
self.client) is True):
self._logger.info(
'Service "{0}" is not configured properly'.format(
service_name))
return False
return True
def _is_openstack(self):
cinder_service = OSManager.get_openstack_cinder_service_name()
return ServiceManager.has_service(cinder_service, self.client)
