def change_service_state(client, name, state, logger=None):
"""
Starts/stops/restarts a service
:param client: SSHClient on which to connect and change service state
:param name: Name of the service
:param state: State to put the service in
:param logger: LogHandler Object
"""
action = None
status, _ = ServiceManager.get_service_status(name, client=client)
if status is False and state in ['start', 'restart']:
if logger is not None:
logger.debug(' {0:<15} - Starting service {1}'.format(client.ip, name))
ServiceManager.start_service(name, client=client)
action = 'started'
elif status is True and state == 'stop':
if logger is not None:
logger.debug(' {0:<15} - Stopping service {1}'.format(client.ip, name))
ServiceManager.stop_service(name, client=client)
action = 'stopped'
elif status is True and state == 'restart':
if logger is not None:
logger.debug(' {0:<15} - Restarting service {1}'.format(client.ip, name))
ServiceManager.restart_service(name, client=client)
action = 'restarted'
if action is None:
print ' [{0}] {1} already {2}'.format(client.ip, name, 'running' if status is True else 'halted')
else:
logger.debug(' {0:<15} - Service {1} {2}'.format(client.ip, name, action))
print ' [{0}] {1} {2}'.format(client.ip, name, action)
def start(cluster_name, client):
"""
Starts an arakoon cluster
"""
if ServiceManager.get_service_status(
'arakoon-{0}'.format(cluster_name), client=client) is False:
ServiceManager.start_service('arakoon-{0}'.format(cluster_name),
client=client)
def start_service(name, client):
"""
Start a service
:param name: Name of the service
:param client: SSHClient object
:return: None
"""
ServiceManager.start_service(name, client)
def start(cluster_name, client):
"""
Starts an etcd cluster
:param client: Client on which to start the service
:param cluster_name: The name of the cluster service to start
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True:
ServiceManager.start_service('etcd-{0}'.format(cluster_name), client=client)
def start(cluster_name, client):
"""
Starts an arakoon cluster
:param client: Client on which to start the service
:param cluster_name: The name of the cluster service to start
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('arakoon-{0}'.format(cluster_name), client=client) is False:
ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def _enable_openstack_events_consumer(self):
"""
Enable service ovs-openstack-events-consumer
"""
from ovs.extensions.services.service import ServiceManager
service_name = 'ovs-openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
def start(cluster_name, client):
"""
Starts an etcd cluster
:param client: Client on which to start the service
:param cluster_name: The name of the cluster service to start
"""
if ServiceManager.has_service('etcd-{0}'.format(cluster_name), client=client) is True and \
ServiceManager.get_service_status('etcd-{0}'.format(cluster_name), client=client) is False:
ServiceManager.start_service('etcd-{0}'.format(cluster_name),
client=client)
def _enable_openstack_events_consumer(self):
"""
Enable service ovs-openstack-events-consumer
"""
from ovs.extensions.services.service import ServiceManager
service_name = 'ovs-openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
def _roll_out_dtl_services(vpool, storagerouters):
"""
Deploy and start the DTL service on all storagerouters
:param storagerouters: StorageRouters to deploy and start a DTL service on
:return: None
"""
service_name = 'dtl_{0}'.format(vpool.name)
for sr in storagerouters.values():
client = SSHClient(sr, 'root')
ServiceManager.add_service(name=service_name, client=client)
ServiceManager.start_service(name=service_name, client=client)
def _roll_out_dtl_services(vpool, storagerouters):
"""
Deploy and start the DTL service on all storagerouters
:param storagerouters: StorageRouters to deploy and start a DTL service on
:return: None
"""
service_name = 'dtl_{0}'.format(vpool.name)
for sr in storagerouters.values():
client = SSHClient(sr, 'root')
ServiceManager.add_service(name=service_name, client=client)
ServiceManager.start_service(name=service_name, client=client)
def start(cluster_name, client):
"""
Starts an arakoon cluster
:param cluster_name: The name of the cluster service to start
:type cluster_name: str
:param client: Client on which to start the service
:type client: SSHClient
:return: None
"""
service_name = ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name)
if ServiceManager.has_service(name=service_name, client=client) is True:
ServiceManager.start_service(name=service_name, client=client)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework',
SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = []
try:
for storagerouter in StorageRouterList.get_storagerouters():
clients.append(SSHClient(storagerouter, username='******'))
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
for client in clients:
for service_name in ['watcher-framework', 'memcached']:
ServiceManager.stop_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(
service_name, client=client) is False:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError(
'Could not stop service: {0}'.format(service_name))
for client in clients:
for service_name in ['memcached', 'watcher-framework']:
ServiceManager.start_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(
service_name, client=client) is True:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError(
'Could not start service: {0}'.format(
service_name))
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
for function in functions:
function(ip=ip)
def start(cluster_name, client):
"""
Starts an arakoon cluster
:param cluster_name: The name of the cluster service to start
:type cluster_name: str
:param client: Client on which to start the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def start(cluster_name, client):
"""
Starts an arakoon cluster
:param cluster_name: The name of the cluster service to start
:type cluster_name: str
:param client: Client on which to start the service
:type client: SSHClient
:return: None
"""
if ServiceManager.has_service('arakoon-{0}'.format(cluster_name), client=client) is True:
ServiceManager.start_service('arakoon-{0}'.format(cluster_name), client=client)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = []
try:
for storagerouter in StorageRouterList.get_storagerouters():
clients.append(SSHClient(storagerouter, username='******'))
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
for client in clients:
for service_name in ['watcher-framework', 'memcached']:
ServiceManager.stop_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(service_name, client=client) is False:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError('Could not stop service: {0}'.format(service_name))
for client in clients:
for service_name in ['memcached', 'watcher-framework']:
ServiceManager.start_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(service_name, client=client) is True:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError('Could not start service: {0}'.format(service_name))
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
for function in functions:
function(ip=ip)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework',
SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
print 'Installing plugin into Open vStorage'
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = {}
masters = StorageRouterList.get_masters()
slaves = StorageRouterList.get_slaves()
try:
for sr in masters + slaves:
clients[sr] = SSHClient(sr, username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
memcached = 'memcached'
watcher = 'watcher-framework'
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Stopping watcher on {0} ({1})'.format(
sr.name, sr.ip)
ServiceManager.stop_service(watcher, clients[sr])
for sr in masters:
print '- Restarting memcached on {0} ({1})'.format(
sr.name, sr.ip)
ServiceManager.restart_service(memcached, clients[sr])
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Starting watcher on {0} ({1})'.format(
sr.name, sr.ip)
ServiceManager.start_service(watcher, clients[sr])
print '- Execute model migrations'
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
if len(functions) > 0:
print '- Execute post installation scripts'
for function in functions:
function(ip=ip)
print 'Installing plugin into Open vStorage: Completed'
def change_service_state(client, name, state, logger=None):
"""
Starts/stops/restarts a service
:param client: SSHClient on which to connect and change service state
:param name: Name of the service
:param state: State to put the service in
:param logger: LogHandler Object
"""
action = None
# Enable service before changing the state
status = ServiceManager.is_enabled(name, client=client)
if status is False:
if logger is not None:
logger.debug(' {0:<15} - Enabling service {1}'.format(
client.ip, name))
ServiceManager.enable_service(name, client=client)
status = ServiceManager.get_service_status(name, client=client)
if status is False and state in ['start', 'restart']:
if logger is not None:
logger.debug(' {0:<15} - Starting service {1}'.format(
client.ip, name))
ServiceManager.start_service(name, client=client)
action = 'started'
elif status is True and state == 'stop':
if logger is not None:
logger.debug(' {0:<15} - Stopping service {1}'.format(
client.ip, name))
ServiceManager.stop_service(name, client=client)
action = 'stopped'
elif status is True and state == 'restart':
if logger is not None:
logger.debug(' {0:<15} - Restarting service {1}'.format(
client.ip, name))
ServiceManager.restart_service(name, client=client)
action = 'restarted'
if action is None:
print ' [{0}] {1} already {2}'.format(
client.ip, name, 'running' if status is True else 'halted')
else:
logger.debug(' {0:<15} - Service {1} {2}'.format(
client.ip, name, action))
print ' [{0}] {1} {2}'.format(client.ip, name, action)
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
print 'Installing plugin into Open vStorage'
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = {}
masters = StorageRouterList.get_masters()
slaves = StorageRouterList.get_slaves()
try:
for sr in masters + slaves:
clients[sr] = SSHClient(sr, username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
memcached = 'memcached'
watcher = 'watcher-framework'
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Stopping watcher on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.stop_service(watcher, clients[sr])
for sr in masters:
print '- Restarting memcached on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.restart_service(memcached, clients[sr])
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Starting watcher on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.start_service(watcher, clients[sr])
print '- Execute model migrations'
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
if len(functions) > 0:
print '- Execute post installation scripts'
for function in functions:
function(ip=ip)
print 'Installing plugin into Open vStorage: Completed'
def execute_scrub_work(queue, vpool, scrub_info, error_messages):
"""
Executes scrub work for a given vDisk queue and vPool, based on scrub_info
:param queue: a Queue with vDisk guids that need to be scrubbed (they should only be member of a single vPool)
:type queue: Queue
:param vpool: the vPool object of the vDisks
:type vpool: VPool
:param scrub_info: A dict containing scrub information: `scrub_path` with the path where to scrub and `storage_router` with the StorageRouter
that needs to do the work
:type scrub_info: dict
:param error_messages: A list of error messages to be filled
:type error_messages: list
:return: a list of error messages
:rtype: list
"""
def _verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics('info')
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
client = None
lock_time = 5 * 60
storagerouter = scrub_info['storage_router']
scrub_directory = '{0}/scrub_work_{1}_{2}'.format(
scrub_info['scrub_path'], vpool.name, storagerouter.name)
scrub_config_key = 'ovs/vpools/{0}/proxies/scrub/scrub_config_{1}'.format(
vpool.guid, storagerouter.guid)
backend_config_key = 'ovs/vpools/{0}/proxies/scrub/backend_config_{1}'.format(
vpool.guid, storagerouter.guid)
alba_proxy_service = 'ovs-albaproxy_{0}_{1}_scrub'.format(
vpool.name, storagerouter.name)
# Deploy a proxy
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Deploying ALBA proxy {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_create(scrub_directory)
client.dir_chmod(
scrub_directory, 0777
) # Celery task executed by 'ovs' user and should be able to write in it
if ServiceManager.has_service(
name=alba_proxy_service, client=client
) is True and ServiceManager.get_service_status(
name=alba_proxy_service, client=client) is True:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Re-using existing proxy service {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
scrub_config = Configuration.get(scrub_config_key)
else:
machine_id = System.get_my_machine_id(client)
port_range = Configuration.get(
'/ovs/framework/hosts/{0}/ports|storagedriver'.format(
machine_id))
port = System.get_free_ports(selected_range=port_range,
nr=1,
client=client)[0]
# Scrub config
# {u'albamgr_cfg_url': u'arakoon://config/ovs/vpools/71e2f717-f270-4a41-bbb0-d4c8c084d43e/proxies/64759516-3471-4321-b912-fb424568fc5b/config/abm?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini',
# u'fragment_cache': [u'none'],
# u'ips': [u'127.0.0.1'],
# u'log_level': u'info',
# u'manifest_cache_size': 17179869184,
# u'port': 0,
# u'transport': u'tcp'}
# Backend config
# {u'alba_connection_host': u'10.100.193.155',
# u'alba_connection_port': 26204,
# u'alba_connection_preset': u'preset',
# u'alba_connection_timeout': 15,
# u'alba_connection_transport': u'TCP',
# u'backend_interface_retries_on_error': 5,
# u'backend_interface_retry_backoff_multiplier': 2.0,
# u'backend_interface_retry_interval_secs': 1,
# u'backend_type': u'ALBA'}
scrub_config = Configuration.get(
'ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid))
scrub_config['port'] = port
scrub_config['transport'] = 'tcp'
Configuration.set(scrub_config_key,
json.dumps(scrub_config, indent=4),
raw=True)
params = {
'VPOOL_NAME':
vpool.name,
'LOG_SINK':
LogHandler.get_sink_path('alba_proxy'),
'CONFIG_PATH':
Configuration.get_configuration_path(scrub_config_key)
}
ServiceManager.add_service(name='ovs-albaproxy',
params=params,
client=client,
target_name=alba_proxy_service)
ServiceManager.start_service(name=alba_proxy_service,
client=client)
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Deployed ALBA proxy {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
backend_config = Configuration.get(
'ovs/vpools/{0}/hosts/{1}/config'.format(
vpool.guid, vpool.storagedrivers[0].storagedriver_id
))['backend_connection_manager']
backend_config['alba_connection_host'] = '127.0.0.1'
backend_config['alba_connection_port'] = scrub_config['port']
Configuration.set(
backend_config_key,
json.dumps({"backend_connection_manager": backend_config},
indent=4),
raw=True)
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - An error occurred deploying ALBA proxy {2}'.format(
vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
if client is not None and ServiceManager.has_service(
name=alba_proxy_service, client=client) is True:
if ServiceManager.get_service_status(name=alba_proxy_service,
client=client) is True:
ServiceManager.stop_service(name=alba_proxy_service,
client=client)
ServiceManager.remove_service(name=alba_proxy_service,
client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
try:
# Empty the queue with vDisks to scrub
with remote(storagerouter.ip, [VDisk]) as rem:
while True:
vdisk = None
vdisk_guid = queue.get(False)
try:
# Check MDS master is local. Trigger MDS handover if necessary
vdisk = rem.VDisk(vdisk_guid)
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Started scrubbing at location {3}'
.format(vpool.name, storagerouter.name, vdisk.name,
scrub_directory))
configs = _verify_mds_config(current_vdisk=vdisk)
storagedriver = StorageDriverList.get_by_storagedriver_id(
vdisk.storagedriver_id)
if configs[0].get(
'ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - MDS master is not local, trigger handover'
.format(vpool.name, storagerouter.name,
vdisk.name))
MDSServiceController.ensure_safety(
VDisk(vdisk_guid)
) # Do not use a remote VDisk instance here
configs = _verify_mds_config(current_vdisk=vdisk)
if configs[0].get(
'ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.warning(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Skipping because master MDS still not local'
.format(vpool.name, storagerouter.name,
vdisk.name))
continue
# Do the actual scrubbing
with vdisk.storagedriver_client.make_locked_client(
str(vdisk.volume_id)) as locked_client:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Retrieve and apply scrub work'
.format(vpool.name, storagerouter.name,
vdisk.name))
work_units = locked_client.get_scrubbing_workunits(
)
for work_unit in work_units:
res = locked_client.scrub(
work_unit=work_unit,
scratch_dir=scrub_directory,
log_sinks=[
LogHandler.get_sink_path(
'scrubber', allow_override=True)
],
backend_config=Configuration.
get_configuration_path(backend_config_key))
locked_client.apply_scrubbing_result(
scrubbing_work_result=res)
if work_units:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - {3} work units successfully applied'
.format(vpool.name, storagerouter.name,
vdisk.name, len(work_units)))
else:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - No scrubbing required'
.format(vpool.name, storagerouter.name,
vdisk.name))
except Exception:
if vdisk is None:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk with guid {2} could not be found'.format(
vpool.name, storagerouter.name, vdisk_guid)
else:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Scrubbing failed'.format(
vpool.name, storagerouter.name, vdisk.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
except Empty: # Raised when all items have been fetched from the queue
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Queue completely processed'
.format(vpool.name, storagerouter.name))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Scrubbing failed'.format(
vpool.name, storagerouter.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
# Delete the proxy again
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_delete(scrub_directory)
if ServiceManager.has_service(alba_proxy_service,
client=client):
ServiceManager.stop_service(alba_proxy_service,
client=client)
ServiceManager.remove_service(alba_proxy_service,
client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - StorageRouter {1} - Removed service {2}'
.format(vpool.name, storagerouter.name,
alba_proxy_service))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2} failed'.format(
vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
def configure_host(self, ip):
if self._is_devstack is False and self._is_openstack is False or self._cinder_installed is False or self._nova_installed is False:
self._logger.warning('Configure host: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed')
return
# 1. Get Driver code
self._logger.info('*** Configuring host with IP {0} ***'.format(ip))
self._logger.info(' Copy driver code')
remote_driver = "/opt/OpenvStorage/config/templates/cinder-volume-driver/{0}/openvstorage.py".format(self._stack_version)
remote_version = '0.0.0'
existing_version = '0.0.0'
try:
from cinder.volume.drivers import openvstorage
if hasattr(openvstorage, 'OVSVolumeDriver'):
existing_version = getattr(openvstorage.OVSVolumeDriver, 'VERSION', '0.0.0')
except ImportError:
pass
for line in self.client.file_read(remote_driver).splitlines():
if 'VERSION = ' in line:
remote_version = line.split('VERSION = ')[-1].strip().replace("'", "").replace('"', "")
break
nova_base_path = self._get_base_path('nova')
cinder_base_path = self._get_base_path('cinder')
if self._is_devstack is True:
local_driver = '{0}/volume/drivers/openvstorage.py'.format(cinder_base_path)
else:
local_driver = '{0}/cinder/volume/drivers/openvstorage.py'.format(self._driver_location)
if remote_version > existing_version:
self._logger.debug('Updating existing driver using {0} from version {1} to version {2}'.format(remote_driver, existing_version, remote_version))
self.client.run('cp -f {0} {1}'.format(remote_driver, local_driver))
else:
self._logger.debug('Using driver {0} version {1}'.format(local_driver, existing_version))
# 2. Configure users and groups
self._logger.info(' Add users to group ovs')
users = ['libvirt-qemu', 'stack'] if self._is_devstack is True else self._openstack_users
for user in users:
self.client.run('usermod -a -G ovs {0}'.format(user))
# 3. Apply patches
self._logger.info(' Applying patches')
if self._stack_version in ('liberty', 'mitaka', 'newton'):
try:
import os_brick
cinder_brick_initiator_file = "{0}/initiator/connector.py".format(os.path.dirname(os_brick.__file__))
except ImportError:
cinder_brick_initiator_file = ''
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume/volume.py'.format(nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume/volume.py'.format(self._driver_location)
else:
cinder_brick_initiator_file = '{0}/cinder/brick/initiator/connector.py'.format(self._driver_location)
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume.py'.format(nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format(self._driver_location)
if self._is_devstack is True:
nova_driver_file = '{0}/virt/libvirt/driver.py'.format(nova_base_path)
else:
nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format(self._driver_location)
self._logger.info(' Patching file {0}'.format(nova_volume_file))
file_contents = self.client.file_read(nova_volume_file)
if 'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):' not in file_contents:
file_contents += '''
class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):
def __init__(self, connection):
super(LibvirtFileVolumeDriver,
self).__init__(connection, is_block_dev=False)
def get_config(self, connection_info, disk_info):
conf = super(LibvirtFileVolumeDriver,
self).get_config(connection_info, disk_info)
conf.source_type = 'file'
conf.source_path = connection_info['data']['device_path']
return conf
'''
self.client.file_write(nova_volume_file, file_contents)
self._logger.info(' Patching file {0}'.format(nova_driver_file))
file_contents = self.client.file_read(nova_driver_file)
if self._stack_version in ('liberty', 'mitaka'):
check_line = 'local=nova.virt.libvirt.volume.volume.LibvirtVolumeDriver'
new_line = 'file=nova.virt.libvirt.volume.volume.LibvirtFileVolumeDriver'
else:
check_line = 'local=nova.virt.libvirt.volume.LibvirtVolumeDriver'
new_line = 'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver'
if new_line not in file_contents:
for line in file_contents.splitlines():
if check_line in line:
stripped_line = line.rstrip()
whitespaces = len(stripped_line) - len(stripped_line.lstrip())
new_line = "{0}'{1}',\n".format(' ' * whitespaces, new_line)
fc = file_contents[:file_contents.index(line)] + new_line + file_contents[file_contents.index(line):]
self.client.file_write(nova_driver_file, "".join(fc))
break
if os.path.exists(cinder_brick_initiator_file):
# fix brick/upload to glance
self._logger.info(' Patching file {0}'.format(cinder_brick_initiator_file))
if self._stack_version in ('liberty', 'mitaka', 'newton'):
self.client.run("""sed -i 's/elif protocol == LOCAL:/elif protocol in [LOCAL, "FILE"]:/g' {0}""".format(cinder_brick_initiator_file))
else:
self.client.run("""sed -i 's/elif protocol == "LOCAL":/elif protocol in ["LOCAL", "FILE"]:/g' {0}""".format(cinder_brick_initiator_file))
# 4. Configure messaging driver
self._logger.info(' - Configure messaging driver')
nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
with remote(ip, [RawConfigParser, open], 'root') as rem:
for config_file, driver in {self._NOVA_CONF: nova_messaging_driver,
self._CINDER_CONF: cinder_messaging_driver}.iteritems():
changed = False
cfg = rem.RawConfigParser()
cfg.read([config_file])
if cfg.has_option("DEFAULT", "notification_driver"):
if cfg.get("DEFAULT", "notification_driver") != driver:
changed = True
cfg.set("DEFAULT", "notification_driver", driver)
else:
changed = True
cfg.set("DEFAULT", "notification_driver", driver)
if cfg.has_option("DEFAULT", "notification_topics"):
notification_topics = cfg.get("DEFAULT", "notification_topics").split(",")
if "notifications" not in notification_topics:
notification_topics.append("notifications")
changed = True
cfg.set("DEFAULT", "notification_topics", ",".join(notification_topics))
else:
changed = True
cfg.set("DEFAULT", "notification_topics", "notifications")
if config_file == self._NOVA_CONF:
for param, value in {'notify_on_any_change': 'True',
'notify_on_state_change': 'vm_and_task_state'}.iteritems():
if not cfg.has_option("DEFAULT", param):
changed = True
cfg.set("DEFAULT", param, value)
if changed is True:
with rem.open(config_file, "w") as fp:
cfg.write(fp)
# 5. Enable events consumer
self._logger.info(' - Enabling events consumer service')
service_name = 'openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
def start(cluster_name, client):
"""
Starts an arakoon cluster
"""
if ServiceManager.get_service_status("arakoon-{0}".format(cluster_name), client=client) is False:
ServiceManager.start_service("arakoon-{0}".format(cluster_name), client=client)
def execute_scrub_work(queue, vpool, scrub_info, error_messages):
"""
Executes scrub work for a given vDisk queue and vPool, based on scrub_info
:param queue: a Queue with vDisk guids that need to be scrubbed (they should only be member of a single vPool)
:type queue: Queue
:param vpool: the vPool object of the vDisks
:type vpool: VPool
:param scrub_info: A dict containing scrub information: `scrub_path` with the path where to scrub and `storage_router` with the StorageRouter
that needs to do the work
:type scrub_info: dict
:param error_messages: A list of error messages to be filled
:type error_messages: list
:return: a list of error messages
:rtype: list
"""
def _verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics('info')
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
client = None
lock_time = 5 * 60
storagerouter = scrub_info['storage_router']
scrub_directory = '{0}/scrub_work_{1}_{2}'.format(scrub_info['scrub_path'], vpool.name, storagerouter.name)
scrub_config_key = 'ovs/vpools/{0}/proxies/scrub/scrub_config_{1}'.format(vpool.guid, storagerouter.guid)
backend_config_key = 'ovs/vpools/{0}/proxies/scrub/backend_config_{1}'.format(vpool.guid, storagerouter.guid)
alba_proxy_service = 'ovs-albaproxy_{0}_{1}_scrub'.format(vpool.name, storagerouter.name)
# Deploy a proxy
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Deploying ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_create(scrub_directory)
client.dir_chmod(scrub_directory, 0777) # Celery task executed by 'ovs' user and should be able to write in it
if ServiceManager.has_service(name=alba_proxy_service, client=client) is True and ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Re-using existing proxy service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
scrub_config = Configuration.get(scrub_config_key)
else:
machine_id = System.get_my_machine_id(client)
port_range = Configuration.get('/ovs/framework/hosts/{0}/ports|storagedriver'.format(machine_id))
port = System.get_free_ports(selected_range=port_range, nr=1, client=client)[0]
# Scrub config
# {u'albamgr_cfg_url': u'arakoon://config/ovs/vpools/71e2f717-f270-4a41-bbb0-d4c8c084d43e/proxies/64759516-3471-4321-b912-fb424568fc5b/config/abm?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini',
# u'fragment_cache': [u'none'],
# u'ips': [u'127.0.0.1'],
# u'log_level': u'info',
# u'manifest_cache_size': 17179869184,
# u'port': 0,
# u'transport': u'tcp'}
# Backend config
# {u'alba_connection_host': u'10.100.193.155',
# u'alba_connection_port': 26204,
# u'alba_connection_preset': u'preset',
# u'alba_connection_timeout': 15,
# u'alba_connection_transport': u'TCP',
# u'backend_interface_retries_on_error': 5,
# u'backend_interface_retry_backoff_multiplier': 2.0,
# u'backend_interface_retry_interval_secs': 1,
# u'backend_type': u'ALBA'}
scrub_config = Configuration.get('ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid))
scrub_config['port'] = port
scrub_config['transport'] = 'tcp'
Configuration.set(scrub_config_key, json.dumps(scrub_config, indent=4), raw=True)
params = {'VPOOL_NAME': vpool.name,
'LOG_SINK': LogHandler.get_sink_path('alba_proxy'),
'CONFIG_PATH': Configuration.get_configuration_path(scrub_config_key)}
ServiceManager.add_service(name='ovs-albaproxy', params=params, client=client, target_name=alba_proxy_service)
ServiceManager.start_service(name=alba_proxy_service, client=client)
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Deployed ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
backend_config = Configuration.get('ovs/vpools/{0}/hosts/{1}/config'.format(vpool.guid, vpool.storagedrivers[0].storagedriver_id))['backend_connection_manager']
backend_config['alba_connection_host'] = '127.0.0.1'
backend_config['alba_connection_port'] = scrub_config['port']
Configuration.set(backend_config_key, json.dumps({"backend_connection_manager": backend_config}, indent=4), raw=True)
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - An error occurred deploying ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
if client is not None and ServiceManager.has_service(name=alba_proxy_service, client=client) is True:
if ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True:
ServiceManager.stop_service(name=alba_proxy_service, client=client)
ServiceManager.remove_service(name=alba_proxy_service, client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
try:
# Empty the queue with vDisks to scrub
with remote(storagerouter.ip, [VDisk]) as rem:
while True:
vdisk = None
vdisk_guid = queue.get(False)
try:
# Check MDS master is local. Trigger MDS handover if necessary
vdisk = rem.VDisk(vdisk_guid)
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Started scrubbing at location {3}'.format(vpool.name, storagerouter.name, vdisk.name, scrub_directory))
configs = _verify_mds_config(current_vdisk=vdisk)
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - MDS master is not local, trigger handover'.format(vpool.name, storagerouter.name, vdisk.name))
MDSServiceController.ensure_safety(VDisk(vdisk_guid)) # Do not use a remote VDisk instance here
configs = _verify_mds_config(current_vdisk=vdisk)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.warning('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Skipping because master MDS still not local'.format(vpool.name, storagerouter.name, vdisk.name))
continue
# Do the actual scrubbing
with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Retrieve and apply scrub work'.format(vpool.name, storagerouter.name, vdisk.name))
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
res = locked_client.scrub(work_unit=work_unit,
scratch_dir=scrub_directory,
log_sinks=[LogHandler.get_sink_path('scrubber', allow_override=True)],
backend_config=Configuration.get_configuration_path(backend_config_key))
locked_client.apply_scrubbing_result(scrubbing_work_result=res)
if work_units:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - {3} work units successfully applied'.format(vpool.name, storagerouter.name, vdisk.name, len(work_units)))
else:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - No scrubbing required'.format(vpool.name, storagerouter.name, vdisk.name))
except Exception:
if vdisk is None:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk with guid {2} could not be found'.format(vpool.name, storagerouter.name, vdisk_guid)
else:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Scrubbing failed'.format(vpool.name, storagerouter.name, vdisk.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
except Empty: # Raised when all items have been fetched from the queue
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Queue completely processed'.format(vpool.name, storagerouter.name))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Scrubbing failed'.format(vpool.name, storagerouter.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
# Delete the proxy again
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Removing service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_delete(scrub_directory)
if ServiceManager.has_service(alba_proxy_service, client=client):
ServiceManager.stop_service(alba_proxy_service, client=client)
ServiceManager.remove_service(alba_proxy_service, client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Removed service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2} failed'.format(vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
def configure_host(self, ip):
if self._is_devstack is False and self._is_openstack is False or self._cinder_installed is False or self._nova_installed is False:
self._logger.warning(
'Configure host: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed'
)
return
# 1. Get Driver code
self._logger.info('*** Configuring host with IP {0} ***'.format(ip))
self._logger.info(' Copy driver code')
remote_driver = "/opt/OpenvStorage/config/templates/cinder-volume-driver/{0}/openvstorage.py".format(
self._stack_version)
remote_version = '0.0.0'
existing_version = '0.0.0'
try:
from cinder.volume.drivers import openvstorage
if hasattr(openvstorage, 'OVSVolumeDriver'):
existing_version = getattr(openvstorage.OVSVolumeDriver,
'VERSION', '0.0.0')
except ImportError:
pass
for line in self.client.file_read(remote_driver).splitlines():
if 'VERSION = ' in line:
remote_version = line.split('VERSION = ')[-1].strip().replace(
"'", "").replace('"', "")
break
nova_base_path = self._get_base_path('nova')
cinder_base_path = self._get_base_path('cinder')
if self._is_devstack is True:
local_driver = '{0}/volume/drivers/openvstorage.py'.format(
cinder_base_path)
else:
local_driver = '{0}/cinder/volume/drivers/openvstorage.py'.format(
self._driver_location)
if remote_version > existing_version:
self._logger.debug(
'Updating existing driver using {0} from version {1} to version {2}'
.format(remote_driver, existing_version, remote_version))
self.client.run('cp -f {0} {1}'.format(remote_driver,
local_driver))
else:
self._logger.debug('Using driver {0} version {1}'.format(
local_driver, existing_version))
# 2. Configure users and groups
self._logger.info(' Add users to group ovs')
users = ['libvirt-qemu', 'stack'
] if self._is_devstack is True else self._openstack_users
for user in users:
self.client.run('usermod -a -G ovs {0}'.format(user))
# 3. Apply patches
self._logger.info(' Applying patches')
if self._stack_version in ('liberty', 'mitaka', 'newton'):
try:
import os_brick
cinder_brick_initiator_file = "{0}/initiator/connector.py".format(
os.path.dirname(os_brick.__file__))
except ImportError:
cinder_brick_initiator_file = ''
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume/volume.py'.format(
nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume/volume.py'.format(
self._driver_location)
else:
cinder_brick_initiator_file = '{0}/cinder/brick/initiator/connector.py'.format(
self._driver_location)
if self._is_devstack is True:
nova_volume_file = '{0}/virt/libvirt/volume.py'.format(
nova_base_path)
else:
nova_volume_file = '{0}/nova/virt/libvirt/volume.py'.format(
self._driver_location)
if self._is_devstack is True:
nova_driver_file = '{0}/virt/libvirt/driver.py'.format(
nova_base_path)
else:
nova_driver_file = '{0}/nova/virt/libvirt/driver.py'.format(
self._driver_location)
self._logger.info(' Patching file {0}'.format(nova_volume_file))
file_contents = self.client.file_read(nova_volume_file)
if 'class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):' not in file_contents:
file_contents += '''
class LibvirtFileVolumeDriver(LibvirtBaseVolumeDriver):
def __init__(self, connection):
super(LibvirtFileVolumeDriver,
self).__init__(connection, is_block_dev=False)
def get_config(self, connection_info, disk_info):
conf = super(LibvirtFileVolumeDriver,
self).get_config(connection_info, disk_info)
conf.source_type = 'file'
conf.source_path = connection_info['data']['device_path']
return conf
'''
self.client.file_write(nova_volume_file, file_contents)
self._logger.info(' Patching file {0}'.format(nova_driver_file))
file_contents = self.client.file_read(nova_driver_file)
if self._stack_version in ('liberty', 'mitaka'):
check_line = 'local=nova.virt.libvirt.volume.volume.LibvirtVolumeDriver'
new_line = 'file=nova.virt.libvirt.volume.volume.LibvirtFileVolumeDriver'
else:
check_line = 'local=nova.virt.libvirt.volume.LibvirtVolumeDriver'
new_line = 'file=nova.virt.libvirt.volume.LibvirtFileVolumeDriver'
if new_line not in file_contents:
for line in file_contents.splitlines():
if check_line in line:
stripped_line = line.rstrip()
whitespaces = len(stripped_line) - len(
stripped_line.lstrip())
new_line = "{0}'{1}',\n".format(' ' * whitespaces,
new_line)
fc = file_contents[:file_contents.
index(line)] + new_line + file_contents[
file_contents.index(line):]
self.client.file_write(nova_driver_file, "".join(fc))
break
if os.path.exists(cinder_brick_initiator_file):
# fix brick/upload to glance
self._logger.info(
' Patching file {0}'.format(cinder_brick_initiator_file))
if self._stack_version in ('liberty', 'mitaka', 'newton'):
self.client.run(
"""sed -i 's/elif protocol == LOCAL:/elif protocol in [LOCAL, "FILE"]:/g' {0}"""
.format(cinder_brick_initiator_file))
else:
self.client.run(
"""sed -i 's/elif protocol == "LOCAL":/elif protocol in ["LOCAL", "FILE"]:/g' {0}"""
.format(cinder_brick_initiator_file))
# 4. Configure messaging driver
self._logger.info(' - Configure messaging driver')
nova_messaging_driver = 'nova.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
cinder_messaging_driver = 'cinder.openstack.common.notifier.rpc_notifier' if self._stack_version == 'juno' else 'messaging'
with remote(ip, [RawConfigParser, open], 'root') as rem:
for config_file, driver in {
self._NOVA_CONF: nova_messaging_driver,
self._CINDER_CONF: cinder_messaging_driver
}.iteritems():
changed = False
cfg = rem.RawConfigParser()
cfg.read([config_file])
if cfg.has_option("DEFAULT", "notification_driver"):
if cfg.get("DEFAULT", "notification_driver") != driver:
changed = True
cfg.set("DEFAULT", "notification_driver", driver)
else:
changed = True
cfg.set("DEFAULT", "notification_driver", driver)
if cfg.has_option("DEFAULT", "notification_topics"):
notification_topics = cfg.get(
"DEFAULT", "notification_topics").split(",")
if "notifications" not in notification_topics:
notification_topics.append("notifications")
changed = True
cfg.set("DEFAULT", "notification_topics",
",".join(notification_topics))
else:
changed = True
cfg.set("DEFAULT", "notification_topics", "notifications")
if config_file == self._NOVA_CONF:
for param, value in {
'notify_on_any_change': 'True',
'notify_on_state_change': 'vm_and_task_state'
}.iteritems():
if not cfg.has_option("DEFAULT", param):
changed = True
cfg.set("DEFAULT", param, value)
if changed is True:
with rem.open(config_file, "w") as fp:
cfg.write(fp)
# 5. Enable events consumer
self._logger.info(' - Enabling events consumer service')
service_name = 'openstack-events-consumer'
if not ServiceManager.has_service(service_name, self.client):
ServiceManager.add_service(service_name, self.client)
ServiceManager.enable_service(service_name, self.client)
ServiceManager.start_service(service_name, self.client)
