def restart_framework_and_memcache_services(clients, logger, offline_node_ips=None):
"""
Restart framework and Memcached services
:param clients: Clients on which to restart these services
:type clients: dict
:param logger: Logger object used for logging
:type logger: ovs.log.log_handler.LogHandler
:param offline_node_ips: IP addresses of offline nodes in the cluster
:type offline_node_ips: list
:return: None
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
master_ips = [sr.ip for sr in StorageRouterList.get_masters()]
slave_ips = [sr.ip for sr in StorageRouterList.get_slaves()]
if offline_node_ips is None:
offline_node_ips = []
memcached = 'memcached'
watcher = 'watcher-framework'
support_agent = 'support-agent'
for ip in master_ips + slave_ips:
if ip not in offline_node_ips:
if ServiceManager.has_service(watcher, clients[ip]):
Toolbox.change_service_state(clients[ip], watcher, 'stop', logger)
for ip in master_ips:
if ip not in offline_node_ips:
Toolbox.change_service_state(clients[ip], memcached, 'restart', logger)
for ip in master_ips + slave_ips:
if ip not in offline_node_ips:
if ServiceManager.has_service(watcher, clients[ip]):
Toolbox.change_service_state(clients[ip], watcher, 'start', logger)
if ServiceManager.has_service(support_agent, clients[ip]):
Toolbox.change_service_state(clients[ip], support_agent, 'restart', logger)
VolatileFactory.store = None
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_result: Port information
:type arakoon_result: Dictionary
:return: The newly created and added service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip,
storagerouter.ip,
cluster_name,
partition.folder
)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
def get_available_arakoon_storagerouters(cls, ssh_clients=None):
# type: (Optional[Dict[StorageRouter, SSHClient]]) -> Dict[StorageRouter, DiskPartition]
"""
Retrieves all Storagerouters which are suitable to deploy Arakoons on
:return: Set of all Storagerouters that are suitable
:rtype: Dict[StorageRouter, DiskPartition]
"""
ssh_clients = ssh_clients or {}
available_storagerouters = {}
masters = StorageRouterList.get_masters()
for storagerouter in masters:
try:
partition = AlbaArakoonInstaller.get_db_partition(
storagerouter)
try:
if ssh_clients:
client = ssh_clients.get(storagerouter)
else:
client = SSHClient(storagerouter)
if client:
available_storagerouters[storagerouter] = partition
except UnableToConnectException:
cls._logger.warning(
'Storage Router with IP {0} is not reachable'.format(
storagerouter.ip))
except ValueError:
pass # Ignore storagerouters without DB parition
return available_storagerouters
def install_plugins():
"""
(Re)load plugins
"""
manager = ServiceFactory.get_manager()
if manager.has_service('ovs-watcher-framework',
SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
print 'Installing plugin into Open vStorage'
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = {}
masters = StorageRouterList.get_masters()
slaves = StorageRouterList.get_slaves()
try:
for sr in masters + slaves:
clients[sr] = SSHClient(sr, username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
memcached = 'memcached'
watcher = 'watcher-framework'
for sr in masters + slaves:
if manager.has_service(watcher, clients[sr]):
print '- Stopping watcher on {0} ({1})'.format(
sr.name, sr.ip)
manager.stop_service(watcher, clients[sr])
for sr in masters:
print '- Restarting memcached on {0} ({1})'.format(
sr.name, sr.ip)
manager.restart_service(memcached, clients[sr])
for sr in masters + slaves:
if manager.has_service(watcher, clients[sr]):
print '- Starting watcher on {0} ({1})'.format(
sr.name, sr.ip)
manager.start_service(watcher, clients[sr])
print '- Execute model migrations'
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
if len(functions) > 0:
print '- Execute post installation scripts'
for fct in functions:
fct(ip=ip)
print 'Installing plugin into Open vStorage: Completed'
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
print 'Installing plugin into Open vStorage'
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = {}
masters = StorageRouterList.get_masters()
slaves = StorageRouterList.get_slaves()
try:
for sr in masters + slaves:
clients[sr] = SSHClient(sr, username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
memcached = 'memcached'
watcher = 'watcher-framework'
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Stopping watcher on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.stop_service(watcher, clients[sr])
for sr in masters:
print '- Restarting memcached on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.restart_service(memcached, clients[sr])
for sr in masters + slaves:
if ServiceManager.has_service(watcher, clients[sr]):
print '- Starting watcher on {0} ({1})'.format(sr.name, sr.ip)
ServiceManager.start_service(watcher, clients[sr])
print '- Execute model migrations'
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
if len(functions) > 0:
print '- Execute post installation scripts'
for function in functions:
function(ip=ip)
print 'Installing plugin into Open vStorage: Completed'
def restart_framework_and_memcache_services(clients,
logger,
offline_node_ips=None):
"""
Restart framework and Memcached services
:param clients: Clients on which to restart these services
:type clients: dict
:param logger: Logger object used for logging
:type logger: ovs.extensions.generic.logger.Logger
:param offline_node_ips: IP addresses of offline nodes in the cluster
:type offline_node_ips: list
:return: None
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
service_manager = ServiceFactory.get_manager()
master_ips = [sr.ip for sr in StorageRouterList.get_masters()]
slave_ips = [sr.ip for sr in StorageRouterList.get_slaves()]
if offline_node_ips is None:
offline_node_ips = []
memcached = 'memcached'
watcher = 'watcher-framework'
support_agent = 'support-agent'
for ip in master_ips + slave_ips:
if ip not in offline_node_ips:
if service_manager.has_service(watcher, clients[ip]):
ServiceFactory.change_service_state(
clients[ip], watcher, 'stop', logger)
for ip in master_ips:
if ip not in offline_node_ips:
ServiceFactory.change_service_state(clients[ip], memcached,
'restart', logger)
for ip in master_ips + slave_ips:
if ip not in offline_node_ips:
if service_manager.has_service(watcher, clients[ip]):
ServiceFactory.change_service_state(
clients[ip], watcher, 'start', logger)
if service_manager.has_service(support_agent, clients[ip]):
ServiceFactory.change_service_state(
clients[ip], support_agent, 'restart', logger)
VolatileFactory.store = None
def change_scheduled_task(task_name, state, disabled=False, cron=None):
if not Configuration.exists(celery_key):
Configuration.set(celery_key, {})
jobs = Configuration.get(celery_key)
if state == 'present':
if disabled:
jobs[task_name] = None
output = 'task {0}: disabled'.format(task_name)
else:
jobs[task_name] = cron
settings = ''
for key, value in cron.iteritems():
settings += "{0}: {1} ".format(key, value)
output = 'task {0}: cron settings {1}'.format(task_name, settings)
else:
jobs.pop(task_name, None)
output = 'task {0}: removed, default settings will be applied.'.format(task_name)
Configuration.set(celery_key, jobs)
service_name = 'scheduled-tasks'
service_manager = ServiceFactory.get_manager()
for storagerouter in StorageRouterList.get_masters():
client = SSHClient(storagerouter, username='******')
service_manager.restart_service(service_name, client=client)
return output
def configure_support(support_info):
"""
Configures support on all StorageRouters
:param support_info: Information about which components should be configured
{'stats_monkey': True, # Enable/disable the stats monkey scheduled task
'support_agent': True, # Responsible for enabling the ovs-support-agent service, which collects heart beat data
'remote_access': False, # Cannot be True when support agent is False. Is responsible for opening an OpenVPN tunnel to allow for remote access
'stats_monkey_config': {}} # Dict with information on how to configure the stats monkey (Only required when enabling the stats monkey
:type support_info: dict
:return: None
:rtype: NoneType
"""
ExtensionsToolbox.verify_required_params(actual_params=support_info,
required_params={
'stats_monkey':
(bool, None, False),
'remote_access':
(bool, None, False),
'support_agent':
(bool, None, False),
'stats_monkey_config':
(dict, None, False)
})
# All settings are optional, so if nothing is specified, no need to change anything
if len(support_info) == 0:
StorageRouterController._logger.warning(
'Configure support called without any specific settings. Doing nothing'
)
return
# Collect information
support_agent_key = '/ovs/framework/support|support_agent'
support_agent_new = support_info.get('support_agent')
support_agent_old = Configuration.get(key=support_agent_key)
support_agent_change = support_agent_new is not None and support_agent_old != support_agent_new
remote_access_key = '/ovs/framework/support|remote_access'
remote_access_new = support_info.get('remote_access')
remote_access_old = Configuration.get(key=remote_access_key)
remote_access_change = remote_access_new is not None and remote_access_old != remote_access_new
stats_monkey_celery_key = '/ovs/framework/scheduling/celery'
stats_monkey_config_key = '/ovs/framework/monitoring/stats_monkey'
stats_monkey_new_config = support_info.get('stats_monkey_config')
stats_monkey_old_config = Configuration.get(
key=stats_monkey_config_key, default={})
stats_monkey_celery_config = Configuration.get(
key=stats_monkey_celery_key, default={})
stats_monkey_new = support_info.get('stats_monkey')
stats_monkey_old = stats_monkey_celery_config.get(
'ovs.stats_monkey.run_all'
) is not None or stats_monkey_celery_config.get(
'alba.stats_monkey.run_all') is not None
stats_monkey_change = stats_monkey_new is not None and (
stats_monkey_old != stats_monkey_new
or stats_monkey_new_config != stats_monkey_old_config)
# Make sure support agent is enabled when trying to enable remote access
if remote_access_new is True:
if support_agent_new is False or (support_agent_new is None
and support_agent_old is False):
raise RuntimeError(
'Remote access cannot be enabled without the heart beat enabled'
)
# Collect root_client information
root_clients = {}
for storagerouter in StorageRouterList.get_storagerouters():
try:
root_clients[storagerouter] = SSHClient(endpoint=storagerouter,
username='******')
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
if stats_monkey_new is True:
ExtensionsToolbox.verify_required_params(
actual_params=stats_monkey_new_config,
required_params={
'host': (str, ExtensionsToolbox.regex_ip),
'port': (int, {
'min': 1,
'max': 65535
}),
'database': (str, None),
'interval': (int, {
'min': 1,
'max': 86400
}),
'transport': (str, ['influxdb', 'redis', 'graphite']),
'environment': (str, None)
})
if stats_monkey_new_config['transport'] in ['influxdb', 'reddis']:
ExtensionsToolbox.verify_required_params(
actual_params=stats_monkey_new_config,
required_params={'password': (str, None)})
if stats_monkey_new_config['transport'] == 'influxdb':
ExtensionsToolbox.verify_required_params(
actual_params=stats_monkey_new_config,
required_params={'username': (str, None)})
# Configure remote access
if remote_access_change is True:
Configuration.set(key=remote_access_key, value=remote_access_new)
cid = Configuration.get('/ovs/framework/cluster_id').replace(
r"'", r"'\''")
for storagerouter, root_client in root_clients.iteritems():
if remote_access_new is False:
StorageRouterController._logger.info(
'Un-configuring remote access on StorageRouter {0}'.
format(root_client.ip))
nid = storagerouter.machine_id.replace(r"'", r"'\''")
service_name = 'openvpn@ovs_{0}-{1}'.format(cid, nid)
if StorageRouterController._service_manager.has_service(
name=service_name, client=root_client):
StorageRouterController._service_manager.stop_service(
name=service_name, client=root_client)
root_client.file_delete(filenames=['/etc/openvpn/ovs_*'])
# Configure support agent
if support_agent_change is True:
service_name = 'support-agent'
Configuration.set(key=support_agent_key, value=support_agent_new)
for root_client in root_clients.itervalues():
if support_agent_new is True:
StorageRouterController._logger.info(
'Configuring support agent on StorageRouter {0}'.
format(root_client.ip))
if StorageRouterController._service_manager.has_service(
name=service_name, client=root_client) is False:
StorageRouterController._service_manager.add_service(
name=service_name, client=root_client)
StorageRouterController._service_manager.restart_service(
name=service_name, client=root_client)
else:
StorageRouterController._logger.info(
'Un-configuring support agent on StorageRouter {0}'.
format(root_client.ip))
if StorageRouterController._service_manager.has_service(
name=service_name, client=root_client):
StorageRouterController._service_manager.stop_service(
name=service_name, client=root_client)
StorageRouterController._service_manager.remove_service(
name=service_name, client=root_client)
# Configure stats monkey
if stats_monkey_change is True:
# 2 keys matter here:
# - /ovs/framework/scheduling/celery --> used to check whether the stats monkey is disabled or not
# - /ovs/framework/monitoring/stats_monkey --> contains the actual configuration parameters when enabling the stats monkey, such as host, port, username, ...
service_name = 'scheduled-tasks'
if stats_monkey_new is True: # Enable the scheduled task by removing the key
StorageRouterController._logger.info(
'Configuring stats monkey')
interval = stats_monkey_new_config['interval']
# The scheduled task cannot be configured to run more than once a minute, so for intervals < 60, the stats monkey task handles this itself
StorageRouterController._logger.debug(
'Requested interval to run at: {0}'.format(interval))
Configuration.set(key=stats_monkey_config_key,
value=stats_monkey_new_config)
if interval > 0:
days, hours, minutes, _ = ExtensionsToolbox.convert_to_days_hours_minutes_seconds(
seconds=interval)
if days == 1: # Max interval is 24 * 60 * 60, so once every day at 3 AM
schedule = {'hour': '3'}
elif hours > 0:
schedule = {'hour': '*/{0}'.format(hours)}
else:
schedule = {'minute': '*/{0}'.format(minutes)}
stats_monkey_celery_config[
'ovs.stats_monkey.run_all'] = schedule
stats_monkey_celery_config[
'alba.stats_monkey.run_all'] = schedule
StorageRouterController._logger.debug(
'Configured schedule is: {0}'.format(schedule))
else:
stats_monkey_celery_config.pop('ovs.stats_monkey.run_all',
None)
stats_monkey_celery_config.pop('alba.stats_monkey.run_all',
None)
else: # Disable the scheduled task by setting the values for the celery tasks to None
StorageRouterController._logger.info(
'Un-configuring stats monkey')
stats_monkey_celery_config['ovs.stats_monkey.run_all'] = None
stats_monkey_celery_config['alba.stats_monkey.run_all'] = None
Configuration.set(key=stats_monkey_celery_key,
value=stats_monkey_celery_config)
for storagerouter in StorageRouterList.get_masters():
root_client = root_clients[storagerouter]
StorageRouterController._logger.debug(
'Restarting ovs-scheduled-tasks service on node with IP {0}'
.format(root_client.ip))
StorageRouterController._service_manager.restart_service(
name=service_name, client=root_client)
def remove_node(node_ip, silent=None):
"""
Remove the node with specified IP from the cluster
:param node_ip: IP of the node to remove
:type node_ip: str
:param silent: If silent == '--force-yes' no question will be asked to confirm the removal
:type silent: str
:return: None
"""
from ovs.lib.storagedriver import StorageDriverController
from ovs.lib.storagerouter import StorageRouterController
from ovs.dal.lists.storagerouterlist import StorageRouterList
Toolbox.log(logger=NodeRemovalController._logger, messages="Remove node", boxed=True)
Toolbox.log(
logger=NodeRemovalController._logger,
messages="WARNING: Some of these steps may take a very long time, please check the logs for more information\n\n",
)
###############
# VALIDATIONS #
###############
try:
node_ip = node_ip.strip()
if not isinstance(node_ip, str):
raise ValueError("Node IP must be a string")
if not re.match(SSHClient.IP_REGEX, node_ip):
raise ValueError("Invalid IP {0} specified".format(node_ip))
storage_router_all = StorageRouterList.get_storagerouters()
storage_router_masters = StorageRouterList.get_masters()
storage_router_all_ips = set([storage_router.ip for storage_router in storage_router_all])
storage_router_master_ips = set([storage_router.ip for storage_router in storage_router_masters])
storage_router_to_remove = StorageRouterList.get_by_ip(node_ip)
if node_ip not in storage_router_all_ips:
raise ValueError(
"Unknown IP specified\nKnown in model:\n - {0}\nSpecified for removal:\n - {1}".format(
"\n - ".join(storage_router_all_ips), node_ip
)
)
if len(storage_router_all_ips) == 1:
raise RuntimeError("Removing the only node is not possible")
if node_ip in storage_router_master_ips and len(storage_router_master_ips) == 1:
raise RuntimeError("Removing the only master node is not possible")
if System.get_my_storagerouter() == storage_router_to_remove:
raise RuntimeError(
"The node to be removed cannot be identical to the node on which the removal is initiated"
)
Toolbox.log(
logger=NodeRemovalController._logger, messages="Creating SSH connections to remaining master nodes"
)
master_ip = None
ip_client_map = {}
storage_routers_offline = []
storage_router_to_remove_online = True
for storage_router in storage_router_all:
try:
client = SSHClient(storage_router, username="******")
if client.run(["pwd"]):
Toolbox.log(
logger=NodeRemovalController._logger,
messages=" Node with IP {0:<15} successfully connected to".format(storage_router.ip),
)
ip_client_map[storage_router.ip] = client
if storage_router != storage_router_to_remove and storage_router.node_type == "MASTER":
master_ip = storage_router.ip
except UnableToConnectException:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=" Node with IP {0:<15} is unreachable".format(storage_router.ip),
)
storage_routers_offline.append(storage_router)
if storage_router == storage_router_to_remove:
storage_router_to_remove_online = False
if len(ip_client_map) == 0 or master_ip is None:
raise RuntimeError("Could not connect to any master node in the cluster")
storage_router_to_remove.invalidate_dynamics("vdisks_guids")
if (
len(storage_router_to_remove.vdisks_guids) > 0
): # vDisks are supposed to be moved away manually before removing a node
raise RuntimeError("Still vDisks attached to Storage Router {0}".format(storage_router_to_remove.name))
internal_memcached = Toolbox.is_service_internally_managed(service="memcached")
internal_rabbit_mq = Toolbox.is_service_internally_managed(service="rabbitmq")
memcached_endpoints = Configuration.get(key="/ovs/framework/memcache|endpoints")
rabbit_mq_endpoints = Configuration.get(key="/ovs/framework/messagequeue|endpoints")
copy_memcached_endpoints = list(memcached_endpoints)
copy_rabbit_mq_endpoints = list(rabbit_mq_endpoints)
for endpoint in memcached_endpoints:
if endpoint.startswith(storage_router_to_remove.ip):
copy_memcached_endpoints.remove(endpoint)
for endpoint in rabbit_mq_endpoints:
if endpoint.startswith(storage_router_to_remove.ip):
copy_rabbit_mq_endpoints.remove(endpoint)
if len(copy_memcached_endpoints) == 0 and internal_memcached is True:
raise RuntimeError(
"Removal of provided nodes will result in a complete removal of the memcached service"
)
if len(copy_rabbit_mq_endpoints) == 0 and internal_rabbit_mq is True:
raise RuntimeError(
"Removal of provided nodes will result in a complete removal of the messagequeue service"
)
except Exception as exception:
Toolbox.log(
logger=NodeRemovalController._logger, messages=[str(exception)], boxed=True, loglevel="exception"
)
sys.exit(1)
#################
# CONFIRMATIONS #
#################
interactive = silent != "--force-yes"
remove_asd_manager = not interactive # Remove ASD manager if non-interactive else ask
if interactive is True:
proceed = Interactive.ask_yesno(
message="Are you sure you want to remove node {0}?".format(storage_router_to_remove.name),
default_value=False,
)
if proceed is False:
Toolbox.log(logger=NodeRemovalController._logger, messages="Abort removal", title=True)
sys.exit(1)
if storage_router_to_remove_online is True:
client = SSHClient(endpoint=storage_router_to_remove, username="******")
if ServiceManager.has_service(name="asd-manager", client=client):
remove_asd_manager = Interactive.ask_yesno(
message="Do you also want to remove the ASD manager and related ASDs?", default_value=False
)
if remove_asd_manager is True or storage_router_to_remove_online is False:
for function in Toolbox.fetch_hooks("setup", "validate_asd_removal"):
validation_output = function(storage_router_to_remove.ip)
if validation_output["confirm"] is True:
if Interactive.ask_yesno(message=validation_output["question"], default_value=False) is False:
remove_asd_manager = False
break
###########
# REMOVAL #
###########
try:
Toolbox.log(
logger=NodeRemovalController._logger,
messages="Starting removal of node {0} - {1}".format(
storage_router_to_remove.name, storage_router_to_remove.ip
),
)
if storage_router_to_remove_online is False:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=" Marking all Storage Drivers served by Storage Router {0} as offline".format(
storage_router_to_remove.ip
),
)
StorageDriverController.mark_offline(storagerouter_guid=storage_router_to_remove.guid)
# Remove vPools
Toolbox.log(
logger=NodeRemovalController._logger,
messages=" Removing vPools from node".format(storage_router_to_remove.ip),
)
storage_routers_offline_guids = [
sr.guid for sr in storage_routers_offline if sr.guid != storage_router_to_remove.guid
]
for storage_driver in storage_router_to_remove.storagedrivers:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=" Removing vPool {0} from node".format(storage_driver.vpool.name),
)
StorageRouterController.remove_storagedriver(
storagedriver_guid=storage_driver.guid, offline_storage_router_guids=storage_routers_offline_guids
)
# Demote if MASTER
if storage_router_to_remove.node_type == "MASTER":
NodeTypeController.demote_node(
cluster_ip=storage_router_to_remove.ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=storage_router_to_remove.machine_id,
unconfigure_memcached=internal_memcached,
unconfigure_rabbitmq=internal_rabbit_mq,
offline_nodes=storage_routers_offline,
)
# Stop / remove services
Toolbox.log(logger=NodeRemovalController._logger, messages="Stopping and removing services")
config_store = Configuration.get_store()
if storage_router_to_remove_online is True:
client = SSHClient(endpoint=storage_router_to_remove, username="******")
NodeRemovalController.remove_services(
client=client,
node_type=storage_router_to_remove.node_type.lower(),
logger=NodeRemovalController._logger,
)
service = "watcher-config"
if ServiceManager.has_service(service, client=client):
Toolbox.log(logger=NodeRemovalController._logger, messages="Removing service {0}".format(service))
ServiceManager.stop_service(service, client=client)
ServiceManager.remove_service(service, client=client)
if config_store == "etcd":
from ovs.extensions.db.etcd.installer import EtcdInstaller
if Configuration.get(key="/ovs/framework/external_config") is None:
Toolbox.log(logger=NodeRemovalController._logger, messages=" Removing Etcd cluster")
try:
EtcdInstaller.stop("config", client)
EtcdInstaller.remove("config", client)
except Exception as ex:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=["\nFailed to unconfigure Etcd", ex],
loglevel="exception",
)
Toolbox.log(logger=NodeRemovalController._logger, messages="Removing Etcd proxy")
EtcdInstaller.remove_proxy("config", client.ip)
Toolbox.run_hooks(
component="noderemoval",
sub_component="remove",
logger=NodeRemovalController._logger,
cluster_ip=storage_router_to_remove.ip,
complete_removal=remove_asd_manager,
)
# Clean up model
Toolbox.log(logger=NodeRemovalController._logger, messages="Removing node from model")
for service in storage_router_to_remove.services:
service.delete()
for disk in storage_router_to_remove.disks:
for partition in disk.partitions:
partition.delete()
disk.delete()
for j_domain in storage_router_to_remove.domains:
j_domain.delete()
Configuration.delete("/ovs/framework/hosts/{0}".format(storage_router_to_remove.machine_id))
NodeTypeController.restart_framework_and_memcache_services(
clients=ip_client_map,
offline_node_ips=[node.ip for node in storage_routers_offline],
logger=NodeRemovalController._logger,
)
if storage_router_to_remove_online is True:
client = SSHClient(endpoint=storage_router_to_remove, username="******")
if config_store == "arakoon":
client.file_delete(filenames=[ArakoonConfiguration.CACC_LOCATION])
client.file_delete(filenames=[Configuration.BOOTSTRAP_CONFIG_LOCATION])
storage_router_to_remove.delete()
Toolbox.log(logger=NodeRemovalController._logger, messages="Successfully removed node\n")
except Exception as exception:
Toolbox.log(logger=NodeRemovalController._logger, messages="\n")
Toolbox.log(
logger=NodeRemovalController._logger,
messages=["An unexpected error occurred:", str(exception)],
boxed=True,
loglevel="exception",
)
sys.exit(1)
except KeyboardInterrupt:
Toolbox.log(logger=NodeRemovalController._logger, messages="\n")
Toolbox.log(
logger=NodeRemovalController._logger,
messages="This setup was aborted. Open vStorage may be in an inconsistent state, make sure to validate the installation.",
boxed=True,
loglevel="error",
)
sys.exit(1)
if remove_asd_manager is True:
Toolbox.log(logger=NodeRemovalController._logger, messages="\nRemoving ASD Manager")
with remote(storage_router_to_remove.ip, [os]) as rem:
rem.os.system("asd-manager remove --force-yes")
Toolbox.log(logger=NodeRemovalController._logger, messages="Remove nodes finished", title=True)
def nsm_checkup(alba_backend_guid=None,
min_internal_nsms=1,
external_nsm_cluster_names=None):
# type: (Optional[str], Optional[int], Optional[List[str]]) -> None
"""
Validates the current NSM setup/configuration and takes actions where required.
Assumptions:
* A 2 node NSM is considered safer than a 1 node NSM.
* When adding an NSM, the nodes with the least amount of NSM participation are preferred
:param alba_backend_guid: Run for a specific ALBA Backend
:type alba_backend_guid: str
:param min_internal_nsms: Minimum amount of NSM hosts that need to be provided
:type min_internal_nsms: int
:param external_nsm_cluster_names: Information about the additional clusters to claim (only for externally managed Arakoon clusters)
:type external_nsm_cluster_names: list
:return: None
:rtype: NoneType
"""
###############
# Validations #
###############
if external_nsm_cluster_names is None:
external_nsm_cluster_names = []
AlbaArakoonController._logger.info('NSM checkup started')
if min_internal_nsms < 1:
raise ValueError(
'Minimum amount of NSM clusters must be 1 or more')
if not isinstance(external_nsm_cluster_names, list):
raise ValueError(
"'external_nsm_cluster_names' must be of type 'list'")
if len(external_nsm_cluster_names) > 0:
if alba_backend_guid is None:
raise ValueError(
'Additional NSMs can only be configured for a specific ALBA Backend'
)
if min_internal_nsms > 1:
raise ValueError(
"'min_internal_nsms' and 'external_nsm_cluster_names' are mutually exclusive"
)
external_nsm_cluster_names = list(set(
external_nsm_cluster_names)) # Remove duplicate cluster names
for cluster_name in external_nsm_cluster_names:
try:
ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=cluster_name)
except NotFoundException:
raise ValueError(
'Arakoon cluster with name {0} does not exist'.format(
cluster_name))
if alba_backend_guid is None:
alba_backends = [
alba_backend
for alba_backend in AlbaBackendList.get_albabackends()
if alba_backend.backend.status == 'RUNNING'
]
else:
alba_backends = [AlbaBackend(alba_backend_guid)]
masters = StorageRouterList.get_masters()
storagerouters = set()
for alba_backend in alba_backends:
if alba_backend.abm_cluster is None:
raise ValueError(
'No ABM cluster found for ALBA Backend {0}'.format(
alba_backend.name))
if len(alba_backend.abm_cluster.abm_services) == 0:
raise ValueError(
'ALBA Backend {0} does not have any registered ABM services'
.format(alba_backend.name))
if len(alba_backend.nsm_clusters) + len(
external_nsm_cluster_names) > MAX_NSM_AMOUNT:
raise ValueError(
'The maximum of {0} NSM Arakoon clusters will be exceeded. Amount of clusters that can be deployed for this ALBA Backend: {1}'
.format(MAX_NSM_AMOUNT,
MAX_NSM_AMOUNT - len(alba_backend.nsm_clusters)))
# Validate enough externally managed Arakoon clusters are available
if alba_backend.abm_cluster.abm_services[
0].service.is_internal is False:
unused_cluster_names = set([
cluster_info['cluster_name'] for cluster_info in
ArakoonInstaller.get_unused_arakoon_clusters(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM)
])
if set(external_nsm_cluster_names).difference(
unused_cluster_names):
raise ValueError(
'Some of the provided cluster_names have already been claimed before'
)
storagerouters.update(
set(masters)
) # For externally managed we need an available master node
else:
for abm_service in alba_backend.abm_cluster.abm_services: # For internally managed we need all StorageRouters online
storagerouters.add(abm_service.service.storagerouter)
for nsm_cluster in alba_backend.nsm_clusters: # For internally managed we need all StorageRouters online
for nsm_service in nsm_cluster.nsm_services:
storagerouters.add(nsm_service.service.storagerouter)
ssh_clients = {}
for storagerouter in storagerouters:
try:
ssh_clients[storagerouter] = SSHClient(endpoint=storagerouter)
except UnableToConnectException:
raise RuntimeError(
'StorageRouter {0} with IP {1} is not reachable'.format(
storagerouter.name, storagerouter.ip))
version_str = AlbaArakoonInstaller.get_alba_version_string()
nsm_installer = NSMInstaller(version_str=version_str,
ssh_clients=ssh_clients)
##################
# Check Clusters #
##################
safety = Configuration.get(
'/ovs/framework/plugins/alba/config|nsm.safety')
maxload = Configuration.get(
'/ovs/framework/plugins/alba/config|nsm.maxload')
AlbaArakoonController._logger.debug(
'NSM safety is configured at: {0}'.format(safety))
AlbaArakoonController._logger.debug(
'NSM max load is configured at: {0}'.format(maxload))
master_client = None
failed_backends = []
for alba_backend in alba_backends:
try:
# Gather information
AlbaArakoonController._logger.info(
'ALBA Backend {0} - Ensuring NSM safety'.format(
alba_backend.name))
internal = AlbaArakoonInstaller.is_internally_managed(
alba_backend)
nsm_loads = AlbaArakoonController.get_nsm_loads(alba_backend)
nsm_storagerouters = AlbaArakoonController.get_nsms_per_storagerouter(
alba_backend)
sorted_nsm_clusters = sorted(alba_backend.nsm_clusters,
key=lambda k: k.number)
if not internal and len(external_nsm_cluster_names) > 0:
for sr, cl in ssh_clients.iteritems():
if sr.node_type == 'MASTER':
master_client = cl
break
if master_client is None:
# Internal is False and we specified the NSM clusters to claim, but no MASTER nodes online
raise ValueError(
'Could not find an online master node')
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Arakoon clusters are {1} managed'.
format(alba_backend.name,
'internally' if internal is True else 'externally'))
for nsm_number, nsm_load in nsm_loads.iteritems():
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - NSM Cluster {1} - Load {2}'.format(
alba_backend.name, nsm_number, nsm_load))
for sr, count in nsm_storagerouters.iteritems():
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - StorageRouter {1} - NSM Services {2}'
.format(alba_backend.name, sr.name, count))
if internal:
# Extend existing NSM clusters if safety not met
for nsm_cluster in sorted_nsm_clusters:
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Processing NSM {1} - Expected safety {2} - Current safety {3}'
.format(alba_backend.name, nsm_cluster.number,
safety, len(nsm_cluster.nsm_services)))
AlbaArakoonController.ensure_nsm_cluster_safety(
nsm_cluster,
nsm_storagerouters,
nsm_installer=nsm_installer)
AlbaArakoonController.ensure_nsm_clusters_load(
alba_backend,
nsms_per_storagerouter=nsm_storagerouters,
ssh_clients=ssh_clients,
version_str=version_str,
min_internal_nsms=min_internal_nsms,
external_nsm_cluster_names=external_nsm_cluster_names)
except Exception:
AlbaArakoonController._logger.exception(
'NSM Checkup failed for Backend {0}'.format(
alba_backend.name))
failed_backends.append(alba_backend.name)
def validate_cluster(cluster_name='ovsdb'):
"""
Validate if the chosen cluster is
* deployed on all required nodes
* running on all required nodes
* working correctly on all required nodes
:param cluster_name: name of a existing arakoon cluster (DEFAULT=ovsdb)
:type cluster_name: str
:return:
"""
ArakoonValidation.LOGGER.info("Starting validating arakoon cluster")
master_storagerouters = [
storagerouter.ip
for storagerouter in StorageRouterList.get_masters()
]
assert len(master_storagerouters
) >= 2, 'Environment has only `{0}` node(s)'.format(
len(master_storagerouters))
master_storagerouters.sort()
arakoon_service_name = "ovs-arakoon-{0}".format(cluster_name)
service_manager = ServiceFactory.get_manager()
for storagerouter_ip in master_storagerouters:
client = SSHClient(storagerouter_ip, username='******')
# check if service file is available
ArakoonValidation.LOGGER.info(
"Validating if cluster service `{0}` is available on node `{1}`"
.format(cluster_name, storagerouter_ip))
assert service_manager.has_service(arakoon_service_name, client), "Service file of `{0}` does not exists on storagerouter `{1}`"\
.format(cluster_name, storagerouter_ip)
# check if service is running on system
ArakoonValidation.LOGGER.info(
"Validating if cluster service `{0}` is running on node `{1}`".
format(cluster_name, storagerouter_ip))
assert service_manager.get_service_status(arakoon_service_name, client) == 'active', \
"Service of `{0}` is not running on storagerouter `{1}`".format(cluster_name, storagerouter_ip)
# perform nop, get and set on cluster
key = 'integration-tests-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
ArakoonValidation.LOGGER.info(
"Validating if cluster `{0}` works".format(cluster_name))
# determine if there is a healthy cluster
configuration = Configuration.get(
'/ovs/arakoon/{0}/config'.format(cluster_name), raw=True)
client = PyrakoonStore(cluster_name, configuration)
client.nop()
# perform set, get & compare
client.set(key, value)
get_value = client.get(key)
assert get_value == value, "Value mismatch on cluster `{0}`, get value `{1}`, " \
"expected value `{2}` on key `{3}`".format(cluster_name, get_value, value, key)
# perform delete
client.delete(key)
try:
assert not client.get(key), "Key `{0}` still exists on cluster `{1}` after deleting it"\
.format(key, cluster_name)
except KeyNotFoundException:
# key not found so test has passed
assert True
ArakoonValidation.LOGGER.info("Finished validating arakoon cluster")
def validate_healthcheck():
"""
Validate if the healthcheck works
Will always check: localhost & master
Will check if available: slave
:return:
"""
HealthCheckCI.LOGGER.info('Starting to validate the healthcheck')
storagerouter_master_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_masters()]
assert len(storagerouter_master_ips) >= 1, 'Not enough MASTER storagerouters'
storagerouter_slave_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
# setup base information
node_ips = [storagerouter_master_ips[0], '127.0.0.1']
HealthCheckCI.LOGGER.info('Added master `{0}` & `127.0.0.1` to the nodes to be tested by the healthcheck'
.format(storagerouter_master_ips[0]))
# add a slave if possible
if len(storagerouter_slave_ips) != 0:
node_ips.append(storagerouter_slave_ips[0])
HealthCheckCI.LOGGER.info('Added slave `{0}` to be checked by the healthcheck ...'
.format(storagerouter_slave_ips[0]))
else:
HealthCheckCI.LOGGER.warning('Did not add a slave to the list of to be tested nodes because '
'none were available')
# check if there are missing packages
for ip in node_ips:
HealthCheckCI.LOGGER.info('Starting the healthcheck on node `{0}`'.format(ip))
########################################
# Testing command line, on remote node #
########################################
client = SSHClient(ip, username='******')
assert client.run(['ovs', 'healthcheck']) is not None
assert client.run(['ovs', 'healthcheck', '--unattended']) is not None
assert client.run(['ovs', 'healthcheck', '--to-json']) is not None
# looping the help seperate modules
help_options = filter(None, client.run(['ovs', 'healthcheck', '--help']).split('\n'))
ignored_help_options = ['ovs healthcheck X X -- will run all checks', 'ovs healthcheck MODULE X -- will run all checks for module']
for help_option in help_options:
if 'Possible' in help_option or help_option in ignored_help_options:
continue
assert client.run(help_option.split()) is not None
assert client.run(['ovs', 'healthcheck', 'alba', '--help']) is not None
assert client.run(['ovs', 'healthcheck', 'alba', 'disk-safety-test', '--help']) is not None
HealthCheckCI.LOGGER.info('Finished running the healthcheck on node `{0}`'.format(ip))
##########################
# Testing by code import #
##########################
from ovs.extensions.healthcheck.expose_to_cli import HealthCheckCLIRunner
hc_output = HealthCheckCLIRunner.run_method()
assert hc_output is not None, 'No results found in the healthcheck output'
assert 'result' in hc_output, 'the result section is missing in the healthcheck output'
assert 'recap' in hc_output, 'the recap section is missing in the healthcheck output'
mapped_result = {'FAILED': {}, 'EXCEPTION': {}}
for test_name, result in hc_output['result'].iteritems():
if result['state'] == 'EXCEPTION':
mapped_result['EXCEPTION'].update({test_name: result})
if result['state'] == 'FAILED':
mapped_result['FAILED'].update({test_name: result})
recap = hc_output['recap']
assert recap['EXCEPTION'] == 0, '{0} exception(s) found during the healthcheck run: {1}'.format(recap['EXCEPTION'], dict(mapped_result['EXCEPTION'], **mapped_result['FAILED']))
assert recap['FAILED'] == 0, '{0} failure(s) found during the healthcheck run: {1}'.format(recap['FAILED'], mapped_result['FAILED'])
HealthCheckCI.LOGGER.info('Finished validating the healthcheck')
return hc_output
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_result: Port information
:type arakoon_result: Dictionary
:return: The newly created and added service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [
arakoon_result['client_port'], arakoon_result['messaging_port']
]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [
storagerouter.ip
for storagerouter in StorageRouterList.get_slaves()
]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(
storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(
current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips,
storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
for storagerouter, partition in available_storagerouters.iteritems(
):
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip, storagerouter.ip,
cluster_name, partition.folder)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips,
storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
def _generate_queue_urls():
mq_user = Configuration.get('/ovs/framework/messagequeue|user')
mq_protocol = Configuration.get('/ovs/framework/messagequeue|protocol')
mq_password = Configuration.get('/ovs/framework/messagequeue|password')
return [{'amqp_uri': '{0}://{1}:{2}@{3}:5672'.format(mq_protocol, mq_user, mq_password, sr.ip)} for sr in StorageRouterList.get_masters()]
def get_slave_ips():
"""
Retrieve all Storage Router slave ips
:return: list of slave ips
"""
return [sr.ip for sr in StorageRouterList.get_masters()]
def _alba_arakoon_checkup(cls,
alba_backend_guid=None,
abm_cluster=None,
nsm_clusters=None):
# type: (Optional[str], Optional[str], Optional[List[str]]) -> None
"""
Creates a new Arakoon cluster if required and extends cluster if possible on all available master nodes
:param alba_backend_guid: Guid of the ALBA Backend
:type alba_backend_guid: str
:param nsm_clusters: NSM clusters for this ALBA Backend
The code will claim the Arakoon clusters for this backend when provided
:type nsm_clusters: list[str]
:param abm_cluster: ABM cluster for this ALBA Backend
The code will claim the Arakoon cluster for this backend when provided
:type abm_cluster: str|None
:return:None
:rtype: NoneType
"""
slaves = StorageRouterList.get_slaves()
masters = StorageRouterList.get_masters()
clients = {}
for storagerouter in masters + slaves:
try:
clients[storagerouter] = SSHClient(storagerouter)
except UnableToConnectException:
cls._logger.warning(
'Storage Router with IP {0} is not reachable'.format(
storagerouter.ip))
available_storagerouters = cls.get_available_arakoon_storagerouters(
clients)
# Call here, because this potentially raises error, which should happen before actually making changes
abm_installer = ABMInstaller(ssh_clients=clients)
nsm_installer = NSMInstaller(version_str=abm_installer.version_str,
ssh_clients=clients)
# Cluster creation
if alba_backend_guid is not None:
alba_backend = AlbaBackend(alba_backend_guid)
# @todo revisit. This might enforce the ABM name for externals (might be unintended)
abm_cluster_name = '{0}-abm'.format(alba_backend.name)
# ABM Arakoon cluster creation
if alba_backend.abm_cluster is None:
# Fallback to installing the cluster on an available storagerouter
storagerouter, partition = available_storagerouters.items()[0]
abm_installer.deploy_abm_cluster(
alba_backend,
abm_cluster_name,
requested_abm_cluster_name=abm_cluster,
storagerouter=storagerouter)
# NSM Arakoon cluster creation
if len(alba_backend.nsm_clusters
) == 0 and nsm_clusters is not None:
storagerouter, partition = available_storagerouters.items()[0]
nsm_installer.deploy_nsm_cluster(alba_backend,
storagerouter=storagerouter,
nsm_clusters=nsm_clusters)
# ABM Cluster extension
for alba_backend in AlbaBackendList.get_albabackends():
if alba_backend.abm_cluster is None:
AlbaArakoonController._logger.warning(
'ALBA Backend {0} does not have an ABM cluster registered'.
format(alba_backend.name))
continue
cls.ensure_abm_cluster_safety(alba_backend.abm_cluster,
available_storagerouters,
abm_installer=abm_installer)
def remove_node(node_ip, silent=None):
"""
Remove the node with specified IP from the cluster
:param node_ip: IP of the node to remove
:type node_ip: str
:param silent: If silent == '--force-yes' no question will be asked to confirm the removal
:type silent: str
:return: None
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.lib.storagedriver import StorageDriverController
from ovs.lib.vpool import VPoolController
Toolbox.log(logger=NodeRemovalController._logger,
messages='Remove node',
boxed=True)
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
'WARNING: Some of these steps may take a very long time, please check the logs for more information\n\n'
)
service_manager = ServiceFactory.get_manager()
###############
# VALIDATIONS #
###############
try:
node_ip = node_ip.strip()
if not isinstance(node_ip, str):
raise ValueError('Node IP must be a string')
if not re.match(SSHClient.IP_REGEX, node_ip):
raise ValueError('Invalid IP {0} specified'.format(node_ip))
storage_router_all = sorted(StorageRouterList.get_storagerouters(),
key=lambda k: k.name)
storage_router_masters = StorageRouterList.get_masters()
storage_router_all_ips = set(
[storage_router.ip for storage_router in storage_router_all])
storage_router_master_ips = set([
storage_router.ip for storage_router in storage_router_masters
])
storage_router_to_remove = StorageRouterList.get_by_ip(node_ip)
offline_reasons = {}
if node_ip not in storage_router_all_ips:
raise ValueError(
'Unknown IP specified\nKnown in model:\n - {0}\nSpecified for removal:\n - {1}'
.format('\n - '.join(storage_router_all_ips), node_ip))
if len(storage_router_all_ips) == 1:
raise RuntimeError("Removing the only node is not possible")
if node_ip in storage_router_master_ips and len(
storage_router_master_ips) == 1:
raise RuntimeError(
"Removing the only master node is not possible")
if System.get_my_storagerouter() == storage_router_to_remove:
raise RuntimeError(
'The node to be removed cannot be identical to the node on which the removal is initiated'
)
Toolbox.log(
logger=NodeRemovalController._logger,
messages='Creating SSH connections to remaining master nodes')
master_ip = None
ip_client_map = {}
storage_routers_offline = []
storage_router_to_remove_online = True
for storage_router in storage_router_all:
try:
client = SSHClient(storage_router,
username='******',
timeout=10)
except (UnableToConnectException, NotAuthenticatedException,
TimeOutException) as ex:
if isinstance(ex, UnableToConnectException):
msg = 'Unable to connect'
elif isinstance(ex, NotAuthenticatedException):
msg = 'Could not authenticate'
elif isinstance(ex, TimeOutException):
msg = 'Connection timed out'
Toolbox.log(
logger=NodeRemovalController._logger,
messages=' * Node with IP {0:<15}- {1}'.format(
storage_router.ip, msg))
offline_reasons[storage_router.ip] = msg
storage_routers_offline.append(storage_router)
if storage_router == storage_router_to_remove:
storage_router_to_remove_online = False
continue
Toolbox.log(
logger=NodeRemovalController._logger,
messages=' * Node with IP {0:<15}- Successfully connected'
.format(storage_router.ip))
ip_client_map[storage_router.ip] = client
if storage_router != storage_router_to_remove and storage_router.node_type == 'MASTER':
master_ip = storage_router.ip
if len(ip_client_map) == 0 or master_ip is None:
raise RuntimeError(
'Could not connect to any master node in the cluster')
storage_router_to_remove.invalidate_dynamics('vdisks_guids')
if len(
storage_router_to_remove.vdisks_guids
) > 0: # vDisks are supposed to be moved away manually before removing a node
raise RuntimeError(
"Still vDisks attached to Storage Router {0}".format(
storage_router_to_remove.name))
internal_memcached = Toolbox.is_service_internally_managed(
service='memcached')
internal_rabbit_mq = Toolbox.is_service_internally_managed(
service='rabbitmq')
memcached_endpoints = Configuration.get(
key='/ovs/framework/memcache|endpoints')
rabbit_mq_endpoints = Configuration.get(
key='/ovs/framework/messagequeue|endpoints')
copy_memcached_endpoints = list(memcached_endpoints)
copy_rabbit_mq_endpoints = list(rabbit_mq_endpoints)
for endpoint in memcached_endpoints:
if endpoint.startswith(storage_router_to_remove.ip):
copy_memcached_endpoints.remove(endpoint)
for endpoint in rabbit_mq_endpoints:
if endpoint.startswith(storage_router_to_remove.ip):
copy_rabbit_mq_endpoints.remove(endpoint)
if len(copy_memcached_endpoints
) == 0 and internal_memcached is True:
raise RuntimeError(
'Removal of provided nodes will result in a complete removal of the memcached service'
)
if len(copy_rabbit_mq_endpoints
) == 0 and internal_rabbit_mq is True:
raise RuntimeError(
'Removal of provided nodes will result in a complete removal of the messagequeue service'
)
Toolbox.run_hooks(component='noderemoval',
sub_component='validate_removal',
logger=NodeRemovalController._logger,
cluster_ip=storage_router_to_remove.ip)
except KeyboardInterrupt:
Toolbox.log(logger=NodeRemovalController._logger, messages='\n')
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
'Removal has been aborted during the validation step. No changes have been applied.',
boxed=True,
loglevel='warning')
sys.exit(1)
except Exception as exception:
Toolbox.log(logger=NodeRemovalController._logger,
messages=[str(exception)],
boxed=True,
loglevel='exception')
sys.exit(1)
#################
# CONFIRMATIONS #
#################
try:
interactive = silent != '--force-yes'
remove_asd_manager = not interactive # Remove ASD manager if non-interactive else ask
if interactive is True:
if len(storage_routers_offline) > 0:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
'Certain nodes appear to be offline. These will not fully removed and will cause issues if they are not really offline.'
)
Toolbox.log(
logger=NodeRemovalController._logger,
messages='Offline nodes: {0}'.format(''.join(
('\n * {0:<15}- {1}.'.format(ip, message)
for ip, message in offline_reasons.iteritems()))))
valid_node_info = Interactive.ask_yesno(
message=
'Continue the removal with these being presumably offline?',
default_value=False)
if valid_node_info is False:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
'Please validate the state of the nodes before removing.',
title=True)
sys.exit(1)
proceed = Interactive.ask_yesno(
message='Are you sure you want to remove node {0}?'.format(
storage_router_to_remove.name),
default_value=False)
if proceed is False:
Toolbox.log(logger=NodeRemovalController._logger,
messages='Abort removal',
title=True)
sys.exit(1)
remove_asd_manager = True
if storage_router_to_remove_online is True:
client = SSHClient(endpoint=storage_router_to_remove,
username='******')
if service_manager.has_service(name='asd-manager',
client=client):
remove_asd_manager = Interactive.ask_yesno(
message=
'Do you also want to remove the ASD manager and related ASDs?',
default_value=False)
if remove_asd_manager is True or storage_router_to_remove_online is False:
for fct in Toolbox.fetch_hooks('noderemoval',
'validate_asd_removal'):
validation_output = fct(storage_router_to_remove.ip)
if validation_output['confirm'] is True:
if Interactive.ask_yesno(
message=validation_output['question'],
default_value=False) is False:
remove_asd_manager = False
break
except KeyboardInterrupt:
Toolbox.log(logger=NodeRemovalController._logger, messages='\n')
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
'Removal has been aborted during the confirmation step. No changes have been applied.',
boxed=True,
loglevel='warning')
sys.exit(1)
except Exception as exception:
Toolbox.log(logger=NodeRemovalController._logger,
messages=[str(exception)],
boxed=True,
loglevel='exception')
sys.exit(1)
###########
# REMOVAL #
###########
try:
Toolbox.log(logger=NodeRemovalController._logger,
messages='Starting removal of node {0} - {1}'.format(
storage_router_to_remove.name,
storage_router_to_remove.ip))
if storage_router_to_remove_online is False:
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
' Marking all Storage Drivers served by Storage Router {0} as offline'
.format(storage_router_to_remove.ip))
StorageDriverController.mark_offline(
storagerouter_guid=storage_router_to_remove.guid)
# Remove vPools
Toolbox.log(logger=NodeRemovalController._logger,
messages=' Removing vPools from node'.format(
storage_router_to_remove.ip))
storage_routers_offline_guids = [
sr.guid for sr in storage_routers_offline
if sr.guid != storage_router_to_remove.guid
]
for storage_driver in storage_router_to_remove.storagedrivers:
Toolbox.log(logger=NodeRemovalController._logger,
messages=' Removing vPool {0} from node'.format(
storage_driver.vpool.name))
VPoolController.shrink_vpool(
storagedriver_guid=storage_driver.guid,
offline_storage_router_guids=storage_routers_offline_guids)
# Demote if MASTER
if storage_router_to_remove.node_type == 'MASTER':
NodeTypeController.demote_node(
cluster_ip=storage_router_to_remove.ip,
master_ip=master_ip,
ip_client_map=ip_client_map,
unique_id=storage_router_to_remove.machine_id,
unconfigure_memcached=internal_memcached,
unconfigure_rabbitmq=internal_rabbit_mq,
offline_nodes=storage_routers_offline)
# Stop / remove services
Toolbox.log(logger=NodeRemovalController._logger,
messages='Stopping and removing services')
if storage_router_to_remove_online is True:
client = SSHClient(endpoint=storage_router_to_remove,
username='******')
NodeRemovalController.remove_services(
client=client,
node_type=storage_router_to_remove.node_type.lower(),
logger=NodeRemovalController._logger)
service = 'watcher-config'
if service_manager.has_service(service, client=client):
Toolbox.log(
logger=NodeRemovalController._logger,
messages='Removing service {0}'.format(service))
service_manager.stop_service(service, client=client)
service_manager.remove_service(service, client=client)
Toolbox.run_hooks(component='noderemoval',
sub_component='remove',
logger=NodeRemovalController._logger,
cluster_ip=storage_router_to_remove.ip,
complete_removal=remove_asd_manager)
# Clean up model
Toolbox.log(logger=NodeRemovalController._logger,
messages='Removing node from model')
for service in storage_router_to_remove.services:
service.delete()
for disk in storage_router_to_remove.disks:
for partition in disk.partitions:
partition.delete()
disk.delete()
for j_domain in storage_router_to_remove.domains:
j_domain.delete()
Configuration.delete('/ovs/framework/hosts/{0}'.format(
storage_router_to_remove.machine_id))
NodeTypeController.restart_framework_and_memcache_services(
clients=ip_client_map,
offline_node_ips=[node.ip for node in storage_routers_offline],
logger=NodeRemovalController._logger)
if storage_router_to_remove_online is True:
client = SSHClient(endpoint=storage_router_to_remove,
username='******')
client.file_delete(filenames=[CACC_LOCATION])
client.file_delete(filenames=[CONFIG_STORE_LOCATION])
storage_router_to_remove.delete()
Toolbox.log(logger=NodeRemovalController._logger,
messages='Successfully removed node\n')
except Exception as exception:
Toolbox.log(logger=NodeRemovalController._logger, messages='\n')
Toolbox.log(
logger=NodeRemovalController._logger,
messages=['An unexpected error occurred:',
str(exception)],
boxed=True,
loglevel='exception')
sys.exit(1)
except KeyboardInterrupt:
Toolbox.log(logger=NodeRemovalController._logger, messages='\n')
Toolbox.log(
logger=NodeRemovalController._logger,
messages=
'This setup was aborted. Open vStorage may be in an inconsistent state, make sure to validate the installation.',
boxed=True,
loglevel='error')
sys.exit(1)
if remove_asd_manager is True and storage_router_to_remove_online is True:
Toolbox.log(logger=NodeRemovalController._logger,
messages='\nRemoving ASD Manager')
with remote(storage_router_to_remove.ip, [os]) as rem:
rem.os.system('asd-manager remove --force-yes')
Toolbox.log(logger=NodeRemovalController._logger,
messages='Remove nodes finished',
title=True)
def restart_masters(cls):
"""
Will restart ovs-workers on all master nodes
:return: None
"""
cls.restart([sr for sr in StorageRouterList.get_masters()])
def run_test(cls, cluster_info, compute_client, vm_info, vm_username=CIConstants.VM_USERNAME, vm_password=CIConstants.VM_PASSWORD,
timeout=TEST_TIMEOUT, data_test_cases=CIConstants.DATA_TEST_CASES, logger=LOGGER):
"""
Runs the test as described in https://github.com/openvstorage/dev_ops/issues/64
:param cluster_info: information about the cluster
:param compute_client: SSHclient of the computenode
:param vm_info: vm information
:param vm_username: username to login on all vms
:param vm_password: password to login on all vms
:param timeout: timeout in seconds
:param data_test_cases: data rw ratios to test
:param logger: logging instance
:return:
"""
compute_str = cluster_info['storagerouters']['compute']
destination_storagedriver = cluster_info['storagedrivers']['destination']
source_storagedriver = cluster_info['storagedrivers']['source']
# Cache to validate properties
values_to_check = {
'source_std': source_storagedriver.serialize(),
'target_std': destination_storagedriver.serialize()
}
# Prep VM listener #
failed_configurations = []
# Extract vdisk info from vm_info - only get the data ones
vdisk_info = {}
disk_amount = 0
for vm_name, vm_object in vm_info.iteritems():
for vdisk in vm_object['vdisks']:
if 'vdisk_data' in vdisk.name:
vdisk_info.update({vdisk.name: vdisk})
disk_amount += 1
try:
cls._adjust_automatic_scrubbing(disable=True)
with remote(compute_str.ip, [SSHClient]) as rem:
configuration = random.choice(data_test_cases)
threads = {'evented': {'io': {'pairs': [], 'r_semaphore': None},
'snapshots': {'pairs': [], 'r_semaphore': None}}}
output_files = []
safety_set = False
try:
logger.info('Starting the following configuration: {0}'.format(configuration))
for vm_name, vm_data in vm_info.iteritems():
vm_client = rem.SSHClient(vm_data['ip'], vm_username, vm_password)
vm_client.file_create('/mnt/data/{0}.raw'.format(vm_data['create_msg']))
vm_data['client'] = vm_client
cls._set_mds_safety(source_storagedriver.vpool, 1, checkup=True) # Set the safety to trigger the mds
safety_set = True
io_thread_pairs, monitoring_data, io_r_semaphore = ThreadingHandler.start_io_polling_threads(volume_bundle=vdisk_info)
threads['evented']['io']['pairs'] = io_thread_pairs
threads['evented']['io']['r_semaphore'] = io_r_semaphore
# @todo snapshot every minute
threads['evented']['snapshots']['pairs'] = ThreadingHandler.start_snapshotting_threads(volume_bundle=vdisk_info, kwargs={'interval': 15})
for vm_name, vm_data in vm_info.iteritems(): # Write data
screen_names, output_files = DataWriter.write_data_fio(client=vm_data['client'],
fio_configuration={
'io_size': cls.AMOUNT_TO_WRITE,
'configuration': configuration},
file_locations=['/mnt/data/{0}.raw'.format(vm_data['create_msg'])])
vm_data['screen_names'] = screen_names
logger.info('Doing IO for {0}s before bringing down the node.'.format(cls.IO_TIME))
ThreadingHandler.keep_threads_running(r_semaphore=io_r_semaphore,
threads=io_thread_pairs,
shared_resource=monitoring_data,
duration=cls.IO_TIME / 2)
ThreadHelper.stop_evented_threads(threads['evented']['snapshots']['pairs'],
threads['evented']['snapshots']['r_semaphore']) # Stop snapshotting
cls._delete_snapshots(volume_bundle=vdisk_info)
# Start scrubbing thread
async_scrubbing = cls.start_scrubbing(volume_bundle=vdisk_info) # Starting to scrub
cls._trigger_mds_issue(cluster_info['vpool'], vdisk_info, destination_storagedriver.storagerouter.guid) # Trigger mds failover while scrubber is busy
# Do some monitoring further for 60s
ThreadingHandler.keep_threads_running(r_semaphore=io_r_semaphore,
threads=io_thread_pairs,
shared_resource=monitoring_data,
duration=cls.IO_TIME / 2)
time.sleep(cls.IO_REFRESH_RATE * 2)
downed_time = time.time()
# Start IO polling to verify nothing went down
ThreadingHandler.poll_io(r_semaphore=io_r_semaphore,
required_thread_amount=len(io_thread_pairs),
shared_resource=monitoring_data,
downed_time=downed_time,
timeout=timeout,
output_files=output_files,
client=compute_client,
disk_amount=disk_amount)
possible_scrub_errors = async_scrubbing.get() # Wait until scrubbing calls have given a result
assert len(possible_scrub_errors) == 0, 'Scrubbing has encountered some errors: {0}'.format(', '.join(possible_scrub_errors))
cls._validate(values_to_check, monitoring_data)
except Exception as ex:
logger.error('Running the test for configuration {0} has failed because {1}'.format(configuration, str(ex)))
failed_configurations.append({'configuration': configuration, 'reason': str(ex)})
raise
finally:
for thread_category, thread_collection in threads['evented'].iteritems():
ThreadHelper.stop_evented_threads(thread_collection['pairs'], thread_collection['r_semaphore'])
for vm_name, vm_data in vm_info.iteritems():
for screen_name in vm_data.get('screen_names', []):
logger.debug('Stopping screen {0} on {1}.'.format(screen_name, vm_data['client'].ip))
vm_data['client'].run(['screen', '-S', screen_name, '-X', 'quit'])
vm_data['screen_names'] = []
if safety_set is True:
cls._set_mds_safety(source_storagedriver.vpool, len(StorageRouterList.get_masters()), checkup=True)
finally:
cls._adjust_automatic_scrubbing(disable=False)
assert len(failed_configurations) == 0, 'Certain configuration failed: {0}'.format(' '.join(failed_configurations))
def ensure_nsm_clusters_load(cls,
alba_backend,
nsms_per_storagerouter=None,
min_internal_nsms=1,
external_nsm_cluster_names=None,
version_str=None,
ssh_clients=None):
# type: (AlbaBackend, Optional[Dict[StorageRouter, int]], Optional[int], Optional[List[str], Optional[str]], Optional[StorageRouter, SSHClient]) -> None
"""
Ensure that all NSM clusters are not overloaded
:param alba_backend: Alba Backend to ensure NSM Cluster load for
:type alba_backend: AlbaBackend
:param nsms_per_storagerouter: Amount of NSMs mapped by StorageRouter
:type nsms_per_storagerouter: Dict[StorageRouter, int]
:param min_internal_nsms: Minimum amount of NSM hosts that need to be provided
:type min_internal_nsms: int
:param external_nsm_cluster_names: Information about the additional clusters to claim (only for externally managed Arakoon clusters)
:type external_nsm_cluster_names: list
:param version_str: Alba version string
:type version_str: str
:param ssh_clients: SSHClients to use
:type ssh_clients: Dict[Storagerouter, SSHClient]
:return: None
:rtype: NoneType
"""
if ssh_clients is None:
ssh_clients = {}
if external_nsm_cluster_names is None:
external_nsm_cluster_names = []
nsms_per_storagerouter = nsms_per_storagerouter if nsms_per_storagerouter is not None else cls.get_nsms_per_storagerouter(
alba_backend)
version_str = version_str or AlbaArakoonInstaller.get_alba_version_string(
)
nsm_loads = cls.get_nsm_loads(alba_backend)
internal = AlbaArakoonInstaller.is_internally_managed(alba_backend)
abm_cluster_name = alba_backend.abm_cluster.name
safety = Configuration.get(
'/ovs/framework/plugins/alba/config|nsm.safety')
maxload = Configuration.get(
'/ovs/framework/plugins/alba/config|nsm.maxload')
overloaded = min(nsm_loads.values()) >= maxload
if not overloaded:
# At least 1 NSM is not overloaded yet
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - NSM load OK'.format(alba_backend.name))
if internal:
# When load is not OK, deploy at least 1 additional NSM
nsms_to_add = max(0, min_internal_nsms - len(nsm_loads))
else:
nsms_to_add = len(external_nsm_cluster_names)
if nsms_to_add == 0:
return
else:
AlbaArakoonController._logger.warning(
'ALBA Backend {0} - NSM load is NOT OK'.format(
alba_backend.name))
if internal:
# When load is not OK, deploy at least 1 additional NSM
nsms_to_add = max(1, min_internal_nsms - len(nsm_loads))
else:
# For externally managed clusters we only claim the specified clusters, if none provided, we just log it
nsms_to_add = len(external_nsm_cluster_names)
if nsms_to_add == 0:
cls._logger.critical(
'ALBA Backend {0} - All NSM clusters are overloaded'.
format(alba_backend.name))
return
# Deploy new (internal) or claim existing (external) NSM clusters
cls._logger.debug(
'ALBA Backend {0} - Currently {1} NSM cluster{2}'.format(
alba_backend.name, len(nsm_loads),
'' if len(nsm_loads) == 1 else 's'))
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Trying to add {1} NSM cluster{2}'.format(
alba_backend.name, nsms_to_add,
'' if nsms_to_add == 1 else 's'))
base_number = max(nsm_loads.keys()) + 1
for index, number in enumerate(
xrange(base_number, base_number + nsms_to_add)):
if not internal:
# External clusters
master_client = None
if not ssh_clients:
for storagerouter in StorageRouterList.get_masters():
try:
master_client = SSHClient(storagerouter)
except UnableToConnectException:
cls._logger.warning(
'StorageRouter {0} with IP {1} is not reachable'
.format(storagerouter.name, storagerouter.ip))
else:
for storagerouter, ssh_client in ssh_clients.iteritems():
if storagerouter.node_type == 'MASTER':
master_client = ssh_client
if not master_client:
raise ValueError('Could not find an online master node')
# @todo this might raise an indexerror?
nsm_cluster_name = external_nsm_cluster_names[index]
cls._logger.debug(
'ALBA Backend {0} - Claiming NSM cluster {1}'.format(
alba_backend.name, nsm_cluster_name))
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM,
cluster_name=nsm_cluster_name)
if metadata is None:
cls._logger.critical(
'ALBA Backend {0} - NSM cluster with name {1} could not be found'
.format(alba_backend.name, nsm_cluster_name))
continue
cls._logger.debug(
'ALBA Backend {0} - Modeling services'.format(
alba_backend.name))
AlbaArakoonInstaller.model_arakoon_service(
alba_backend=alba_backend,
cluster_name=nsm_cluster_name,
number=number)
cls._logger.debug('ALBA Backend {0} - Registering NSM'.format(
alba_backend.name))
NSMInstaller.register_nsm(abm_name=abm_cluster_name,
nsm_name=nsm_cluster_name,
ip=master_client.ip)
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Extended cluster'.format(
alba_backend.name))
else:
# Internal clusters
nsm_cluster_name = '{0}-nsm_{1}'.format(
alba_backend.name, number)
cls._logger.debug(
'ALBA Backend {0} - Adding NSM cluster {1}'.format(
alba_backend.name, nsm_cluster_name))
# One of the NSM nodes is overloaded. This means the complete NSM is considered overloaded
# Figure out which StorageRouters are the least occupied
loads = sorted(nsms_per_storagerouter.values())[:safety]
storagerouters = []
for storagerouter, load in nsms_per_storagerouter.iteritems():
if load in loads:
storagerouters.append(storagerouter)
if len(storagerouters) == safety:
break
# Creating a new NSM cluster
for sub_index, storagerouter in enumerate(storagerouters):
nsms_per_storagerouter[storagerouter] += 1
partition = AlbaArakoonInstaller.get_db_partition(
storagerouter)
arakoon_installer = ArakoonInstaller(
cluster_name=nsm_cluster_name)
# @todo Use deploy and extend code. (Disable register nsm in those parts)
if sub_index == 0:
arakoon_installer.create_cluster(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM,
ip=storagerouter.ip,
base_dir=partition.folder,
plugins={NSM_PLUGIN: version_str})
else:
cls._logger.debug(
'ALBA Backend {0} - Extending NSM cluster {1}'.
format(alba_backend.name, nsm_cluster_name))
arakoon_installer.load()
arakoon_installer.extend_cluster(
new_ip=storagerouter.ip,
base_dir=partition.folder,
plugins={NSM_PLUGIN: version_str})
cls._logger.debug(
'ALBA Backend {0} - Linking plugins'.format(
alba_backend.name))
ssh_client = ssh_clients.get(storagerouter) or SSHClient(
StorageRouter)
AlbaArakoonInstaller.link_plugins(
client=ssh_client,
data_dir=partition.folder,
plugins=[NSM_PLUGIN],
cluster_name=nsm_cluster_name)
cls._logger.debug(
'ALBA Backend {0} - Modeling services'.format(
alba_backend.name))
AlbaArakoonInstaller.model_arakoon_service(
alba_backend=alba_backend,
cluster_name=nsm_cluster_name,
ports=arakoon_installer.ports[storagerouter.ip],
storagerouter=storagerouter,
number=number)
if sub_index == 0:
cls._logger.debug(
'ALBA Backend {0} - Starting cluster'.format(
alba_backend.name))
arakoon_installer.start_cluster()
else:
AlbaArakoonController._logger.debug(
'ALBA Backend {0} - Restarting cluster'.format(
alba_backend.name))
arakoon_installer.restart_cluster_after_extending(
new_ip=storagerouter.ip)
cls._logger.debug('ALBA Backend {0} - Registering NSM'.format(
alba_backend.name))
NSMInstaller.register_nsm(abm_name=abm_cluster_name,
nsm_name=nsm_cluster_name,
ip=storagerouters[0].ip)
cls._logger.debug(
'ALBA Backend {0} - Added NSM cluster {1}'.format(
alba_backend.name, nsm_cluster_name))
def _voldrv_arakoon_checkup(create_cluster):
def _add_service(service_storagerouter, arakoon_ports, service_name):
""" Add a service to the storage router """
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = arakoon_ports
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
current_ips = []
current_services = []
service_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
cluster_name = Configuration.get(
'/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is not None:
arakoon_service_name = ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)
for service in service_type.services:
if service.name == arakoon_service_name:
current_services.append(service)
if service.is_internal is True:
current_ips.append(service.storagerouter.ip)
all_sr_ips = [
storagerouter.ip
for storagerouter in StorageRouterList.get_slaves()
]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(
storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(
current_services) == 0: # Create new cluster
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
if metadata is None: # No externally managed cluster found, we create 1 ourselves
if not available_storagerouters:
raise RuntimeError(
'Could not find any Storage Router with a DB role')
storagerouter, partition = available_storagerouters.items()[0]
arakoon_voldrv_cluster = 'voldrv'
arakoon_installer = ArakoonInstaller(
cluster_name=arakoon_voldrv_cluster)
arakoon_installer.create_cluster(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouter.ip,
base_dir=partition.folder,
log_sinks=LogHandler.get_sink_path(
'arakoon-server_{0}'.format(arakoon_voldrv_cluster)),
crash_log_sinks=LogHandler.get_sink_path(
'arakoon-server-crash_{0}'.format(
arakoon_voldrv_cluster)))
arakoon_installer.start_cluster()
ports = arakoon_installer.ports[storagerouter.ip]
metadata = arakoon_installer.metadata
current_ips.append(storagerouter.ip)
else:
ports = []
storagerouter = None
cluster_name = metadata['cluster_name']
Configuration.set('/ovs/framework/arakoon_clusters|voldrv',
cluster_name)
StorageDriverController._logger.info(
'Claiming {0} managed arakoon cluster: {1}'.format(
'externally' if storagerouter is None else 'internally',
cluster_name))
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
current_services.append(
_add_service(
service_storagerouter=storagerouter,
arakoon_ports=ports,
service_name=ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)))
cluster_name = Configuration.get(
'/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is None:
return
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=cluster_name)
if 0 < len(current_services) < len(
available_storagerouters) and metadata['internal'] is True:
for storagerouter, partition in available_storagerouters.iteritems(
):
if storagerouter.ip in current_ips:
continue
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
arakoon_installer.load()
arakoon_installer.extend_cluster(
new_ip=storagerouter.ip,
base_dir=partition.folder,
log_sinks=LogHandler.get_sink_path(
'arakoon-server_{0}'.format(cluster_name)),
crash_log_sinks=LogHandler.get_sink_path(
'arakoon-server-crash_{0}'.format(cluster_name)))
_add_service(
service_storagerouter=storagerouter,
arakoon_ports=arakoon_installer.ports[storagerouter.ip],
service_name=ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name))
current_ips.append(storagerouter.ip)
arakoon_installer.restart_cluster_after_extending(
new_ip=storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_ports):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_ports: Port information
:type arakoon_ports: list
:return: The newly created and added service
:rtype: Service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = arakoon_ports
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)
current_ips = []
current_services = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
if service.is_internal is True:
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0: # Create new cluster
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
if metadata is None: # No externally managed cluster found, we create 1 ourselves
if not available_storagerouters:
raise RuntimeError('Could not find any Storage Router with a DB role')
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name='voldrv',
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouter.ip,
base_dir=partition.folder,
filesystem=False)
ports = [result['client_port'], result['messaging_port']]
metadata = result['metadata']
ArakoonInstaller.restart_cluster_add(cluster_name='voldrv',
current_ips=current_ips,
new_ip=storagerouter.ip,
filesystem=False)
ArakoonInstaller.claim_cluster(cluster_name='voldrv',
master_ip=storagerouter.ip,
filesystem=False,
metadata=metadata)
current_ips.append(storagerouter.ip)
else:
ports = []
storagerouter = None
cluster_name = metadata['cluster_name']
Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name)
StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name))
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports))
cluster_name = Configuration.get('/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is None:
return
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if 0 < len(current_services) < len(available_storagerouters) and metadata['internal'] is True:
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip,
new_ip=storagerouter.ip,
cluster_name=cluster_name,
base_dir=partition.folder)
add_service(storagerouter, [result['client_port'], result['messaging_port']])
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name=cluster_name,
current_ips=current_ips,
new_ip=storagerouter.ip,
filesystem=False)
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
exclude_ports=ServiceList.get_ports_for_ip(storagerouter.ip),
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
for sr_ip in all_sr_ips:
if sr_ip not in current_ips:
ArakoonInstaller.deploy_to_slave(storagerouter.ip, sr_ip, cluster_name)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
distributed = False
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip,
storagerouter.ip,
cluster_name,
ServiceList.get_ports_for_ip(storagerouter.ip),
partition.folder
)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
if distributed is False:
distributed = True
for sr_ip in all_sr_ips:
if sr_ip not in current_ips:
ArakoonInstaller.deploy_to_slave(current_services[0].storagerouter.ip, sr_ip, cluster_name)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
def migrate():
"""
Executes async migrations. It doesn't matter too much when they are executed, as long as they get eventually
executed. This code will typically contain:
* "dangerous" migration code (it needs certain running services)
* Migration code depending on a cluster-wide state
* ...
* Successfully finishing a piece of migration code, should create an entry in /ovs/framework/migration in case it should not be executed again
* Eg: /ovs/framework/migration|stats_monkey_integration: True
"""
MigrationController._logger.info('Preparing out of band migrations...')
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.vpoollist import VPoolList
from ovs.extensions.db.arakooninstaller import ArakoonInstaller
from ovs.extensions.generic.configuration import Configuration
from ovs.extensions.generic.sshclient import SSHClient
from ovs_extensions.generic.toolbox import ExtensionsToolbox
from ovs.extensions.migration.migration.ovsmigrator import ExtensionMigrator
from ovs.extensions.packages.packagefactory import PackageFactory
from ovs_extensions.services.interfaces.systemd import Systemd
from ovs.extensions.services.servicefactory import ServiceFactory
from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration
from ovs.lib.helpers.storagedriver.installer import StorageDriverInstaller
MigrationController._logger.info('Start out of band migrations...')
service_manager = ServiceFactory.get_manager()
sr_client_map = {}
for storagerouter in StorageRouterList.get_storagerouters():
sr_client_map[storagerouter.guid] = SSHClient(endpoint=storagerouter.ip, # Is triggered during post-update code too during which the ovs-watcher-framework service is still down and thus not refreshing the heartbeat --> use IP i/o StorageRouter
username='******')
#########################################################
# Addition of 'ExecReload' for AlbaProxy SystemD services
if ServiceFactory.get_service_type() == 'systemd':
changed_clients = set()
for storagedriver in StorageDriverList.get_storagedrivers():
root_client = sr_client_map[storagedriver.storagerouter_guid]
for alba_proxy in storagedriver.alba_proxies:
service = alba_proxy.service
service_name = 'ovs-{0}'.format(service.name)
if not service_manager.has_service(name=service_name, client=root_client):
continue
if 'ExecReload=' in root_client.file_read(filename='/lib/systemd/system/{0}.service'.format(service_name)):
continue
try:
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY, client=root_client, target_name=service_name)
changed_clients.add(root_client)
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
##################################################################
# Adjustment of open file descriptors for Arakoon services to 8192
changed_clients = set()
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map[storagerouter.guid]
for service_name in service_manager.list_services(client=root_client):
if not service_name.startswith('ovs-arakoon-'):
continue
if ServiceFactory.get_service_type() == 'systemd':
path = '/lib/systemd/system/{0}.service'.format(service_name)
check = 'LimitNOFILE=8192'
else:
path = '/etc/init/{0}.conf'.format(service_name)
check = 'limit nofile 8192 8192'
if not root_client.file_exists(path):
continue
if check in root_client.file_read(path):
continue
try:
service_manager.regenerate_service(name='ovs-arakoon', client=root_client, target_name=service_name)
changed_clients.add(root_client)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='arakoon',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, service_name))
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
#############################
# Migrate to multiple proxies
for storagedriver in StorageDriverList.get_storagedrivers():
vpool = storagedriver.vpool
root_client = sr_client_map[storagedriver.storagerouter_guid]
for alba_proxy in storagedriver.alba_proxies:
# Rename alba_proxy service in model
service = alba_proxy.service
old_service_name = 'albaproxy_{0}'.format(vpool.name)
new_service_name = 'albaproxy_{0}_0'.format(vpool.name)
if old_service_name != service.name:
continue
service.name = new_service_name
service.save()
if not service_manager.has_service(name=old_service_name, client=root_client):
continue
old_configuration_key = '/ovs/framework/hosts/{0}/services/{1}'.format(storagedriver.storagerouter.machine_id, old_service_name)
if not Configuration.exists(key=old_configuration_key):
continue
# Add '-reboot' to alba_proxy services (because of newly created services and removal of old service)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='alba',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, old_service_name),
new_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, new_service_name))
# Register new service and remove old service
service_manager.add_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY,
client=root_client,
params=Configuration.get(old_configuration_key),
target_name='ovs-{0}'.format(new_service_name))
# Update scrub proxy config
proxy_config_key = '/ovs/vpools/{0}/proxies/{1}/config/main'.format(vpool.guid, alba_proxy.guid)
proxy_config = None if Configuration.exists(key=proxy_config_key) is False else Configuration.get(proxy_config_key)
if proxy_config is not None:
fragment_cache = proxy_config.get(StorageDriverConfiguration.CACHE_FRAGMENT, ['none', {}])
if fragment_cache[0] == 'alba' and fragment_cache[1].get('cache_on_write') is True: # Accelerated ALBA configured
fragment_cache_scrub_info = copy.deepcopy(fragment_cache)
fragment_cache_scrub_info[1]['cache_on_read'] = False
proxy_scrub_config_key = '/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid)
proxy_scrub_config = None if Configuration.exists(key=proxy_scrub_config_key) is False else Configuration.get(proxy_scrub_config_key)
if proxy_scrub_config is not None and proxy_scrub_config[StorageDriverConfiguration.CACHE_FRAGMENT] == ['none']:
proxy_scrub_config[StorageDriverConfiguration.CACHE_FRAGMENT] = fragment_cache_scrub_info
Configuration.set(key=proxy_scrub_config_key, value=proxy_scrub_config)
# Update 'backend_connection_manager' section
changes = False
storagedriver_config = StorageDriverConfiguration(vpool.guid, storagedriver.storagedriver_id)
if 'backend_connection_manager' not in storagedriver_config.configuration:
continue
current_config = storagedriver_config.configuration['backend_connection_manager']
if current_config.get('backend_type') != 'MULTI':
changes = True
backend_connection_manager = {'backend_type': 'MULTI'}
for index, proxy in enumerate(sorted(storagedriver.alba_proxies, key=lambda pr: pr.service.ports[0])):
backend_connection_manager[str(index)] = copy.deepcopy(current_config)
# noinspection PyUnresolvedReferences
backend_connection_manager[str(index)]['alba_connection_use_rora'] = True
# noinspection PyUnresolvedReferences
backend_connection_manager[str(index)]['alba_connection_rora_manifest_cache_capacity'] = 5000
# noinspection PyUnresolvedReferences
for key, value in backend_connection_manager[str(index)].items():
if key.startswith('backend_interface'):
backend_connection_manager[key] = value
# noinspection PyUnresolvedReferences
del backend_connection_manager[str(index)][key]
for key, value in {'backend_interface_retries_on_error': 5,
'backend_interface_retry_interval_secs': 1,
'backend_interface_retry_backoff_multiplier': 2.0}.iteritems():
if key not in backend_connection_manager:
backend_connection_manager[key] = value
else:
backend_connection_manager = current_config
for value in backend_connection_manager.values():
if isinstance(value, dict):
for key, val in value.items():
if key.startswith('backend_interface'):
backend_connection_manager[key] = val
changes = True
del value[key]
for key, value in {'backend_interface_retries_on_error': 5,
'backend_interface_retry_interval_secs': 1,
'backend_interface_retry_backoff_multiplier': 2.0}.iteritems():
if key not in backend_connection_manager:
changes = True
backend_connection_manager[key] = value
if changes is True:
storagedriver_config.clear_backend_connection_manager()
storagedriver_config.configure_backend_connection_manager(**backend_connection_manager)
storagedriver_config.save(root_client)
# Add '-reboot' to volumedriver services (because of updated 'backend_connection_manager' section)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='volumedriver',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, 'volumedriver_{0}'.format(vpool.name)))
if service_manager.__class__ == Systemd:
root_client.run(['systemctl', 'daemon-reload'])
########################################
# Update metadata_store_bits information
vpools = VPoolList.get_vpools()
for vpool in vpools:
bits = None
for storagedriver in vpool.storagedrivers:
key = '/ovs/framework/hosts/{0}/services/volumedriver_{1}'.format(storagedriver.storagerouter.machine_id, vpool.name)
if Configuration.exists(key=key) and 'METADATASTORE_BITS' not in Configuration.get(key=key):
if bits is None:
entries = service_manager.extract_from_service_file(name='ovs-volumedriver_{0}'.format(vpool.name),
client=sr_client_map[storagedriver.storagerouter_guid],
entries=['METADATASTORE_BITS='])
if len(entries) == 1:
bits = entries[0].split('=')[-1]
bits = int(bits) if bits.isdigit() else 5
if bits is not None:
try:
content = Configuration.get(key=key)
content['METADATASTORE_BITS'] = bits
Configuration.set(key=key, value=content)
except:
MigrationController._logger.exception('Error updating volumedriver info for vPool {0} on StorageRouter {1}'.format(vpool.name, storagedriver.storagerouter.name))
if bits is not None:
vpool.metadata_store_bits = bits
vpool.save()
#####################################
# Update the vPool metadata structure
def _update_metadata_structure(metadata):
metadata = copy.deepcopy(metadata)
cache_structure = {'read': False,
'write': False,
'is_backend': False,
'quota': None,
'backend_info': {'name': None, # Will be filled in when is_backend is true
'backend_guid': None,
'alba_backend_guid': None,
'policies': None,
'preset': None,
'arakoon_config': None,
'connection_info': {'client_id': None,
'client_secret': None,
'host': None,
'port': None,
'local': None}}
}
structure_map = {StorageDriverConfiguration.CACHE_BLOCK: {'read': 'block_cache_on_read',
'write': 'block_cache_on_write',
'quota': 'quota_bc',
'backend_prefix': 'backend_bc_{0}'},
StorageDriverConfiguration.CACHE_FRAGMENT: {'read': 'fragment_cache_on_read',
'write': 'fragment_cache_on_write',
'quota': 'quota_fc',
'backend_prefix': 'backend_aa_{0}'}}
if 'arakoon_config' in metadata['backend']: # Arakoon config should be placed under the backend info
metadata['backend']['backend_info']['arakoon_config'] = metadata['backend'].pop('arakoon_config')
if 'connection_info' in metadata['backend']: # Connection info sohuld be placed under the backend info
metadata['backend']['backend_info']['connection_info'] = metadata['backend'].pop('connection_info')
if 'caching_info' not in metadata: # Caching info is the new key
would_be_caching_info = {}
metadata['caching_info'] = would_be_caching_info
# Extract all caching data for every storagerouter
current_caching_info = metadata['backend'].pop('caching_info') # Pop to mutate metadata
for storagerouter_guid in current_caching_info.iterkeys():
current_cache_data = current_caching_info[storagerouter_guid]
storagerouter_caching_info = {}
would_be_caching_info[storagerouter_guid] = storagerouter_caching_info
for cache_type, cache_type_mapping in structure_map.iteritems():
new_cache_structure = copy.deepcopy(cache_structure)
storagerouter_caching_info[cache_type] = new_cache_structure
for new_structure_key, old_structure_key in cache_type_mapping.iteritems():
if new_structure_key == 'backend_prefix':
# Get possible backend related info
metadata_key = old_structure_key.format(storagerouter_guid)
if metadata_key not in metadata:
continue
backend_data = metadata.pop(metadata_key) # Pop to mutate metadata
new_cache_structure['is_backend'] = True
# Copy over the old data
new_cache_structure['backend_info']['arakoon_config'] = backend_data['arakoon_config']
new_cache_structure['backend_info'].update(backend_data['backend_info'])
new_cache_structure['backend_info']['connection_info'].update(backend_data['connection_info'])
else:
new_cache_structure[new_structure_key] = current_cache_data.get(old_structure_key)
return metadata
vpools = VPoolList.get_vpools()
for vpool in vpools:
try:
new_metadata = _update_metadata_structure(vpool.metadata)
vpool.metadata = new_metadata
vpool.save()
except KeyError:
MigrationController._logger.exception('Exceptions occurred when updating the metadata for vPool {0}'.format(vpool.name))
##############################################
# Always use indent=4 during Configuration set
def _resave_all_config_entries(config_path='/ovs'):
"""
Recursive functions which checks every config management key if its a directory or not.
If not a directory, we retrieve the config and just save it again using the new indentation logic
"""
for item in Configuration.list(config_path):
new_path = config_path + '/' + item
print new_path
if Configuration.dir_exists(new_path) is True:
_resave_all_config_entries(config_path=new_path)
else:
try:
_config = Configuration.get(new_path)
Configuration.set(new_path, _config)
except:
_config = Configuration.get(new_path, raw=True)
Configuration.set(new_path, _config, raw=True)
if ExtensionMigrator.THIS_VERSION <= 13: # There is no way of checking whether this new indentation logic has been applied, so we only perform this for version 13 and lower
MigrationController._logger.info('Re-saving every configuration setting with new indentation rules')
_resave_all_config_entries()
############################
# Update some default values
def _update_manifest_cache_size(_proxy_config_key):
updated = False
manifest_cache_size = 500 * 1024 * 1024
if Configuration.exists(key=_proxy_config_key):
_proxy_config = Configuration.get(key=_proxy_config_key)
for cache_type in [StorageDriverConfiguration.CACHE_BLOCK, StorageDriverConfiguration.CACHE_FRAGMENT]:
if cache_type in _proxy_config and _proxy_config[cache_type][0] == 'alba':
if _proxy_config[cache_type][1]['manifest_cache_size'] != manifest_cache_size:
updated = True
_proxy_config[cache_type][1]['manifest_cache_size'] = manifest_cache_size
if _proxy_config['manifest_cache_size'] != manifest_cache_size:
updated = True
_proxy_config['manifest_cache_size'] = manifest_cache_size
if updated is True:
Configuration.set(key=_proxy_config_key, value=_proxy_config)
return updated
for storagedriver in StorageDriverList.get_storagedrivers():
try:
vpool = storagedriver.vpool
root_client = sr_client_map[storagedriver.storagerouter_guid]
_update_manifest_cache_size('/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid)) # Generic scrub proxy is deployed every time scrubbing kicks in, so no need to restart these services
for alba_proxy in storagedriver.alba_proxies:
if _update_manifest_cache_size('/ovs/vpools/{0}/proxies/{1}/config/main'.format(vpool.guid, alba_proxy.guid)) is True:
# Add '-reboot' to alba_proxy services (because of newly created services and removal of old service)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='alba',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, alba_proxy.service.name))
# Update 'backend_connection_manager' section
changes = False
storagedriver_config = StorageDriverConfiguration(vpool.guid, storagedriver.storagedriver_id)
if 'backend_connection_manager' not in storagedriver_config.configuration:
continue
current_config = storagedriver_config.configuration['backend_connection_manager']
for key, value in current_config.iteritems():
if key.isdigit() is True:
if value.get('alba_connection_asd_connection_pool_capacity') != 10:
changes = True
value['alba_connection_asd_connection_pool_capacity'] = 10
if value.get('alba_connection_timeout') != 30:
changes = True
value['alba_connection_timeout'] = 30
if value.get('alba_connection_rora_manifest_cache_capacity') != 25000:
changes = True
value['alba_connection_rora_manifest_cache_capacity'] = 25000
if changes is True:
storagedriver_config.clear_backend_connection_manager()
storagedriver_config.configure_backend_connection_manager(**current_config)
storagedriver_config.save(root_client)
# Add '-reboot' to volumedriver services (because of updated 'backend_connection_manager' section)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='volumedriver',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, 'volumedriver_{0}'.format(vpool.name)))
except Exception:
MigrationController._logger.exception('Updating default configuration values failed for StorageDriver {0}'.format(storagedriver.storagedriver_id))
####################################################
# Adding proxy fail fast as env variable for proxies
changed_clients = set()
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map[storagerouter.guid]
for service_name in service_manager.list_services(client=root_client):
if not service_name.startswith('ovs-albaproxy_'):
continue
if ServiceFactory.get_service_type() == 'systemd':
path = '/lib/systemd/system/{0}.service'.format(service_name)
check = 'Environment=ALBA_FAIL_FAST=true'
else:
path = '/etc/init/{0}.conf'.format(service_name)
check = 'env ALBA_FAIL_FAST=true'
if not root_client.file_exists(path):
continue
if check in root_client.file_read(path):
continue
try:
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY, client=root_client, target_name=service_name)
changed_clients.add(root_client)
ExtensionsToolbox.edit_version_file(client=root_client,
package_name='alba',
old_run_file='{0}/{1}.version'.format(ServiceFactory.RUN_FILE_DIR, service_name))
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
######################################
# Integration of stats monkey (2.10.2)
if Configuration.get(key='/ovs/framework/migration|stats_monkey_integration', default=False) is False:
try:
# Get content of old key into new key
old_stats_monkey_key = '/statsmonkey/statsmonkey'
if Configuration.exists(key=old_stats_monkey_key) is True:
Configuration.set(key='/ovs/framework/monitoring/stats_monkey', value=Configuration.get(key=old_stats_monkey_key))
Configuration.delete(key=old_stats_monkey_key)
# Make sure to disable the stats monkey by default or take over the current schedule if it was configured manually before
celery_key = '/ovs/framework/scheduling/celery'
current_value = None
scheduling_config = Configuration.get(key=celery_key, default={})
if 'statsmonkey.run_all_stats' in scheduling_config: # Old celery task name of the stats monkey
current_value = scheduling_config.pop('statsmonkey.run_all_stats')
scheduling_config['ovs.stats_monkey.run_all'] = current_value
scheduling_config['alba.stats_monkey.run_all'] = current_value
Configuration.set(key=celery_key, value=scheduling_config)
support_key = '/ovs/framework/support'
support_config = Configuration.get(key=support_key)
support_config['support_agent'] = support_config.pop('enabled', True)
support_config['remote_access'] = support_config.pop('enablesupport', False)
Configuration.set(key=support_key, value=support_config)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|stats_monkey_integration', value=True)
except Exception:
MigrationController._logger.exception('Integration of stats monkey failed')
######################################################
# Write away cluster ID to a file for back-up purposes
try:
cluster_id = Configuration.get(key='/ovs/framework/cluster_id', default=None)
with open(Configuration.CONFIG_STORE_LOCATION, 'r') as config_file:
config = json.load(config_file)
if cluster_id is not None and config.get('cluster_id', None) is None:
config['cluster_id'] = cluster_id
with open(Configuration.CONFIG_STORE_LOCATION, 'w') as config_file:
json.dump(config, config_file, indent=4)
except Exception:
MigrationController._logger.exception('Writing cluster id to a file failed.')
#########################################################
# Additional string formatting in Arakoon services (2.11)
try:
if Configuration.get(key='/ovs/framework/migration|arakoon_service_update', default=False) is False:
arakoon_service_names = [ArakoonInstaller.get_service_name_for_cluster(cluster_name=cluster_name) for cluster_name in Configuration.list(key='ovs/arakoon')]
for storagerouter in StorageRouterList.get_masters():
for service_name in arakoon_service_names:
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(storagerouter.machine_id, service_name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['ARAKOON_PKG_NAME'] = PackageFactory.PKG_ARAKOON
config['ARAKOON_VERSION_CMD'] = PackageFactory.VERSION_CMD_ARAKOON
Configuration.set(key=config_key, value=config)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|arakoon_service_update', value=True)
except Exception:
MigrationController._logger.exception('Updating the string formatting for the Arakoon services failed')
############################################################
# Additional string formatting in ALBA proxy services (2.11)
changed_clients = set()
try:
if Configuration.get(key='/ovs/framework/migration|alba_proxy_service_update', default=False) is False:
alba_pkg_name, alba_version_cmd = PackageFactory.get_package_and_version_cmd_for(component=PackageFactory.COMP_ALBA)
for service in ServiceTypeList.get_by_name('AlbaProxy').services:
root_client = sr_client_map[service.storagerouter_guid]
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(service.storagerouter.machine_id, service.name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['ALBA_PKG_NAME'] = alba_pkg_name
config['ALBA_VERSION_CMD'] = alba_version_cmd
Configuration.set(key=config_key, value=config)
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_PROXY,
client=root_client,
target_name='ovs-{0}'.format(service.name))
changed_clients.add(root_client)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|alba_proxy_service_update', value=True)
except Exception:
MigrationController._logger.exception('Updating the string formatting for the Arakoon services failed')
############################################################
# Additional string formatting in DTL/VOLDRV services (2.11)
try:
if Configuration.get(key='/ovs/framework/migration|voldrv_service_update', default=False) is False:
sd_pkg_name, sd_version_cmd = PackageFactory.get_package_and_version_cmd_for(component=PackageFactory.COMP_SD)
for vpool in VPoolList.get_vpools():
for storagedriver in vpool.storagedrivers:
root_client = sr_client_map[storagedriver.storagerouter_guid]
for entry in ['dtl', 'volumedriver']:
service_name = '{0}_{1}'.format(entry, vpool.name)
service_template = StorageDriverInstaller.SERVICE_TEMPLATE_DTL if entry == 'dtl' else StorageDriverInstaller.SERVICE_TEMPLATE_SD
config_key = ServiceFactory.SERVICE_CONFIG_KEY.format(storagedriver.storagerouter.machine_id, service_name)
if Configuration.exists(key=config_key):
config = Configuration.get(key=config_key)
config['RUN_FILE_DIR'] = ServiceFactory.RUN_FILE_DIR
config['VOLDRV_PKG_NAME'] = sd_pkg_name
config['VOLDRV_VERSION_CMD'] = sd_version_cmd
Configuration.set(key=config_key, value=config)
service_manager.regenerate_service(name=service_template,
client=root_client,
target_name='ovs-{0}'.format(service_name))
changed_clients.add(root_client)
# Make sure once this finished, it never runs again by setting this key to True
Configuration.set(key='/ovs/framework/migration|voldrv_service_update', value=True)
except Exception:
MigrationController._logger.exception('Updating the string formatting for the Arakoon services failed')
#######################################################
# Storing actual package name in version files (2.11.0) (https://github.com/openvstorage/framework/issues/1876)
if Configuration.get(key='/ovs/framework/migration|actual_package_name_in_version_file', default=False) is False:
try:
voldrv_pkg_name, _ = PackageFactory.get_package_and_version_cmd_for(component=PackageFactory.COMP_SD)
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map.get(storagerouter.guid)
if root_client is None:
continue
for file_name in root_client.file_list(directory=ServiceFactory.RUN_FILE_DIR):
if not file_name.endswith('.version'):
continue
file_path = '{0}/{1}'.format(ServiceFactory.RUN_FILE_DIR, file_name)
contents = root_client.file_read(filename=file_path)
regenerate = False
if voldrv_pkg_name == PackageFactory.PKG_VOLDRV_SERVER:
if 'volumedriver-server' in contents:
regenerate = True
contents = contents.replace('volumedriver-server', PackageFactory.PKG_VOLDRV_SERVER)
root_client.file_write(filename=file_path, contents=contents)
elif voldrv_pkg_name == PackageFactory.PKG_VOLDRV_SERVER_EE:
if 'volumedriver-server' in contents or PackageFactory.PKG_VOLDRV_SERVER in contents:
regenerate = True
contents = contents.replace('volumedriver-server', PackageFactory.PKG_VOLDRV_SERVER_EE)
contents = contents.replace(PackageFactory.PKG_VOLDRV_SERVER, PackageFactory.PKG_VOLDRV_SERVER_EE)
root_client.file_write(filename=file_path, contents=contents)
if regenerate is True:
service_manager.regenerate_service(name=StorageDriverInstaller.SERVICE_TEMPLATE_DTL if file_name.startswith('dtl') else StorageDriverInstaller.SERVICE_TEMPLATE_SD,
client=root_client,
target_name='ovs-{0}'.format(file_name.split('.')[0])) # Leave out .version
changed_clients.add(root_client)
Configuration.set(key='/ovs/framework/migration|actual_package_name_in_version_file', value=True)
except Exception:
MigrationController._logger.exception('Updating actual package name for version files failed')
for root_client in changed_clients:
try:
root_client.run(['systemctl', 'daemon-reload'])
except Exception:
MigrationController._logger.exception('Executing command "systemctl daemon-reload" failed')
#########################################################
# Addition of 'Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50' for AlbaProxy SystemD services
if ServiceFactory.get_service_type() == 'systemd':
changed_clients = set()
for storagedriver in StorageDriverList.get_storagedrivers():
root_client = sr_client_map[storagedriver.storagerouter_guid]
for alba_proxy in storagedriver.alba_proxies:
service = alba_proxy.service
service_name = 'ovs-{0}'.format(service.name)
if not service_manager.has_service(name=service_name, client=root_client):
continue
if "Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50" in root_client.file_read(filename='/lib/systemd/system/{0}.service'.format(service_name)):
continue
try:
service_manager.regenerate_service(name='ovs-albaproxy', client=root_client, target_name=service_name)
changed_clients.add(root_client)
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
#########################################################
# Addition of 'Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50' for Arakoon SystemD services
if ServiceFactory.get_service_type() == 'systemd':
changed_clients = set()
for storagerouter in StorageRouterList.get_storagerouters():
root_client = sr_client_map[storagerouter.guid]
for service_name in service_manager.list_services(client=root_client):
if not service_name.startswith('ovs-arakoon-'):
continue
if not service_manager.has_service(name=service_name, client=root_client):
continue
if "Environment=OCAMLRUNPARAM='b,a=1,s=4096k,O=50" in root_client.file_read(filename='/lib/systemd/system/{0}.service'.format(service_name)):
continue
try:
service_manager.regenerate_service(name='ovs-arakoon', client=root_client, target_name=service_name)
changed_clients.add(root_client)
except:
MigrationController._logger.exception('Error rebuilding service {0}'.format(service_name))
for root_client in changed_clients:
root_client.run(['systemctl', 'daemon-reload'])
MigrationController._logger.info('Finished out of band migrations')
def get_masters():
"""
Retrieve all Storage Router masters
:return: Data-object list with Storage Routers
"""
return StorageRouterList.get_masters()
