def _deploy(config, offline_nodes=None):
"""
Deploys a complete cluster: Distributing the configuration files, creating directories and services
"""
ArakoonInstaller._logger.debug('Deploying cluster {0}'.format(config.cluster_id))
if offline_nodes is None:
offline_nodes = []
for node in config.nodes:
if node.ip in offline_nodes:
continue
ArakoonInstaller._logger.debug(' Deploying cluster {0} on {1}'.format(config.cluster_id, node.ip))
root_client = SSHClient(node.ip, username='******')
# Distributes a configuration file to all its nodes
config.write_config()
# Create dirs as root because mountpoint /mnt/cache1 is typically owned by root
abs_paths = [node.log_dir, node.tlog_dir, node.home]
if not root_client.dir_exists(abs_paths):
root_client.dir_create(abs_paths)
root_client.dir_chmod(abs_paths, 0755, recursive=True)
root_client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
# Creates services for/on all nodes in the config
base_name = 'ovs-arakoon'
target_name = 'ovs-arakoon-{0}'.format(config.cluster_id)
ServiceManager.add_service(base_name, root_client,
params={'CLUSTER': config.cluster_id,
'NODE_ID': node.name,
'CONFIG_PATH': ArakoonInstaller.ETCD_CONFIG_PATH.format(config.cluster_id)},
target_name=target_name)
ArakoonInstaller._logger.debug(' Deploying cluster {0} on {1} completed'.format(config.cluster_id, node.ip))
def _deploy(config):
"""
Deploys a complete cluster: Distributing the configuration files, creating directories and services
"""
logger.debug("Deploying cluster {0}".format(config.cluster_id))
for node in config.nodes:
logger.debug(" Deploying cluster {0} on {1}".format(config.cluster_id, node.ip))
ovs_client = SSHClient(node.ip)
root_client = SSHClient(node.ip, username="******")
# Distributes a configuration file to all its nodes
config.write_config(ovs_client)
# Create dirs as root because mountpoint /mnt/cache1 is typically owned by root
abs_paths = [node.log_dir, node.tlog_dir, node.home]
root_client.dir_create(abs_paths)
root_client.dir_chmod(abs_paths, 0755, recursive=True)
root_client.dir_chown(abs_paths, "ovs", "ovs", recursive=True)
# Creates services for/on all nodes in the config
base_name = "ovs-arakoon"
target_name = "ovs-arakoon-{0}".format(config.cluster_id)
ServiceManager.prepare_template(base_name, target_name, ovs_client)
ServiceManager.add_service(target_name, root_client, params={"CLUSTER": config.cluster_id})
logger.debug(" Deploying cluster {0} on {1} completed".format(config.cluster_id, node.ip))
def extend_cluster(master_ip, new_ip, cluster_name):
"""
Extends a cluster to a given new node
:param cluster_name: Name of the cluster to be extended
:param new_ip: IP address of the node to be added
:param master_ip: IP of one of the already existing nodes
"""
logger.debug('Extending cluster "{0}" from {1} to {2}'.format(cluster_name, master_ip, new_ip))
client = SSHClient(master_ip, username='******')
if not EtcdInstaller._is_healty(cluster_name, client):
raise RuntimeError('Cluster "{0}" unhealthy, aborting extend'.format(cluster_name))
cluster_members = client.run('etcdctl member list').splitlines()
for cluster_member in cluster_members:
if EtcdInstaller.SERVER_URL.format(new_ip) in cluster_member:
logger.info('Node {0} already member of etcd cluster'.format(new_ip))
return
current_cluster = []
for item in client.run('etcdctl member list').splitlines():
info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict()
current_cluster.append('{0}={1}'.format(info['name'], info['peer']))
client = SSHClient(new_ip, username='******')
node_name = System.get_my_machine_id(client)
current_cluster.append('{0}={1}'.format(node_name, EtcdInstaller.SERVER_URL.format(new_ip)))
data_dir = EtcdInstaller.DATA_DIR.format(EtcdInstaller.DB_DIR, cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(EtcdInstaller.DB_DIR, cluster_name)
abs_paths = [data_dir, wal_dir]
client.dir_delete(abs_paths)
client.dir_create(abs_paths)
client.dir_chmod(abs_paths, 0755, recursive=True)
client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
base_name = 'ovs-etcd'
target_name = 'ovs-etcd-{0}'.format(cluster_name)
EtcdInstaller.stop(cluster_name, client) # Stop a possible proxy service
ServiceManager.add_service(base_name, client,
params={'CLUSTER': cluster_name,
'NODE_ID': node_name,
'DATA_DIR': data_dir,
'WAL_DIR': wal_dir,
'SERVER_URL': EtcdInstaller.SERVER_URL.format(new_ip),
'CLIENT_URL': EtcdInstaller.CLIENT_URL.format(new_ip),
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1'),
'INITIAL_CLUSTER': ','.join(current_cluster),
'INITIAL_STATE': 'existing',
'INITIAL_PEERS': ''},
target_name=target_name)
master_client = SSHClient(master_ip, username='******')
master_client.run('etcdctl member add {0} {1}'.format(node_name, EtcdInstaller.SERVER_URL.format(new_ip)))
EtcdInstaller.start(cluster_name, client)
EtcdInstaller.wait_for_cluster(cluster_name, client)
logger.debug('Extending cluster "{0}" from {1} to {2} completed'.format(cluster_name, master_ip, new_ip))
def _deploy(config, filesystem, offline_nodes=None, plugins=None, delay_service_registration=False):
"""
Deploys a complete cluster: Distributing the configuration files, creating directories and services
"""
if os.environ.get('RUNNING_UNITTESTS') == 'True':
if filesystem is True:
raise NotImplementedError('At this moment, there is no support for unit-testing filesystem backend Arakoon clusters')
ArakoonInstaller._logger.debug('Deploying cluster {0}'.format(config.cluster_id))
if offline_nodes is None:
offline_nodes = []
service_metadata = {}
for node in config.nodes:
if node.ip in offline_nodes:
continue
ArakoonInstaller._logger.debug(' Deploying cluster {0} on {1}'.format(config.cluster_id, node.ip))
root_client = SSHClient(node.ip, username='******')
# Distributes a configuration file to all its nodes
config.write_config(node.ip)
# Create dirs as root because mountpoint /mnt/cache1 is typically owned by root
abs_paths = {node.tlog_dir, node.home} # That's a set
if node.log_sinks.startswith('/'):
abs_paths.add(os.path.dirname(os.path.abspath(node.log_sinks)))
if node.crash_log_sinks.startswith('/'):
abs_paths.add(os.path.dirname(os.path.abspath(node.crash_log_sinks)))
abs_paths = list(abs_paths)
root_client.dir_create(abs_paths)
root_client.dir_chmod(abs_paths, 0755, recursive=True)
root_client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
# Creates services for/on all nodes in the config
if config.filesystem is True:
config_path = config.config_path
else:
config_path = Configuration.get_configuration_path(config.config_path)
extra_version_cmd = ''
if plugins is not None:
extra_version_cmd = ';'.join(plugins)
metadata = ServiceManager.add_service(name='ovs-arakoon',
client=root_client,
params={'CLUSTER': config.cluster_id,
'NODE_ID': node.name,
'CONFIG_PATH': config_path,
'EXTRA_VERSION_CMD': extra_version_cmd},
target_name='ovs-arakoon-{0}'.format(config.cluster_id),
startup_dependency=('ovs-watcher-config' if filesystem is False else None),
delay_registration=delay_service_registration)
service_metadata[node.ip] = metadata
ArakoonInstaller._logger.debug(' Deploying cluster {0} on {1} completed'.format(config.cluster_id, node.ip))
return service_metadata
def create_cluster(cluster_name, ip, server_port=DEFAULT_SERVER_PORT, client_port=DEFAULT_CLIENT_PORT):
"""
Creates a cluster
:param cluster_name: Name of the cluster
:type cluster_name: str
:param ip: IP address of the first node of the new cluster
:type ip: str
:param server_port: Port to be used by server
:type server_port: int
:param client_port: Port to be used by client
:type client_port: int
:return: None
"""
EtcdInstaller._logger.debug('Creating cluster "{0}" on {1}'.format(cluster_name, ip))
client = SSHClient(ip, username='******')
target_name = 'ovs-etcd-{0}'.format(cluster_name)
if ServiceManager.has_service(target_name, client) and ServiceManager.get_service_status(target_name, client) is True:
EtcdInstaller._logger.info('Service {0} already configured and running'.format(target_name))
return
node_name = System.get_my_machine_id(client)
data_dir = EtcdInstaller.DATA_DIR.format(cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(cluster_name)
abs_paths = [data_dir, wal_dir]
client.dir_delete(abs_paths)
client.dir_create(abs_paths)
client.dir_chmod(abs_paths, 0755, recursive=True)
client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
base_name = 'ovs-etcd'
ServiceManager.add_service(base_name, client,
params={'CLUSTER': cluster_name,
'NODE_ID': node_name,
'DATA_DIR': data_dir,
'WAL_DIR': wal_dir,
'SERVER_URL': EtcdInstaller.SERVER_URL.format(ip, server_port),
'CLIENT_URL': EtcdInstaller.CLIENT_URL.format(ip, client_port),
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1', client_port),
'INITIAL_CLUSTER': '{0}={1}'.format(node_name, EtcdInstaller.SERVER_URL.format(ip, server_port)),
'INITIAL_STATE': 'new',
'INITIAL_PEERS': '-initial-advertise-peer-urls {0}'.format(EtcdInstaller.SERVER_URL.format(ip, server_port))},
target_name=target_name)
EtcdInstaller.start(cluster_name, client)
EtcdInstaller.wait_for_cluster(cluster_name, client, client_port=client_port)
EtcdInstaller._logger.debug('Creating cluster "{0}" on {1} completed'.format(cluster_name, ip))
def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_test():
"""
Validate arakoon archiving on extending a cluster with already existing data
"""
first_sr = GeneralStorageRouter.get_storage_routers()[0]
cluster_name = 'OVS_3671-single-node-cluster'
cluster_basedir = '/var/tmp'
root_client = SSHClient(first_sr, username='******')
for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']:
root_client.dir_create(os.path.dirname(directory))
root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True)
root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True)
files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']),
'/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])]
client = SSHClient(first_sr, username='******')
for filename in files_to_create:
client.dir_create(os.path.dirname(filename))
client.dir_chmod(os.path.dirname(filename), 0755, recursive=True)
client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True)
client.file_create(files_to_create)
for filename in files_to_create:
assert client.file_exists(filename) is True, 'File {0} not present'.format(filename)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate single node cluster')
create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip,
cluster_basedir, filesystem=False)
TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False)
ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata'])
TestArakoon.validate_arakoon_config_files([first_sr], cluster_name)
TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True)
for filename in files_to_create:
assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('remove cluster')
ArakoonInstaller.delete_cluster(cluster_name, first_sr.ip, False)
for filename in files_to_create:
assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
TestArakoon.verify_arakoon_structure(root_client, cluster_name, False, False)
def _deploy(config, filesystem, offline_nodes=None):
"""
Deploys a complete cluster: Distributing the configuration files, creating directories and services
"""
if os.environ.get('RUNNING_UNITTESTS') == 'True':
if filesystem is True:
raise NotImplementedError('At this moment, there is no support for unittesting filesystem backend Arakoon clusters')
ArakoonInstaller._logger.debug('Deploying cluster {0}'.format(config.cluster_id))
if offline_nodes is None:
offline_nodes = []
for node in config.nodes:
if node.ip in offline_nodes:
continue
ArakoonInstaller._logger.debug(' Deploying cluster {0} on {1}'.format(config.cluster_id, node.ip))
root_client = SSHClient(node.ip, username='******')
# Distributes a configuration file to all its nodes
config.write_config(node.ip)
# Create dirs as root because mountpoint /mnt/cache1 is typically owned by root
abs_paths = {node.tlog_dir, node.home} # That's a set
if node.log_sinks.startswith('/'):
abs_paths.add(os.path.dirname(os.path.abspath(node.log_sinks)))
if node.crash_log_sinks.startswith('/'):
abs_paths.add(os.path.dirname(os.path.abspath(node.crash_log_sinks)))
abs_paths = list(abs_paths)
root_client.dir_create(abs_paths)
root_client.dir_chmod(abs_paths, 0755, recursive=True)
root_client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
# Creates services for/on all nodes in the config
if config.filesystem is True:
config_path = config.config_path
else:
config_path = Configuration.get_configuration_path(config.config_path)
base_name = 'ovs-arakoon'
target_name = 'ovs-arakoon-{0}'.format(config.cluster_id)
ServiceManager.add_service(base_name, root_client,
params={'CLUSTER': config.cluster_id,
'NODE_ID': node.name,
'CONFIG_PATH': config_path,
'STARTUP_DEPENDENCY': 'started ovs-watcher-config' if filesystem is False else '(local-filesystems and started networking)'},
target_name=target_name)
ArakoonInstaller._logger.debug(' Deploying cluster {0} on {1} completed'.format(config.cluster_id, node.ip))
def create_cluster(cluster_name, ip):
"""
Creates a cluster
:param base_dir: Base directory that should contain the data
:param ip: IP address of the first node of the new cluster
:param cluster_name: Name of the cluster
"""
logger.debug('Creating cluster "{0}" on {1}'.format(cluster_name, ip))
client = SSHClient(ip, username='******')
node_name = System.get_my_machine_id(client)
data_dir = EtcdInstaller.DATA_DIR.format(EtcdInstaller.DB_DIR, cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(EtcdInstaller.DB_DIR, cluster_name)
abs_paths = [data_dir, wal_dir]
client.dir_delete(abs_paths)
client.dir_create(abs_paths)
client.dir_chmod(abs_paths, 0755, recursive=True)
client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
base_name = 'ovs-etcd'
target_name = 'ovs-etcd-{0}'.format(cluster_name)
ServiceManager.add_service(base_name, client,
params={'CLUSTER': cluster_name,
'NODE_ID': node_name,
'DATA_DIR': data_dir,
'WAL_DIR': wal_dir,
'SERVER_URL': EtcdInstaller.SERVER_URL.format(ip),
'CLIENT_URL': EtcdInstaller.CLIENT_URL.format(ip),
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1'),
'INITIAL_CLUSTER': '{0}={1}'.format(node_name, EtcdInstaller.SERVER_URL.format(ip)),
'INITIAL_STATE': 'new',
'INITIAL_PEERS': '-initial-advertise-peer-urls {0}'.format(EtcdInstaller.SERVER_URL.format(ip))},
target_name=target_name)
EtcdInstaller.start(cluster_name, client)
EtcdInstaller.wait_for_cluster(cluster_name, client)
logger.debug('Creating cluster "{0}" on {1} completed'.format(cluster_name, ip))
def extend_cluster(master_ip, new_ip, cluster_name, server_port=DEFAULT_SERVER_PORT, client_port=DEFAULT_CLIENT_PORT):
"""
Extends a cluster to a given new node
:param master_ip: IP of one of the already existing nodes
:type master_ip: str
:param new_ip: IP address of the node to be added
:type new_ip: str
:param cluster_name: Name of the cluster to be extended
:type cluster_name: str
:param server_port: Port to be used by server
:type server_port: int
:param client_port: Port to be used by client
:type client_port: int
"""
EtcdInstaller._logger.debug('Extending cluster "{0}" from {1} to {2}'.format(cluster_name, master_ip, new_ip))
master_client = SSHClient(master_ip, username='******')
if not EtcdInstaller._is_healty(cluster_name, master_client, client_port=client_port):
raise RuntimeError('Cluster "{0}" unhealthy, aborting extend'.format(cluster_name))
command = ['etcdctl', 'member', 'list']
new_server_url = EtcdInstaller.SERVER_URL.format(new_ip, server_port)
if client_port != EtcdInstaller.DEFAULT_CLIENT_PORT:
command = ['etcdctl', '--peers={0}:{1}'.format(master_ip, client_port), 'member', 'list']
cluster_members = master_client.run(command).splitlines()
for cluster_member in cluster_members:
if new_server_url in cluster_member:
EtcdInstaller._logger.info('Node {0} already member of etcd cluster'.format(new_ip))
return
current_cluster = []
for item in cluster_members:
info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict()
current_cluster.append('{0}={1}'.format(info['name'], info['peer']))
new_client = SSHClient(new_ip, username='******')
node_name = System.get_my_machine_id(new_client)
current_cluster.append('{0}={1}'.format(node_name, new_server_url))
data_dir = EtcdInstaller.DATA_DIR.format(cluster_name)
wal_dir = EtcdInstaller.WAL_DIR.format(cluster_name)
abs_paths = [data_dir, wal_dir]
new_client.dir_delete(abs_paths)
new_client.dir_create(abs_paths)
new_client.dir_chmod(abs_paths, 0755, recursive=True)
new_client.dir_chown(abs_paths, 'ovs', 'ovs', recursive=True)
base_name = 'ovs-etcd'
target_name = 'ovs-etcd-{0}'.format(cluster_name)
EtcdInstaller.stop(cluster_name, new_client) # Stop a possible proxy service
params = ServiceManager.add_service(name=base_name,
client=new_client,
target_name=target_name,
delay_registration=True,
params={'CLUSTER': cluster_name,
'NODE_ID': node_name,
'DATA_DIR': data_dir,
'WAL_DIR': wal_dir,
'SERVER_URL': new_server_url,
'CLIENT_URL': EtcdInstaller.CLIENT_URL.format(new_ip, client_port),
'LOCAL_CLIENT_URL': EtcdInstaller.CLIENT_URL.format('127.0.0.1', client_port),
'INITIAL_CLUSTER': ','.join(current_cluster),
'INITIAL_STATE': 'existing',
'INITIAL_PEERS': ''})
add_command = ['etcdctl', 'member', 'add', node_name, new_server_url]
if client_port != EtcdInstaller.DEFAULT_CLIENT_PORT:
add_command = ['etcdctl', '--peers={0}:{1}'.format(master_ip, client_port), 'member', 'add', node_name, new_server_url]
master_client.run(add_command)
EtcdInstaller.start(cluster_name, new_client)
EtcdInstaller.wait_for_cluster(cluster_name, new_client, client_port=client_port)
ServiceManager.register_service(service_metadata=params, node_name=node_name)
EtcdInstaller._logger.debug('Extending cluster "{0}" from {1} to {2} completed'.format(cluster_name, master_ip, new_ip))
def migrate(previous_version):
"""
Migrates from any version to any version, running all migrations required
If previous_version is for example 0 and this script is at
version 3 it will execute two steps:
- 1 > 2
- 2 > 3
@param previous_version: The previous version from which to start the migration.
"""
working_version = previous_version
# Version 1 introduced:
# - The datastore is still empty, add defaults
if working_version < 1:
from ovs.dal.hybrids.user import User
from ovs.dal.hybrids.group import Group
from ovs.dal.hybrids.role import Role
from ovs.dal.hybrids.client import Client
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.hybrids.j_rolegroup import RoleGroup
from ovs.dal.hybrids.j_roleclient import RoleClient
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.hybrids.branding import Branding
from ovs.dal.lists.backendtypelist import BackendTypeList
# Create groups
admin_group = Group()
admin_group.name = 'administrators'
admin_group.description = 'Administrators'
admin_group.save()
viewers_group = Group()
viewers_group.name = 'viewers'
viewers_group.description = 'Viewers'
viewers_group.save()
# Create users
admin = User()
admin.username = '******'
admin.password = hashlib.sha256('admin').hexdigest()
admin.is_active = True
admin.group = admin_group
admin.save()
# Create internal OAuth 2 clients
admin_pw_client = Client()
admin_pw_client.ovs_type = 'INTERNAL'
admin_pw_client.grant_type = 'PASSWORD'
admin_pw_client.user = admin
admin_pw_client.save()
admin_cc_client = Client()
admin_cc_client.ovs_type = 'INTERNAL'
admin_cc_client.grant_type = 'CLIENT_CREDENTIALS'
admin_cc_client.client_secret = ''.join(random.choice(string.ascii_letters +
string.digits +
'|_=+*#@!/-[]{}<>.?,\'";:~')
for _ in range(128))
admin_cc_client.user = admin
admin_cc_client.save()
# Create roles
read_role = Role()
read_role.code = 'read'
read_role.name = 'Read'
read_role.description = 'Can read objects'
read_role.save()
write_role = Role()
write_role.code = 'write'
write_role.name = 'Write'
write_role.description = 'Can write objects'
write_role.save()
manage_role = Role()
manage_role.code = 'manage'
manage_role.name = 'Manage'
manage_role.description = 'Can manage the system'
manage_role.save()
# Attach groups to roles
mapping = [
(admin_group, [read_role, write_role, manage_role]),
(viewers_group, [read_role])
]
for setting in mapping:
for role in setting[1]:
rolegroup = RoleGroup()
rolegroup.group = setting[0]
rolegroup.role = role
rolegroup.save()
for user in setting[0].users:
for role in setting[1]:
for client in user.clients:
roleclient = RoleClient()
roleclient.client = client
roleclient.role = role
roleclient.save()
# Add backends
for backend_type_info in [('Ceph', 'ceph_s3'), ('Amazon', 'amazon_s3'), ('Swift', 'swift_s3'),
('Local', 'local'), ('Distributed', 'distributed'), ('ALBA', 'alba')]:
code = backend_type_info[1]
backend_type = BackendTypeList.get_backend_type_by_code(code)
if backend_type is None:
backend_type = BackendType()
backend_type.name = backend_type_info[0]
backend_type.code = code
backend_type.save()
# Add service types
for service_type_info in [ServiceType.SERVICE_TYPES.MD_SERVER, ServiceType.SERVICE_TYPES.ALBA_PROXY, ServiceType.SERVICE_TYPES.ARAKOON]:
service_type = ServiceType()
service_type.name = service_type_info
service_type.save()
# Branding
branding = Branding()
branding.name = 'Default'
branding.description = 'Default bootstrap theme'
branding.css = 'bootstrap-default.min.css'
branding.productname = 'Open vStorage'
branding.is_default = True
branding.save()
slate = Branding()
slate.name = 'Slate'
slate.description = 'Dark bootstrap theme'
slate.css = 'bootstrap-slate.min.css'
slate.productname = 'Open vStorage'
slate.is_default = False
slate.save()
# Failure Domain
failure_domain = FailureDomain()
failure_domain.name = 'Default'
failure_domain.save()
# We're now at version 1
working_version = 1
# Version 2 introduced:
# - new Descriptor format
if working_version < 2:
import imp
from ovs.dal.helpers import Descriptor
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
keys = client.prefix('ovs_data')
for key in keys:
data = client.get(key)
modified = False
for entry in data.keys():
if isinstance(data[entry], dict) and 'source' in data[entry] and 'hybrids' in data[entry]['source']:
filename = data[entry]['source']
if not filename.startswith('/'):
filename = '/opt/OpenvStorage/ovs/dal/{0}'.format(filename)
module = imp.load_source(data[entry]['name'], filename)
cls = getattr(module, data[entry]['type'])
new_data = Descriptor(cls, cached=False).descriptor
if 'guid' in data[entry]:
new_data['guid'] = data[entry]['guid']
data[entry] = new_data
modified = True
if modified is True:
data['_version'] += 1
client.set(key, data)
# We're now at version 2
working_version = 2
# Version 3 introduced:
# - new Descriptor format
if working_version < 3:
import imp
from ovs.dal.helpers import Descriptor
from ovs.extensions.storage.persistentfactory import PersistentFactory
client = PersistentFactory.get_client()
keys = client.prefix('ovs_data')
for key in keys:
data = client.get(key)
modified = False
for entry in data.keys():
if isinstance(data[entry], dict) and 'source' in data[entry]:
module = imp.load_source(data[entry]['name'], data[entry]['source'])
cls = getattr(module, data[entry]['type'])
new_data = Descriptor(cls, cached=False).descriptor
if 'guid' in data[entry]:
new_data['guid'] = data[entry]['guid']
data[entry] = new_data
modified = True
if modified is True:
data['_version'] += 1
client.set(key, data)
working_version = 3
# Version 4 introduced:
# - Flexible SSD layout
if working_version < 4:
import os
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.hybrids.j_storagedriverpartition import StorageDriverPartition
from ovs.dal.hybrids.servicetype import ServiceType
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.extensions.generic.remote import remote
from ovs.extensions.generic.sshclient import SSHClient
from ovs.extensions.storageserver.storagedriver import StorageDriverConfiguration
for service in ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.MD_SERVER).services:
mds_service = service.mds_service
storagedriver = None
for current_storagedriver in service.storagerouter.storagedrivers:
if current_storagedriver.vpool_guid == mds_service.vpool_guid:
storagedriver = current_storagedriver
break
tasks = {}
if storagedriver._data.get('mountpoint_md'):
tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_md'),
storagedriver.vpool.name,
mds_service.number)] = (DiskPartition.ROLES.DB, StorageDriverPartition.SUBROLE.MDS)
if storagedriver._data.get('mountpoint_temp'):
tasks['{0}/mds_{1}_{2}'.format(storagedriver._data.get('mountpoint_temp'),
storagedriver.vpool.name,
mds_service.number)] = (DiskPartition.ROLES.SCRUB, StorageDriverPartition.SUBROLE.MDS)
for disk in service.storagerouter.disks:
for partition in disk.partitions:
for directory, (role, subrole) in tasks.iteritems():
with remote(storagedriver.storagerouter.ip, [os], username='******') as rem:
stat_dir = directory
while not rem.os.path.exists(stat_dir) and stat_dir != '/':
stat_dir = stat_dir.rsplit('/', 1)[0]
if not stat_dir:
stat_dir = '/'
inode = rem.os.stat(stat_dir).st_dev
if partition.inode == inode:
if role not in partition.roles:
partition.roles.append(role)
partition.save()
number = 0
migrated = False
for sd_partition in storagedriver.partitions:
if sd_partition.role == role and sd_partition.sub_role == subrole:
if sd_partition.mds_service == mds_service:
migrated = True
break
if sd_partition.partition_guid == partition.guid:
number = max(sd_partition.number, number)
if migrated is False:
sd_partition = StorageDriverPartition()
sd_partition.role = role
sd_partition.sub_role = subrole
sd_partition.partition = partition
sd_partition.storagedriver = storagedriver
sd_partition.mds_service = mds_service
sd_partition.size = None
sd_partition.number = number + 1
sd_partition.save()
client = SSHClient(storagedriver.storagerouter, username='******')
path = sd_partition.path.rsplit('/', 1)[0]
if path:
client.dir_create(path)
client.dir_chown(path, 'ovs', 'ovs')
client.dir_create(directory)
client.dir_chown(directory, 'ovs', 'ovs')
client.symlink({sd_partition.path: directory})
for storagedriver in StorageDriverList.get_storagedrivers():
migrated_objects = {}
for disk in storagedriver.storagerouter.disks:
for partition in disk.partitions:
# Process all mountpoints that are unique and don't have a specified size
for key, (role, sr_info) in {'mountpoint_md': (DiskPartition.ROLES.DB, {'metadata_{0}': StorageDriverPartition.SUBROLE.MD,
'tlogs_{0}': StorageDriverPartition.SUBROLE.TLOG}),
'mountpoint_fragmentcache': (DiskPartition.ROLES.WRITE, {'fcache_{0}': StorageDriverPartition.SUBROLE.FCACHE}),
'mountpoint_foc': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD,
'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}),
'mountpoint_dtl': (DiskPartition.ROLES.WRITE, {'fd_{0}': StorageDriverPartition.SUBROLE.FD,
'dtl_{0}': StorageDriverPartition.SUBROLE.DTL}),
'mountpoint_readcaches': (DiskPartition.ROLES.READ, {'': None}),
'mountpoint_writecaches': (DiskPartition.ROLES.WRITE, {'sco_{0}': StorageDriverPartition.SUBROLE.SCO})}.iteritems():
if key in storagedriver._data:
is_list = isinstance(storagedriver._data[key], list)
entries = storagedriver._data[key][:] if is_list is True else [storagedriver._data[key]]
for entry in entries:
if not entry:
if is_list:
storagedriver._data[key].remove(entry)
if len(storagedriver._data[key]) == 0:
del storagedriver._data[key]
else:
del storagedriver._data[key]
else:
with remote(storagedriver.storagerouter.ip, [os], username='******') as rem:
inode = rem.os.stat(entry).st_dev
if partition.inode == inode:
if role not in partition.roles:
partition.roles.append(role)
partition.save()
for folder, subrole in sr_info.iteritems():
number = 0
migrated = False
for sd_partition in storagedriver.partitions:
if sd_partition.role == role and sd_partition.sub_role == subrole:
if sd_partition.partition_guid == partition.guid:
number = max(sd_partition.number, number)
if migrated is False:
sd_partition = StorageDriverPartition()
sd_partition.role = role
sd_partition.sub_role = subrole
sd_partition.partition = partition
sd_partition.storagedriver = storagedriver
sd_partition.size = None
sd_partition.number = number + 1
sd_partition.save()
if folder:
source = '{0}/{1}'.format(entry, folder.format(storagedriver.vpool.name))
else:
source = entry
client = SSHClient(storagedriver.storagerouter, username='******')
path = sd_partition.path.rsplit('/', 1)[0]
if path:
client.dir_create(path)
client.dir_chown(path, 'ovs', 'ovs')
client.symlink({sd_partition.path: source})
migrated_objects[source] = sd_partition
if is_list:
storagedriver._data[key].remove(entry)
if len(storagedriver._data[key]) == 0:
del storagedriver._data[key]
else:
del storagedriver._data[key]
storagedriver.save()
if 'mountpoint_bfs' in storagedriver._data:
storagedriver.mountpoint_dfs = storagedriver._data['mountpoint_bfs']
if not storagedriver.mountpoint_dfs:
storagedriver.mountpoint_dfs = None
del storagedriver._data['mountpoint_bfs']
storagedriver.save()
if 'mountpoint_temp' in storagedriver._data:
del storagedriver._data['mountpoint_temp']
storagedriver.save()
if migrated_objects:
print 'Loading sizes'
config = StorageDriverConfiguration('storagedriver', storagedriver.vpool_guid, storagedriver.storagedriver_id)
config.load()
for readcache in config.configuration.get('content_addressed_cache', {}).get('clustercache_mount_points', []):
path = readcache.get('path', '').rsplit('/', 1)[0]
size = int(readcache['size'].strip('KiB')) * 1024 if 'size' in readcache else None
if path in migrated_objects:
migrated_objects[path].size = long(size)
migrated_objects[path].save()
for writecache in config.configuration.get('scocache', {}).get('scocache_mount_points', []):
path = writecache.get('path', '')
size = int(writecache['size'].strip('KiB')) * 1024 if 'size' in writecache else None
if path in migrated_objects:
migrated_objects[path].size = long(size)
migrated_objects[path].save()
working_version = 4
# Version 5 introduced:
# - Failure Domains
if working_version < 5:
import os
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.lists.failuredomainlist import FailureDomainList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.extensions.generic.remote import remote
from ovs.extensions.generic.sshclient import SSHClient
failure_domains = FailureDomainList.get_failure_domains()
if len(failure_domains) > 0:
failure_domain = failure_domains[0]
else:
failure_domain = FailureDomain()
failure_domain.name = 'Default'
failure_domain.save()
for storagerouter in StorageRouterList.get_storagerouters():
change = False
if storagerouter.primary_failure_domain is None:
storagerouter.primary_failure_domain = failure_domain
change = True
if storagerouter.rdma_capable is None:
client = SSHClient(storagerouter, username='******')
rdma_capable = False
with remote(client.ip, [os], username='******') as rem:
for root, dirs, files in rem.os.walk('/sys/class/infiniband'):
for directory in dirs:
ports_dir = '/'.join([root, directory, 'ports'])
if not rem.os.path.exists(ports_dir):
continue
for sub_root, sub_dirs, _ in rem.os.walk(ports_dir):
if sub_root != ports_dir:
continue
for sub_directory in sub_dirs:
state_file = '/'.join([sub_root, sub_directory, 'state'])
if rem.os.path.exists(state_file):
if 'ACTIVE' in client.run('cat {0}'.format(state_file)):
rdma_capable = True
storagerouter.rdma_capable = rdma_capable
change = True
if change is True:
storagerouter.save()
working_version = 5
# Version 6 introduced:
# - Distributed scrubbing
if working_version < 6:
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.extensions.generic.sshclient import SSHClient
for storage_driver in StorageDriverList.get_storagedrivers():
root_client = SSHClient(storage_driver.storagerouter, username='******')
for partition in storage_driver.partitions:
if partition.role == DiskPartition.ROLES.SCRUB:
old_path = partition.path
partition.sub_role = None
partition.save()
partition.invalidate_dynamics(['folder', 'path'])
if root_client.dir_exists(partition.path):
continue # New directory already exists
if '_mds_' in old_path:
if root_client.dir_exists(old_path):
root_client.symlink({partition.path: old_path})
if not root_client.dir_exists(partition.path):
root_client.dir_create(partition.path)
root_client.dir_chmod(partition.path, 0777)
working_version = 6
# Version 7 introduced:
# - vPool status
if working_version < 7:
from ovs.dal.hybrids import vpool
reload(vpool)
from ovs.dal.hybrids.vpool import VPool
from ovs.dal.lists.vpoollist import VPoolList
for _vpool in VPoolList.get_vpools():
vpool = VPool(_vpool.guid)
if hasattr(vpool, 'status') and vpool.status is None:
vpool.status = VPool.STATUSES.RUNNING
vpool.save()
working_version = 7
# Version 10 introduced:
# - Reverse indexes are stored in persistent store
# - Store more non-changing metadata on disk iso using a dynamic property
if working_version < 10:
from ovs.dal.helpers import HybridRunner, Descriptor
from ovs.dal.datalist import DataList
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.extensions.storage.volatilefactory import VolatileFactory
persistent = PersistentFactory.get_client()
for prefix in ['ovs_listcache', 'ovs_reverseindex']:
for key in persistent.prefix(prefix):
persistent.delete(key)
for key in persistent.prefix('ovs_data_'):
persistent.set(key, persistent.get(key))
base_reverse_key = 'ovs_reverseindex_{0}_{1}|{2}|{3}'
hybrid_structure = HybridRunner.get_hybrids()
for class_descriptor in hybrid_structure.values():
cls = Descriptor().load(class_descriptor).get_object()
all_objects = DataList(cls, {'type': DataList.where_operator.AND,
'items': []})
for item in all_objects:
guid = item.guid
for relation in item._relations:
if relation.foreign_type is None:
rcls = cls
rclsname = rcls.__name__.lower()
else:
rcls = relation.foreign_type
rclsname = rcls.__name__.lower()
key = relation.name
rguid = item._data[key]['guid']
if rguid is not None:
reverse_key = base_reverse_key.format(rclsname, rguid, relation.foreign_key, guid)
persistent.set(reverse_key, 0)
volatile = VolatileFactory.get_client()
try:
volatile._client.flush_all()
except:
pass
from ovs.dal.lists.vdisklist import VDiskList
for vdisk in VDiskList.get_vdisks():
try:
vdisk.metadata = {'lba_size': vdisk.info['lba_size'],
'cluster_multiplier': vdisk.info['cluster_multiplier']}
vdisk.save()
except:
pass
working_version = 10
# Version 11 introduced:
# - ALBA accelerated ALBA, meaning different vpool.metadata information
if working_version < 11:
from ovs.dal.lists.vpoollist import VPoolList
for vpool in VPoolList.get_vpools():
vpool.metadata = {'backend': vpool.metadata}
if 'metadata' in vpool.metadata['backend']:
vpool.metadata['backend']['arakoon_config'] = vpool.metadata['backend'].pop('metadata')
if 'backend_info' in vpool.metadata['backend']:
vpool.metadata['backend']['backend_info']['fragment_cache_on_read'] = True
vpool.metadata['backend']['backend_info']['fragment_cache_on_write'] = False
vpool.save()
working_version = 11
return working_version
def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_and_extend_test():
"""
Validate arakoon archiving when creating and extending an arakoon cluster
"""
storagerouters = GeneralStorageRouter.get_storage_routers()
storagerouters.sort(key=lambda k: k.ip)
if len(storagerouters) < 2:
TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters)))
return
cluster_name = 'OVS_3671-multi-node-cluster'
cluster_basedir = '/var/tmp'
archived_files = []
files_to_create = []
for index, sr in enumerate(storagerouters):
root_client = SSHClient(sr, username='******')
for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']:
root_client.dir_create(os.path.dirname(directory))
root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True)
root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True)
files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']),
'/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])]
client = SSHClient(sr, username='******')
for filename in files_to_create:
client.dir_create(os.path.dirname(filename))
client.dir_chmod(os.path.dirname(filename), 0755, recursive=True)
client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True)
client.file_create(files_to_create)
for filename in files_to_create:
assert client.file_exists(filename) is True, 'File {0} not present'.format(filename)
archived_files = ['/'.join(['/var/log/arakoon', cluster_name, 'archive', 'one.log'])]
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate single node cluster')
if index == 0:
create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
sr.ip, cluster_basedir, filesystem=False)
TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
ArakoonInstaller.start_cluster(cluster_name, sr.ip, False)
ArakoonInstaller.claim_cluster(cluster_name, sr, False, metadata=create_info['metadata'])
else:
ArakoonInstaller.extend_cluster(storagerouters[0].ip, sr.ip, cluster_name, cluster_basedir)
TestArakoon.validate_arakoon_config_files(storagerouters[:index + 1], cluster_name)
TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True)
TestArakoon.check_archived_directory(client, archived_files)
for filename in files_to_create:
assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('remove cluster')
ArakoonInstaller.delete_cluster(cluster_name, storagerouters[0].ip, False)
for sr in storagerouters:
client = SSHClient(sr, username='******')
TestArakoon.check_archived_directory(client, archived_files)
for filename in files_to_create:
assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)
def execute_scrub_work(queue, vpool, scrub_info, error_messages):
"""
Executes scrub work for a given vDisk queue and vPool, based on scrub_info
:param queue: a Queue with vDisk guids that need to be scrubbed (they should only be member of a single vPool)
:type queue: Queue
:param vpool: the vPool object of the vDisks
:type vpool: VPool
:param scrub_info: A dict containing scrub information: `scrub_path` with the path where to scrub and `storage_router` with the StorageRouter
that needs to do the work
:type scrub_info: dict
:param error_messages: A list of error messages to be filled
:type error_messages: list
:return: a list of error messages
:rtype: list
"""
def _verify_mds_config(current_vdisk):
current_vdisk.invalidate_dynamics('info')
vdisk_configs = current_vdisk.info['metadata_backend_config']
if len(vdisk_configs) == 0:
raise RuntimeError('Could not load MDS configuration')
return vdisk_configs
client = None
lock_time = 5 * 60
storagerouter = scrub_info['storage_router']
scrub_directory = '{0}/scrub_work_{1}_{2}'.format(scrub_info['scrub_path'], vpool.name, storagerouter.name)
scrub_config_key = 'ovs/vpools/{0}/proxies/scrub/scrub_config_{1}'.format(vpool.guid, storagerouter.guid)
backend_config_key = 'ovs/vpools/{0}/proxies/scrub/backend_config_{1}'.format(vpool.guid, storagerouter.guid)
alba_proxy_service = 'ovs-albaproxy_{0}_{1}_scrub'.format(vpool.name, storagerouter.name)
# Deploy a proxy
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Deploying ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_create(scrub_directory)
client.dir_chmod(scrub_directory, 0777) # Celery task executed by 'ovs' user and should be able to write in it
if ServiceManager.has_service(name=alba_proxy_service, client=client) is True and ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Re-using existing proxy service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
scrub_config = Configuration.get(scrub_config_key)
else:
machine_id = System.get_my_machine_id(client)
port_range = Configuration.get('/ovs/framework/hosts/{0}/ports|storagedriver'.format(machine_id))
port = System.get_free_ports(selected_range=port_range, nr=1, client=client)[0]
# Scrub config
# {u'albamgr_cfg_url': u'arakoon://config/ovs/vpools/71e2f717-f270-4a41-bbb0-d4c8c084d43e/proxies/64759516-3471-4321-b912-fb424568fc5b/config/abm?ini=%2Fopt%2FOpenvStorage%2Fconfig%2Farakoon_cacc.ini',
# u'fragment_cache': [u'none'],
# u'ips': [u'127.0.0.1'],
# u'log_level': u'info',
# u'manifest_cache_size': 17179869184,
# u'port': 0,
# u'transport': u'tcp'}
# Backend config
# {u'alba_connection_host': u'10.100.193.155',
# u'alba_connection_port': 26204,
# u'alba_connection_preset': u'preset',
# u'alba_connection_timeout': 15,
# u'alba_connection_transport': u'TCP',
# u'backend_interface_retries_on_error': 5,
# u'backend_interface_retry_backoff_multiplier': 2.0,
# u'backend_interface_retry_interval_secs': 1,
# u'backend_type': u'ALBA'}
scrub_config = Configuration.get('ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(vpool.guid))
scrub_config['port'] = port
scrub_config['transport'] = 'tcp'
Configuration.set(scrub_config_key, json.dumps(scrub_config, indent=4), raw=True)
params = {'VPOOL_NAME': vpool.name,
'LOG_SINK': LogHandler.get_sink_path('alba_proxy'),
'CONFIG_PATH': Configuration.get_configuration_path(scrub_config_key)}
ServiceManager.add_service(name='ovs-albaproxy', params=params, client=client, target_name=alba_proxy_service)
ServiceManager.start_service(name=alba_proxy_service, client=client)
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Deployed ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
backend_config = Configuration.get('ovs/vpools/{0}/hosts/{1}/config'.format(vpool.guid, vpool.storagedrivers[0].storagedriver_id))['backend_connection_manager']
backend_config['alba_connection_host'] = '127.0.0.1'
backend_config['alba_connection_port'] = scrub_config['port']
Configuration.set(backend_config_key, json.dumps({"backend_connection_manager": backend_config}, indent=4), raw=True)
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - An error occurred deploying ALBA proxy {2}'.format(vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
if client is not None and ServiceManager.has_service(name=alba_proxy_service, client=client) is True:
if ServiceManager.get_service_status(name=alba_proxy_service, client=client) is True:
ServiceManager.stop_service(name=alba_proxy_service, client=client)
ServiceManager.remove_service(name=alba_proxy_service, client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
try:
# Empty the queue with vDisks to scrub
with remote(storagerouter.ip, [VDisk]) as rem:
while True:
vdisk = None
vdisk_guid = queue.get(False)
try:
# Check MDS master is local. Trigger MDS handover if necessary
vdisk = rem.VDisk(vdisk_guid)
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Started scrubbing at location {3}'.format(vpool.name, storagerouter.name, vdisk.name, scrub_directory))
configs = _verify_mds_config(current_vdisk=vdisk)
storagedriver = StorageDriverList.get_by_storagedriver_id(vdisk.storagedriver_id)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - MDS master is not local, trigger handover'.format(vpool.name, storagerouter.name, vdisk.name))
MDSServiceController.ensure_safety(VDisk(vdisk_guid)) # Do not use a remote VDisk instance here
configs = _verify_mds_config(current_vdisk=vdisk)
if configs[0].get('ip') != storagedriver.storagerouter.ip:
ScheduledTaskController._logger.warning('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Skipping because master MDS still not local'.format(vpool.name, storagerouter.name, vdisk.name))
continue
# Do the actual scrubbing
with vdisk.storagedriver_client.make_locked_client(str(vdisk.volume_id)) as locked_client:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Retrieve and apply scrub work'.format(vpool.name, storagerouter.name, vdisk.name))
work_units = locked_client.get_scrubbing_workunits()
for work_unit in work_units:
res = locked_client.scrub(work_unit=work_unit,
scratch_dir=scrub_directory,
log_sinks=[LogHandler.get_sink_path('scrubber', allow_override=True)],
backend_config=Configuration.get_configuration_path(backend_config_key))
locked_client.apply_scrubbing_result(scrubbing_work_result=res)
if work_units:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - {3} work units successfully applied'.format(vpool.name, storagerouter.name, vdisk.name, len(work_units)))
else:
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - No scrubbing required'.format(vpool.name, storagerouter.name, vdisk.name))
except Exception:
if vdisk is None:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk with guid {2} could not be found'.format(vpool.name, storagerouter.name, vdisk_guid)
else:
message = 'Scrubber - vPool {0} - StorageRouter {1} - vDisk {2} - Scrubbing failed'.format(vpool.name, storagerouter.name, vdisk.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
except Empty: # Raised when all items have been fetched from the queue
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Queue completely processed'.format(vpool.name, storagerouter.name))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Scrubbing failed'.format(vpool.name, storagerouter.name)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
# Delete the proxy again
try:
with file_mutex(name='ovs_albaproxy_scrub', wait=lock_time):
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Removing service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
client = SSHClient(storagerouter, 'root')
client.dir_delete(scrub_directory)
if ServiceManager.has_service(alba_proxy_service, client=client):
ServiceManager.stop_service(alba_proxy_service, client=client)
ServiceManager.remove_service(alba_proxy_service, client=client)
if Configuration.exists(scrub_config_key):
Configuration.delete(scrub_config_key)
ScheduledTaskController._logger.info('Scrubber - vPool {0} - StorageRouter {1} - Removed service {2}'.format(vpool.name, storagerouter.name, alba_proxy_service))
except Exception:
message = 'Scrubber - vPool {0} - StorageRouter {1} - Removing service {2} failed'.format(vpool.name, storagerouter.name, alba_proxy_service)
error_messages.append(message)
ScheduledTaskController._logger.exception(message)
