def _features(self):
"""
Returns information about installed/available features
:return: Dictionary containing edition and available features per component
"""
try:
enterprise = 'enterprise'
community = 'community'
client = SSHClient(self, username='******')
enterprise_regex = re.compile('^(?P<edition>ee-)?(?P<version>.*)$')
version = client.run(
"volumedriver_fs --version | grep version: | awk '{print $2}'",
allow_insecure=True,
allow_nonzero=True)
volumedriver_version = enterprise_regex.match(version).groupdict()
volumedriver_edition = enterprise if volumedriver_version[
'edition'] == 'ee-' else community
volumedriver_version_lv = LooseVersion(
volumedriver_version['version'])
volumedriver_features = [
feature for feature, version in {
'directory_unlink': ('6.15.0', None)
}.iteritems()
if volumedriver_version_lv >= LooseVersion(version[0]) and (
version[1] is None or version[1] == volumedriver_edition)
]
version = client.run("alba version --terse",
allow_insecure=True,
allow_nonzero=True)
alba_version = enterprise_regex.match(version).groupdict()
alba_edition = enterprise if alba_version[
'edition'] == 'ee-' else community
alba_version_lv = LooseVersion(alba_version['version'])
alba_features = [
feature for feature, version in {
'cache-quota': ('1.4.4', enterprise),
'block-cache': ('1.4.0', enterprise)
}.iteritems()
if alba_version_lv >= LooseVersion(version[0]) and (
version[1] is None or version[1] == alba_edition)
]
return {
'volumedriver': {
'edition': volumedriver_edition,
'features': volumedriver_features
},
'alba': {
'edition': alba_edition,
'features': alba_features
}
}
except UnableToConnectException:
pass
except Exception:
StorageRouter._logger.exception(
'Could not load feature information')
return None
def add_extra_node(**kwargs):
ip = kwargs['cluster_ip']
license_contents = []
for lic in LicenseList.get_licenses():
license_contents.append(lic.hash)
client = SSHClient(ip)
client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents)))
def get_current_memory_usage_of_process(storagerouter_ip, pid):
"""
get residential memory usage of a certain storagerouter (through /proc/<PID>/status)
VmPeak: 8110620 kB
VmSize: 3252752 kB
VmLck: 0 kB
VmPin: 0 kB
VmHWM: 4959820 kB
VmRSS: 570764 kB
VmData: 3019468 kB
VmStk: 136 kB
VmExe: 12464 kB
VmLib: 58852 kB
VmPTE: 2644 kB
VmPMD: 24 kB
VmSwap: 394224 kB
:param storagerouter_ip: ip address of a existing storagerouter
:type storagerouter_ip: str
:param pid: process ID of the process you want to monitor
:type pid: int
:return: current usage
:rtype: str
"""
client = SSHClient(storagerouter_ip, username='******')
return client.run("grep Vm /proc/{0}/status | tr -s ' '".format(pid),
allow_insecure=True)
def create_cluster(cluster_name, ip, exclude_ports, plugins=None):
"""
Creates a cluster
"""
logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip))
client = SSHClient(ip)
base_dir = client.config_read('ovs.arakoon.location').rstrip('/')
port_range = client.config_read('ovs.ports.arakoon')
ports = System.get_free_ports(port_range, exclude_ports, 2, client)
node_name = System.get_my_machine_id(client)
config = ArakoonClusterConfig(cluster_name, plugins)
if not [node.name for node in config.nodes if node.name == node_name]:
config.nodes.append(
ArakoonNodeConfig(
name=node_name,
ip=ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=ArakoonInstaller.ARAKOON_LOG_DIR.format(
cluster_name),
home=ArakoonInstaller.ARAKOON_HOME_DIR.format(
base_dir, cluster_name),
tlog_dir=ArakoonInstaller.ARAKOON_TLOG_DIR.format(
base_dir, cluster_name)))
ArakoonInstaller._deploy(config)
logger.debug('Creating cluster {0} on {1} completed'.format(
cluster_name, ip))
return {'client_port': ports[0], 'messaging_port': ports[1]}
def setup():
"""
Setup for Arakoon package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
assert backend_name, 'Please fill out a backend name in the autotest.cfg file'
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is True:
GeneralService.stop_service(name='ovs-scheduled-tasks',
client=root_client)
storagerouters = GeneralStorageRouter.get_storage_routers()
for sr in storagerouters:
root_client = SSHClient(sr, username='******')
GeneralDisk.add_db_role(sr)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
GeneralAlba.add_alba_backend(backend_name)
GeneralArakoon.voldrv_arakoon_checkup()
def system_services_check_test():
"""
Verify some system services
"""
services_to_commands = {
"nginx": """ps -efx|grep nginx|grep -v grep""",
"rabbitmq-server": """ps -ef|grep rabbitmq-|grep -v grep""",
"memcached": """ps -ef|grep memcached|grep -v grep""",
"ovs-arakoon-ovsdb": """initctl list| grep ovsdb""",
"ovs-snmp": """initctl list| grep ovs-snmp""",
"ovs-support-agent": """initctl list| grep support""",
"ovs-volumerouter-consumer": """initctl list| grep volumerou""",
"ovs-watcher-framework": """initctl list| grep watcher-fr"""
}
errors = ''
services_checked = 'Following services found running:\n'
grid_ip = General.get_config().get('main', 'grid_ip')
ssh_pass = General.get_config().get('mgmtcenter', 'password')
client = SSHClient(grid_ip, username='******', password=ssh_pass)
for service_to_check in services_to_commands.iterkeys():
out, err = client.run(services_to_commands[service_to_check], debug=True)
if len(err):
errors += "Error when trying to run {0}:\n{1}".format(services_to_commands[service_to_check], err)
else:
if len(out):
services_checked += "{0}\n".format(service_to_check)
else:
errors += "Couldn't find {0} running process\n".format(service_to_check)
print services_checked
assert len(errors) == 0, "Found the following errors while checking for the system services:{0}\n".format(errors)
def collapse_arakoon():
"""
Collapse Arakoon's Tlogs
:return: None
"""
logger.info('Starting arakoon collapse')
arakoon_clusters = {}
for service in ServiceList.get_services():
if service.type.name in ('Arakoon', 'NamespaceManager', 'AlbaManager'):
arakoon_clusters[service.name.replace('arakoon-', '')] = service.storagerouter
for cluster, storagerouter in arakoon_clusters.iteritems():
logger.info(' Collapsing cluster {0}'.format(cluster))
client = SSHClient(storagerouter)
parser = client.rawconfig_read(PyrakoonStore.ARAKOON_CONFIG_FILE.format(cluster))
nodes = {}
for node in parser.get('global', 'cluster').split(','):
node = node.strip()
nodes[node] = ([parser.get(node, 'ip')], parser.get(node, 'client_port'))
config = ArakoonClientConfig(str(cluster), nodes)
for node in nodes.keys():
logger.info(' Collapsing node: {0}'.format(node))
client = ArakoonAdminClient(node, config)
try:
client.collapse_tlogs(2)
except:
logger.exception('Error during collapsing cluster {0} node {1}'.format(cluster, node))
logger.info('Arakoon collapse finished')
def test_single_node(self):
node = sorted(TestArakoonInstaller.nodes.keys())[0]
result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, node, '/tmp/db')
contents = SSHClient(node).file_read(TestArakoonInstaller.cluster_config_file)
expected = TestArakoonInstaller.expected_global.format(TestArakoonInstaller.nodes[node], TestArakoonInstaller.cluster_name)
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node)
self.assertEqual(contents.strip(), expected.strip())
def deploy_to_slave(master_ip, slave_ip, cluster_name):
"""
Deploys the configuration file to a slave
:param master_ip: IP of the node to deploy from
:type master_ip: str
:param slave_ip: IP of the slave to deploy to
:type slave_ip: str
:param cluster_name: Name of the cluster of which to deploy the configuration file
:type cluster_name: str
:return: None
"""
EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2}'.format(cluster_name, master_ip, slave_ip))
master_client = SSHClient(master_ip, username='******')
slave_client = SSHClient(slave_ip, username='******')
current_cluster = []
for item in master_client.run('etcdctl member list').splitlines():
info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict()
current_cluster.append('{0}={1}'.format(info['name'], info['peer']))
EtcdInstaller._setup_proxy(','.join(current_cluster), slave_client, cluster_name, force=True)
EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2} completed'.format(cluster_name, master_ip, slave_ip))
def shrink_cluster(remaining_node_ip, deleted_node_ip, cluster_name):
"""
Removes a node from a cluster, the old node will become a slave
:param cluster_name: The name of the cluster to shrink
:param deleted_node_ip: The ip of the node that should be deleted
:param remaining_node_ip: The ip of a remaining node
"""
logger.debug('Shrinking cluster "{0}" from {1}'.format(cluster_name, deleted_node_ip))
current_client = SSHClient(remaining_node_ip, username='******')
if not EtcdInstaller._is_healty(cluster_name, current_client):
raise RuntimeError('Cluster "{0}" unhealthy, aborting shrink'.format(cluster_name))
old_client = SSHClient(deleted_node_ip, username='******')
node_name = System.get_my_machine_id(old_client)
node_id = None
for item in current_client.run('etcdctl member list').splitlines():
info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict()
if info['name'] == node_name:
node_id = info['id']
if node_id is None:
raise RuntimeError('Could not locate {0} in the cluster'.format(deleted_node_ip))
current_client.run('etcdctl member remove {0}'.format(node_id))
EtcdInstaller.deploy_to_slave(remaining_node_ip, deleted_node_ip, cluster_name)
EtcdInstaller.wait_for_cluster(cluster_name, current_client)
logger.debug('Shrinking cluster "{0}" from {1} completed'.format(cluster_name, deleted_node_ip))
def create_cluster(cluster_name, ip, exclude_ports, base_dir, plugins=None):
"""
Creates a cluster
"""
logger.debug("Creating cluster {0} on {1}".format(cluster_name, ip))
client = SSHClient(ip)
base_dir = base_dir.rstrip("/")
port_range = client.config_read("ovs.ports.arakoon")
ports = System.get_free_ports(port_range, exclude_ports, 2, client)
node_name = System.get_my_machine_id(client)
config = ArakoonClusterConfig(cluster_name, plugins)
if not [node.name for node in config.nodes if node.name == node_name]:
config.nodes.append(
ArakoonNodeConfig(
name=node_name,
ip=ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name),
home=ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name),
tlog_dir=ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name),
)
)
ArakoonInstaller._deploy(config)
logger.debug("Creating cluster {0} on {1} completed".format(cluster_name, ip))
return {"client_port": ports[0], "messaging_port": ports[1]}
def delete_volume(vdisk, vpool, loop_device=None, root_client=None, wait=True):
"""
Delete a volume
:param vdisk: Virtual disk to delete
:param vpool: vPool which hosts the Virtual Disk
:param loop_device: Loop device where volume is mounted on
:param root_client: SSHClient object
:param wait: Wait for the volume to be deleted from model
:return: None
"""
location = GeneralVDisk.get_filesystem_location(vpool=vpool,
vdisk_name=vdisk.name)
if root_client is None:
root_client = SSHClient('127.0.0.1', username='******')
if loop_device is not None:
GeneralVDisk.disconnect_volume(loop_device, root_client)
root_client.file_delete(location)
if wait is True:
counter = 0
timeout = 60
volume_name = '/' + os.path.basename(location)
while True and counter < timeout:
time.sleep(1)
vdisks = VDiskList.get_by_devicename_and_vpool(volume_name, vpool)
if vdisks is None:
break
counter += 1
if counter == timeout:
raise RuntimeError('Disk {0} was not deleted from model after {1} seconds'.format(volume_name, timeout))
def setUp(cls):
for node in TestArakoonInstaller.nodes:
client = SSHClient(node)
client.dir_delete('/tmp/db')
client.dir_delete('/tmp/cfg')
client.dir_create('/tmp/db')
client.dir_create('/tmp/cfg')
def test_single_node(self):
node = sorted(TestArakoonInstaller.nodes.keys())[0]
result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, node, '/tmp/db')
contents = SSHClient(node).file_read(TestArakoonInstaller.cluster_config_file)
expected = TestArakoonInstaller.expected_global.format(TestArakoonInstaller.nodes[node], TestArakoonInstaller.cluster_name)
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node)
self.assertEqual(contents.strip(), expected.strip())
def deploy_to_slave(master_ip, slave_ip, cluster_name, client_port=DEFAULT_CLIENT_PORT):
"""
Deploys the configuration file to a slave
:param master_ip: IP of the node to deploy from
:type master_ip: str
:param slave_ip: IP of the slave to deploy to
:type slave_ip: str
:param cluster_name: Name of the cluster of which to deploy the configuration file
:type cluster_name: str
:param client_port: Port to be used by client
:type client_port: int
:return: None
"""
EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2}'.format(cluster_name, master_ip, slave_ip))
master_client = SSHClient(master_ip, username='******')
slave_client = SSHClient(slave_ip, username='******')
current_cluster = []
list_command = ['etcdctl', 'member', 'list']
if client_port != EtcdInstaller.DEFAULT_CLIENT_PORT:
list_command = ['etcdctl', '--peers={0}:{1}'.format(master_ip, client_port), 'member', 'list']
for item in master_client.run(list_command).splitlines():
info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict()
current_cluster.append('{0}={1}'.format(info['name'], info['peer']))
EtcdInstaller._setup_proxy(','.join(current_cluster), slave_client, cluster_name, force=True)
EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2} completed'.format(cluster_name, master_ip, slave_ip))
def config_files_check_test():
"""
Verify some configuration files
"""
issues_found = ''
config_keys = {
"/ovs/framework/memcache",
"/ovs/arakoon/ovsdb/config"
}
for key_to_check in config_keys:
if not Configuration.exists(key_to_check, raw=True):
issues_found += "Couldn't find {0}\n".format(key_to_check)
config_files = {
"rabbitmq.config": "/etc/rabbitmq/rabbitmq.config",
}
grid_ip = General.get_config().get('main', 'grid_ip')
ssh_pass = General.get_config().get('mgmtcenter', 'password')
client = SSHClient(grid_ip, username='******', password=ssh_pass)
for config_file_to_check in config_files.iterkeys():
if not client.file_exists(config_files[config_file_to_check]):
issues_found += "Couldn't find {0}\n".format(config_file_to_check)
assert issues_found == '',\
"Found the following issues while checking for the config files:{0}\n".format(issues_found)
def teardown():
"""
Teardown for Arakoon package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is False:
GeneralService.start_service(name='ovs-scheduled-tasks',
client=root_client)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
for key in KEY_CLEANUP:
if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True):
EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def shrink_vpool(storage_driver):
"""
Remove a Storage Driver from a vPool
:param storage_driver: Storage Driver to remove from the vPool
:return: None
"""
vpool = storage_driver.vpool
if GeneralHypervisor.get_hypervisor_type() == "VMWARE":
root_client = SSHClient(storage_driver.storagerouter, username="******")
if storage_driver.mountpoint in General.get_mountpoints(root_client):
root_client.run(["umount", "storage_driver.mountpoint"])
task_result = GeneralVPool.api.execute_post_action(
component="vpools",
guid=vpool.guid,
action="shrink_vpool",
data={"storagerouter_guid": storage_driver.storagerouter.guid},
wait=True,
timeout=GeneralVPool.TIMEOUT_ADD_VPOOL,
)
if task_result[0] is not True:
raise RuntimeError(
"Storage Driver with ID {0} was not successfully removed from vPool {1}".format(
storage_driver.storagedriver_id, vpool.name
),
task_result,
)
return GeneralVPool.get_vpool_by_name(vpool_name=vpool.name)
def restart_cluster_add(cluster_name, current_ips, new_ip):
"""
Execute a (re)start sequence after adding a new node to a cluster.
"""
logger.debug('Restart sequence (add) for {0}'.format(cluster_name))
logger.debug('Current ips: {0}'.format(', '.join(current_ips)))
logger.debug('New ip: {0}'.format(new_ip))
logger.debug('Catching up new node {0} for cluster {1}'.format(
new_ip, cluster_name))
with Remote(new_ip, [ArakoonManagementEx], 'ovs') as remote:
cluster = remote.ArakoonManagementEx().getCluster(cluster_name)
cluster.catchup_node()
logger.debug(
'Catching up new node {0} for cluster {1} completed'.format(
new_ip, cluster_name))
threshold = 2 if new_ip in current_ips else 1
for ip in current_ips:
if ip == new_ip:
continue
client = SSHClient(ip, username='******')
ArakoonInstaller.stop(cluster_name, client=client)
ArakoonInstaller.start(cluster_name, client=client)
logger.debug(' Restarted node {0} for cluster {1}'.format(
client.ip, cluster_name))
if len(current_ips
) > threshold: # A two node cluster needs all nodes running
ArakoonInstaller.wait_for_cluster(cluster_name)
new_client = SSHClient(new_ip, username='******')
ArakoonInstaller.start(cluster_name, client=new_client)
ArakoonInstaller.wait_for_cluster(cluster_name)
logger.debug('Started node {0} for cluster {1}'.format(
new_ip, cluster_name))
def system_services_check_test():
"""
Verify some system services
"""
services_to_commands = {
"nginx": """ps -efx|grep nginx|grep -v grep""",
"rabbitmq-server": """ps -ef|grep rabbitmq-|grep -v grep""",
"memcached": """ps -ef|grep memcached|grep -v grep""",
}
errors = ''
services_checked = 'Following services found running:\n'
grid_ip = General.get_config().get('main', 'grid_ip')
ssh_pass = General.get_config().get('mgmtcenter', 'password')
client = SSHClient(grid_ip, username='******', password=ssh_pass)
for service_to_check in services_to_commands.iterkeys():
out, err = client.run(services_to_commands[service_to_check], debug=True, allow_insecure=True,
return_stderr=True)
if len(err):
errors += "Error when trying to run {0}:\n{1}".format(services_to_commands[service_to_check], err)
else:
if len(out):
services_checked += "{0}\n".format(service_to_check)
else:
errors += "Couldn't find {0} running process\n".format(service_to_check)
for non_running_service in GeneralSystem.list_non_running_ovs_services(grid_ip):
errors += str(non_running_service)
assert len(errors) == 0,\
"Found the following errors while checking for the system services:{0}\n".format(errors)
def set_rdma_capability(storagerouter_guid):
"""
Check if the StorageRouter has been reconfigured to be able to support RDMA
:param storagerouter_guid: Guid of the StorageRouter to check and set
:type storagerouter_guid: str
:return: None
:rtype: NoneType
"""
storagerouter = StorageRouter(storagerouter_guid)
client = SSHClient(storagerouter, username='******')
rdma_capable = False
with remote(client.ip, [os], username='******') as rem:
for root, dirs, files in rem.os.walk('/sys/class/infiniband'):
for directory in dirs:
ports_dir = '/'.join([root, directory, 'ports'])
if not rem.os.path.exists(ports_dir):
continue
for sub_root, sub_dirs, _ in rem.os.walk(ports_dir):
if sub_root != ports_dir:
continue
for sub_directory in sub_dirs:
state_file = '/'.join(
[sub_root, sub_directory, 'state'])
if rem.os.path.exists(state_file):
if 'ACTIVE' in client.run(['cat', state_file]):
rdma_capable = True
storagerouter.rdma_capable = rdma_capable
storagerouter.save()
def shrink_cluster(remaining_node_ip, ip_to_remove, cluster_name, offline_node_ips=None):
"""
Removes a node from a cluster, the old node will become a slave
:param cluster_name: The name of the cluster to shrink
:param ip_to_remove: The ip of the node that should be removed from the cluster
:param remaining_node_ip: The ip of a remaining node in the cluster
:param offline_node_ips: IPs of offline nodes
"""
logger.debug('Shrinking cluster "{0}" from {1}'.format(cluster_name, ip_to_remove))
current_client = SSHClient(remaining_node_ip, username='******')
if not EtcdInstaller._is_healty(cluster_name, current_client):
raise RuntimeError('Cluster "{0}" unhealthy, aborting shrink'.format(cluster_name))
node_id = None
for item in current_client.run('etcdctl member list').splitlines():
info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict()
if EtcdInstaller.CLIENT_URL.format(ip_to_remove) == info['client']:
node_id = info['id']
if node_id is not None:
current_client.run('etcdctl member remove {0}'.format(node_id))
if ip_to_remove not in offline_node_ips:
EtcdInstaller.deploy_to_slave(remaining_node_ip, ip_to_remove, cluster_name, force=True)
EtcdInstaller.wait_for_cluster(cluster_name, current_client)
logger.debug('Shrinking cluster "{0}" from {1} completed'.format(cluster_name, ip_to_remove))
def __init__(self, host='127.0.0.1', login='******', passwd=None):
logger.debug('Init libvirt')
self.states = {
libvirt.VIR_DOMAIN_NOSTATE: 'NO STATE',
libvirt.VIR_DOMAIN_RUNNING: 'RUNNING',
libvirt.VIR_DOMAIN_BLOCKED: 'BLOCKED',
libvirt.VIR_DOMAIN_PAUSED: 'PAUSED',
libvirt.VIR_DOMAIN_SHUTDOWN: 'SHUTDOWN',
libvirt.VIR_DOMAIN_SHUTOFF: 'TURNEDOFF',
libvirt.VIR_DOMAIN_CRASHED: 'CRASHED'
}
pattern = re.compile(
r"^(?<!\S)((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\b|\.\b){7}(?!\S)$")
if pattern.match(host):
self.host = host
else:
raise ValueError("{0} is not a valid ip.".format(host))
self.host = host
self.login = login
self.streams = {}
self.ssh_client = SSHClient(host, username=login, password=passwd)
self._conn = self.connect(login, host)
# Enable event registering
libvirt.virEventRegisterDefaultImpl()
logger.debug('Init complete')
def restart_cluster(cluster_name, master_ip):
"""
Execute a restart sequence (Executed after arakoon and/or alba package upgrade)
"""
logger.debug('Restart sequence for {0} via {1}'.format(
cluster_name, master_ip))
client = SSHClient(master_ip)
config = ArakoonClusterConfig(cluster_name)
config.load_config(client)
all_clients = [
SSHClient(node.ip) for node in config.nodes if node.ip != master_ip
] + [client]
if len(config.nodes) <= 2:
logger.debug(
' Insufficient nodes in cluster {0}. Full restart'.format(
cluster_name))
for function in [ArakoonInstaller.stop, ArakoonInstaller.start]:
for client in all_clients:
function(cluster_name, client)
ArakoonInstaller.wait_for_cluster(cluster_name)
else:
logger.debug(
' Sufficient nodes in cluster {0}. Sequential restart'.format(
cluster_name))
for client in all_clients:
ArakoonInstaller.stop(cluster_name, client)
ArakoonInstaller.start(cluster_name, client)
logger.debug(' Restarted node {0} on cluster {1}'.format(
client.ip, cluster_name))
ArakoonInstaller.wait_for_cluster(cluster_name)
logger.debug('Restart sequence for {0} via {1} completed'.format(
cluster_name, master_ip))
def has_cluster(ip, cluster_name):
logger.debug('Checking whether {0} has cluster "{1}" running'.format(ip, cluster_name))
client = SSHClient(ip, username='******')
try:
return client.run('etcdctl member list').strip() != ''
except CalledProcessError:
return False
def write_to_volume(vdisk=None, vpool=None, location=None, count=1024, bs='1M', input_type='random',
root_client=None):
"""
Write some data to a file
:param vdisk: Virtual disk to write on
:param vpool: vPool which hosts the Virtual Disk
:param location: Absolute path to file
:param count: amount of blocks to write
:param bs: Size of the blocks to write
:param input_type: Type of input (null, zero, random)
:param root_client: SSHClient object
:return: None
"""
if location is None and (vdisk is None or vpool is None):
raise ValueError('vDisk and vPool must be provided if no location has been provided')
if location is None:
location = GeneralVDisk.get_filesystem_location(vpool=vpool,
vdisk_name=vdisk.name)
if root_client is None:
root_client = SSHClient('127.0.0.1', username='******')
if input_type not in ('null', 'zero', 'random'):
raise ValueError('Invalid input type provided')
if General.check_file_is_link(location, root_client.ip, root_client.username, root_client.password):
print "Writing to {0}".format(root_client.file_read_link(location))
else:
if not root_client.file_exists(location):
raise ValueError('File {0} does not exist on Storage Router {1}'.format(location, root_client.ip))
if not isinstance(count, int) or count < 1:
raise ValueError('Count must be an integer > 0')
root_client.run('dd conv=notrunc if=/dev/{0} of={1} bs={2} count={3}'.format(input_type, location, bs, count))
def unpartition_disk(disk, partitions=None, wait=True):
"""
Return disk to RAW state
:param disk: Disk DAL object
:param partitions: Partitions DAL object list
:return: None
"""
if partitions is None:
partitions = disk.partitions
else:
for partition in partitions:
if partition not in disk.partitions:
raise RuntimeError('Partition {0} does not belong to disk {1}'.format(partition.mountpoint, disk.name))
if len(disk.partitions) == 0:
return
root_client = SSHClient(disk.storagerouter, username='******')
for partition in partitions:
General.unmount_partition(root_client, partition)
root_client.run(['parted', '-s', '/dev/' + disk.name, 'mklabel', 'gpt'])
GeneralStorageRouter.sync_with_reality(disk.storagerouter)
counter = 0
timeout = 60
while counter < timeout:
time.sleep(1)
disk = GeneralDisk.get_disk(guid=disk.guid)
if len(disk.partitions) == 0:
break
counter += 1
if counter == timeout:
raise RuntimeError('Removing partitions failed for disk:\n {0} '.format(disk.name))
def extend_cluster(master_ip, new_ip, cluster_name, exclude_ports, base_dir):
"""
Extends a cluster to a given new node
"""
logger.debug("Extending cluster {0} from {1} to {2}".format(cluster_name, master_ip, new_ip))
client = SSHClient(master_ip)
config = ArakoonClusterConfig(cluster_name)
config.load_config(client)
client = SSHClient(new_ip)
base_dir = base_dir.rstrip("/")
port_range = client.config_read("ovs.ports.arakoon")
ports = System.get_free_ports(port_range, exclude_ports, 2, client)
node_name = System.get_my_machine_id(client)
if not [node.name for node in config.nodes if node.name == node_name]:
config.nodes.append(
ArakoonNodeConfig(
name=node_name,
ip=new_ip,
client_port=ports[0],
messaging_port=ports[1],
log_dir=ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name),
home=ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name),
tlog_dir=ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name),
)
)
ArakoonInstaller._deploy(config)
logger.debug("Extending cluster {0} from {1} to {2} completed".format(cluster_name, master_ip, new_ip))
return {"client_port": ports[0], "messaging_port": ports[1]}
def _clean():
volatile = VolatileFactory.get_client()
persistent = PersistentFactory.get_client()
# noinspection PyProtectedMember
volatile._clean()
# noinspection PyProtectedMember
persistent._clean()
# noinspection PyProtectedMember
SSHClient._clean()
# noinspection PyProtectedMember
SystemdMock._clean()
# noinspection PyProtectedMember
MDSClient._clean()
# noinspection PyProtectedMember
Decorators._clean()
# noinspection PyProtectedMember
MockedSSHClient._clean()
# noinspection PyProtectedMember
StorageRouterClient._clean()
Logger._logs = {}
DataList._test_hooks = {}
Toolbox._function_pointers = {}
# Clean underlying persistent store
Configuration.get_client()._clean()
for file_name in glob.glob(
ArakoonClusterConfig.CONFIG_FILE.format('unittest*')):
os.remove(file_name)
for full_path in glob.glob(DalHelper.UNITTEST_DIR.format('*')):
shutil.rmtree(full_path)
return volatile, persistent
def setUp(cls):
for node in TestArakoonInstaller.nodes:
client = SSHClient(node)
root_client = SSHClient(node, username='******')
root_client.dir_delete('/tmp/db')
root_client.dir_create('/tmp/db')
client.dir_delete(TestArakoonInstaller.cluster_config_path)
client.dir_create(TestArakoonInstaller.cluster_config_path)
def add_extra_node(**kwargs):
ip = kwargs['cluster_ip']
license_contents = []
for lic in LicenseList.get_licenses():
license_contents.append(lic.hash)
client = SSHClient(ip)
client.file_write('/opt/OpenvStorage/config/licenses',
'{0}\n'.format('\n'.join(license_contents)))
def is_datastore_available(self, mountpoint):
"""
Check if datastore is available
:param mountpoint: Mountpoint of the NFS datastore
"""
if self.ssh_client is None:
self.ssh_client = SSHClient(self.host, username='******')
return self.ssh_client.run("[ -d {0} ] && echo 'yes' || echo 'no'".format(mountpoint)) == 'yes'
def migrate(previous_version, master_ips=None, extra_ips=None):
"""
Migrates from any version to any version, running all migrations required
If previous_version is for example 0 and this script is at
verison 3 it will execute two steps:
- 1 > 2
- 2 > 3
:param previous_version: The previous version from which to start the migration.
:param master_ips: IP addresses of the MASTER nodes
:param extra_ips: IP addresses of the EXTRA nodes
"""
working_version = previous_version
# Version 1 introduced:
# - Flexible SSD layout
if working_version < 1:
from ovs.extensions.generic.configuration import Configuration
if Configuration.exists('ovs.arakoon'):
Configuration.delete('ovs.arakoon', remove_root=True)
Configuration.set('ovs.core.ovsdb', '/opt/OpenvStorage/db')
working_version = 1
# Version 2 introduced:
# - Registration
if working_version < 2:
import time
from ovs.extensions.generic.configuration import Configuration
if not Configuration.exists('ovs.core.registered'):
Configuration.set('ovs.core.registered', False)
Configuration.set('ovs.core.install_time', time.time())
working_version = 2
# Version 3 introduced:
# - New arakoon clients
if working_version < 3:
from ovs.extensions.db.arakoon import ArakoonInstaller
reload(ArakoonInstaller)
from ovs.extensions.db.arakoon import ArakoonInstaller
from ovs.extensions.generic.sshclient import SSHClient
from ovs.extensions.generic.configuration import Configuration
if master_ips is not None:
for ip in master_ips:
client = SSHClient(ip)
if client.dir_exists(ArakoonInstaller.ArakoonInstaller.ARAKOON_CONFIG_DIR):
for cluster_name in client.dir_list(ArakoonInstaller.ArakoonInstaller.ARAKOON_CONFIG_DIR):
try:
ArakoonInstaller.ArakoonInstaller.deploy_cluster(cluster_name, ip)
except:
pass
if Configuration.exists('ovs.core.storage.persistent'):
Configuration.set('ovs.core.storage.persistent', 'pyrakoon')
working_version = 3
return working_version
def remove_from_slave(master_ip, slave_ip, cluster_name):
"""
Removes everything related to a given cluster from the slave
"""
client = SSHClient(master_ip)
config = ArakoonClusterConfig(cluster_name)
config.load_config(client)
client = SSHClient(slave_ip)
config.delete_config(client)
def is_devstack_installed():
"""
Check if OpenStack or DevStack is installed
:return: True if installed
"""
client = SSHClient('127.0.0.1', username='******')
is_openstack = ServiceManager.has_service(OSManager.get_openstack_cinder_service_name(), client)
is_devstack = 'stack' in str(client.run('ps aux | grep SCREEN | grep stack | grep -v grep || true'), allow_insecure=True)
return is_openstack or is_devstack
def remove_mds_service(mds_service, vpool, reload_config):
"""
Removes an MDS service
:param mds_service: The MDS service to remove
:param vpool: The vPool for which the MDS service will be removed
:param reload_config: If True, the volumedriver's updated configuration will be reloaded
"""
if len(mds_service.vdisks_guids) > 0:
raise RuntimeError('Cannot remove MDSService that is still serving disks')
storagerouter = mds_service.service.storagerouter
client = SSHClient(storagerouter)
mdsservice_type = ServiceTypeList.get_by_name('MetadataServer')
# Clean up model
directories_to_clean = []
for sd_partition in mds_service.storagedriver_partitions:
directories_to_clean.append(sd_partition.path)
sd_partition.delete()
mds_service.delete()
mds_service.service.delete()
# Generate new mds_nodes section
mds_nodes = []
for service in mdsservice_type.services:
if service.storagerouter_guid == storagerouter.guid:
mds_service = service.mds_service
if mds_service.vpool_guid == vpool.guid:
sdp = [sd_partition.path for sd_partition in mds_service.storagedriver_partitions if sd_partition.role == DiskPartition.ROLES.DB]
mds_nodes.append({'host': service.storagerouter.ip,
'port': service.ports[0],
'db_directory': sdp[0],
'scratch_directory': sdp[0]})
# Generate the correct section in the Storage Driver's configuration
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name)
storagedriver_config.load(client)
storagedriver_config.clean() # Clean out obsolete values
storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes)
storagedriver_config.save(client, reload_config=reload_config)
tries = 5
while tries > 0:
try:
root_client = SSHClient(storagerouter, username='******')
root_client.dir_delete(directories=directories_to_clean,
follow_symlinks=True)
for dir_name in directories_to_clean:
logger.debug('Recursively removed {0}'.format(dir_name))
break
except Exception:
time.sleep(5)
logger.debug('Waiting for the MDS service to go down...')
tries -= 1
if tries == 0:
raise
def test_single_node(self):
base_port = Configuration.get('ovs.ports.arakoon')[0]
cluster = 'one'
node = sorted(TestArakoonInstaller.nodes.keys())[0]
ArakoonInstaller.create_cluster(cluster, node, [])
contents = SSHClient(node).file_read(self._get_config_path(cluster))
expected = TestArakoonInstaller.expected_global.format(cluster, TestArakoonInstaller.nodes[node])
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
self.assertEqual(contents.strip(), expected.strip())
def deploy_to_slave(master_ip, slave_ip, cluster_name):
"""
Deploys the configuration file to a slave
"""
client = SSHClient(master_ip)
config = ArakoonClusterConfig(cluster_name)
config.load_config(client)
client = SSHClient(slave_ip)
config.write_config(client)
def get_mds_storagedriver_config_set(vpool, check_online=False):
"""
Builds a configuration for all StorageRouters from a given VPool with following goals:
* Primary MDS is the local one
* All slaves are on different hosts
* Maximum `mds.safety` nodes are returned
The configuration returned is the default configuration used by the volumedriver of which in normal use-cases
only the 1st entry is used, because at volume creation time, the volumedriver needs to create 1 master MDS
During ensure_safety, we actually create/set the MDS slaves for each volume
:param vpool: vPool to get storagedriver configuration for
:param check_online: Check whether the storage routers are actually responsive
"""
mds_per_storagerouter = {}
mds_per_load = {}
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
if check_online is True:
try:
client = SSHClient(storagerouter)
client.run('pwd')
except UnableToConnectException:
continue
mds_service, load = MDSServiceController.get_preferred_mds(storagerouter, vpool, include_load=True)
mds_per_storagerouter[storagerouter] = {'host': storagerouter.ip, 'port': mds_service.service.ports[0]}
if load not in mds_per_load:
mds_per_load[load] = []
mds_per_load[load].append(storagerouter)
safety = EtcdConfiguration.get('/ovs/framework/storagedriver|mds_safety')
config_set = {}
for storagerouter, ip_info in mds_per_storagerouter.iteritems():
primary_failure_domain = storagerouter.primary_failure_domain
secondary_failure_domain = storagerouter.secondary_failure_domain
config_set[storagerouter.guid] = [ip_info]
for load in sorted(mds_per_load):
if len(config_set[storagerouter.guid]) >= safety:
break
other_storagerouters = mds_per_load[load]
random.shuffle(other_storagerouters)
for other_storagerouter in other_storagerouters:
if len(config_set[storagerouter.guid]) >= safety:
break
if other_storagerouter != storagerouter and other_storagerouter in primary_failure_domain.primary_storagerouters:
config_set[storagerouter.guid].append(mds_per_storagerouter[other_storagerouter])
if secondary_failure_domain is not None:
for load in sorted(mds_per_load):
if len(config_set[storagerouter.guid]) >= safety:
break
other_storagerouters = mds_per_load[load]
random.shuffle(other_storagerouters)
for other_storagerouter in other_storagerouters:
if len(config_set[storagerouter.guid]) >= safety:
break
if other_storagerouter != storagerouter and other_storagerouter in secondary_failure_domain.primary_storagerouters:
config_set[storagerouter.guid].append(mds_per_storagerouter[other_storagerouter])
return config_set
def get_configuration(vpool_guid):
"""
Retrieve the running storagedriver configuration for the vPool
:param vpool_guid: Guid of the vPool to retrieve running configuration for
:return: Dictionary with configuration
"""
vpool = VPool(vpool_guid)
if not vpool.storagedrivers or not vpool.storagedrivers[0].storagerouter:
return {}
client = None
for sd in vpool.storagedrivers:
try:
client = SSHClient(sd.storagerouter)
client.run("pwd")
break
except UnableToConnectException:
client = None
pass
if client is None:
raise RuntimeError("Could not find an online storage router to retrieve vPool configuration from")
storagedriver_config = StorageDriverConfiguration("storagedriver", vpool.name)
storagedriver_config.load(client)
dtl = storagedriver_config.configuration.get("failovercache", {})
file_system = storagedriver_config.configuration.get("filesystem", {})
volume_router = storagedriver_config.configuration.get("volume_router", {})
volume_manager = storagedriver_config.configuration.get("volume_manager", {})
dtl_mode = file_system.get("fs_dtl_mode", StorageDriverClient.VOLDRV_DTL_ASYNC)
dedupe_mode = volume_manager.get("read_cache_default_mode", StorageDriverClient.VOLDRV_CONTENT_BASED)
dtl_transport = dtl.get("failovercache_transport", StorageDriverClient.VOLDRV_DTL_TRANSPORT_TCP)
cache_strategy = volume_manager.get("read_cache_default_behaviour", StorageDriverClient.VOLDRV_CACHE_ON_READ)
sco_multiplier = volume_router.get("vrouter_sco_multiplier", 1024)
dtl_config_mode = file_system.get("fs_dtl_config_mode", StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE)
tlog_multiplier = volume_manager.get("number_of_scos_in_tlog", 20)
non_disposable_sco_factor = volume_manager.get("non_disposable_scos_factor", 12)
sco_size = sco_multiplier * 4 / 1024 # SCO size is in MiB ==> SCO multiplier * cluster size (4 KiB by default)
write_buffer = tlog_multiplier * sco_size * non_disposable_sco_factor
dtl_mode = StorageDriverClient.REVERSE_DTL_MODE_MAP[dtl_mode]
dtl_enabled = dtl_config_mode == StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE
if dtl_enabled is False:
dtl_mode = StorageDriverClient.FRAMEWORK_DTL_NO_SYNC
return {
"sco_size": sco_size,
"dtl_mode": dtl_mode,
"dedupe_mode": StorageDriverClient.REVERSE_DEDUPE_MAP[dedupe_mode],
"dtl_enabled": dtl_enabled,
"write_buffer": write_buffer,
"dtl_transport": StorageDriverClient.REVERSE_DTL_TRANSPORT_MAP[dtl_transport],
"cache_strategy": StorageDriverClient.REVERSE_CACHE_MAP[cache_strategy],
"tlog_multiplier": tlog_multiplier,
}
def get_configuration(vpool_guid):
"""
Retrieve the running storagedriver configuration for the vPool
:param vpool_guid: Guid of the vPool to retrieve running configuration for
:return: Dictionary with configuration
"""
vpool = VPool(vpool_guid)
if not vpool.storagedrivers or not vpool.storagedrivers[0].storagerouter:
return {}
client = None
for sd in vpool.storagedrivers:
try:
client = SSHClient(sd.storagerouter)
client.run('pwd')
break
except UnableToConnectException:
client = None
pass
if client is None:
raise RuntimeError('Could not find an online storage router to retrieve vPool configuration from')
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.guid, vpool.storagedrivers[0].storagedriver_id)
storagedriver_config.load()
dtl = storagedriver_config.configuration.get('distributed_transaction_log', {})
file_system = storagedriver_config.configuration.get('filesystem', {})
volume_router = storagedriver_config.configuration.get('volume_router', {})
volume_manager = storagedriver_config.configuration.get('volume_manager', {})
dtl_mode = file_system.get('fs_dtl_mode', StorageDriverClient.VOLDRV_DTL_ASYNC)
dedupe_mode = volume_manager.get('read_cache_default_mode', StorageDriverClient.VOLDRV_CONTENT_BASED)
cluster_size = volume_manager.get('default_cluster_size', 4096) / 1024
dtl_transport = dtl.get('dtl_transport', StorageDriverClient.VOLDRV_DTL_TRANSPORT_TCP)
cache_strategy = volume_manager.get('read_cache_default_behaviour', StorageDriverClient.VOLDRV_CACHE_ON_READ)
sco_multiplier = volume_router.get('vrouter_sco_multiplier', 1024)
dtl_config_mode = file_system.get('fs_dtl_config_mode', StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE)
tlog_multiplier = volume_manager.get('number_of_scos_in_tlog', 20)
non_disposable_sco_factor = volume_manager.get('non_disposable_scos_factor', 12)
sco_size = sco_multiplier * cluster_size / 1024 # SCO size is in MiB ==> SCO multiplier * cluster size (4 KiB by default)
write_buffer = tlog_multiplier * sco_size * non_disposable_sco_factor
dtl_mode = StorageDriverClient.REVERSE_DTL_MODE_MAP[dtl_mode]
dtl_enabled = dtl_config_mode == StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE
if dtl_enabled is False:
dtl_mode = StorageDriverClient.FRAMEWORK_DTL_NO_SYNC
return {'sco_size': sco_size,
'dtl_mode': dtl_mode,
'dedupe_mode': StorageDriverClient.REVERSE_DEDUPE_MAP[dedupe_mode],
'dtl_enabled': dtl_enabled,
'cluster_size': cluster_size,
'write_buffer': write_buffer,
'dtl_transport': StorageDriverClient.REVERSE_DTL_TRANSPORT_MAP[dtl_transport],
'cache_strategy': StorageDriverClient.REVERSE_CACHE_MAP[cache_strategy],
'tlog_multiplier': tlog_multiplier}
def run_test_fuse(cls,
storagedriver,
disk_amount,
write_amount,
logger=LOGGER):
"""
Deploy and run a small io test using the FUSE interface
:param storagedriver: chosen storagedriver for testing
:param disk_amount: amount of disks to deploy and write/read to
:param write_amount: amount of data to parse for writing/reading
:param logger: logging instance
:return:
"""
vpool = storagedriver.vpool
client = SSHClient(storagedriver.storagerouter, username='******')
vdisk_info = {}
try:
for vdisk_number in xrange(disk_amount):
vdisk_name = '{0}{1}-fuse'.format(cls.PREFIX, vdisk_number)
disk_location = "/mnt/{0}/{1}.raw".format(
vpool.name, vdisk_name)
logger.info("Truncating vdisk {0} on {1}:{2}".format(
vdisk_name, storagedriver.storage_ip, vpool.name))
client.run(
["truncate", "-s",
str(cls.VDISK_SIZE), disk_location])
vdisk = cls._get_vdisk('{0}.raw'.format(vdisk_name),
vpool.name)
vdisk_info[disk_location] = vdisk
fio_configuration = {
'io_size': write_amount,
'configuration': (0, 100)
}
DataWriter.write_data_fio(client,
fio_configuration,
file_locations=vdisk_info.keys(),
screen=False,
loop_screen=False)
fio_configuration = {
'io_size': write_amount,
'configuration': (100, 0)
}
DataWriter.write_data_fio(client,
fio_configuration,
file_locations=vdisk_info.keys(),
screen=False,
loop_screen=False)
except Exception as ex:
logger.error(
'An exception occur while testing edge+blktap: {0}'.format(
str(ex)))
raise
finally:
for vdisk in vdisk_info.values():
VDiskRemover.remove_vdisk_by_name(vdisk.devicename,
vdisk.vpool.name)
def execute_command_on_node(host, command, password=None):
"""
Execute a command on a specific host
:param host: Host to execute command on
:param command: Command to execute
:param password: Password used to login on host
:return: Output of command
"""
cl = SSHClient(host, username='******', password=password)
return cl.run(command)
def update_components(components):
"""
Initiate the update through commandline for all StorageRouters
This is called upon by the API
:return: None
"""
components = [component.strip() for component in components]
root_client = SSHClient(endpoint=System.get_my_storagerouter(),
username='******')
root_client.run(['ovs', 'update', ','.join(components)])
def collapse_arakoon():
"""
Collapse Arakoon's Tlogs
:return: None
"""
from ovs_extensions.generic.toolbox import ExtensionsToolbox
GenericController._logger.info('Arakoon collapse started')
cluster_info = []
storagerouters = StorageRouterList.get_storagerouters()
if os.environ.get('RUNNING_UNITTESTS') != 'True':
cluster_info = [('cacc', storagerouters[0])]
cluster_names = []
for service in ServiceList.get_services():
if service.is_internal is True and service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON,
ServiceType.SERVICE_TYPES.NS_MGR,
ServiceType.SERVICE_TYPES.ALBA_MGR):
cluster = ExtensionsToolbox.remove_prefix(service.name, 'arakoon-')
if cluster in cluster_names and cluster not in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST]:
continue
cluster_names.append(cluster)
cluster_info.append((cluster, service.storagerouter))
workload = {}
cluster_config_map = {}
for cluster, storagerouter in cluster_info:
GenericController._logger.debug(' Collecting info for cluster {0}'.format(cluster))
ip = storagerouter.ip if cluster in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST] else None
try:
config = ArakoonClusterConfig(cluster_id=cluster, source_ip=ip)
cluster_config_map[cluster] = config
except:
GenericController._logger.exception(' Retrieving cluster information on {0} for {1} failed'.format(storagerouter.ip, cluster))
continue
for node in config.nodes:
if node.ip not in workload:
workload[node.ip] = {'node_id': node.name,
'clusters': []}
workload[node.ip]['clusters'].append((cluster, ip))
for storagerouter in storagerouters:
try:
if storagerouter.ip not in workload:
continue
node_workload = workload[storagerouter.ip]
client = SSHClient(storagerouter)
for cluster, ip in node_workload['clusters']:
try:
GenericController._logger.debug(' Collapsing cluster {0} on {1}'.format(cluster, storagerouter.ip))
client.run(['arakoon', '--collapse-local', node_workload['node_id'], '2', '-config', cluster_config_map[cluster].external_config_path])
GenericController._logger.debug(' Collapsing cluster {0} on {1} completed'.format(cluster, storagerouter.ip))
except:
GenericController._logger.exception(' Collapsing cluster {0} on {1} failed'.format(cluster, storagerouter.ip))
except UnableToConnectException:
GenericController._logger.error(' Could not collapse any cluster on {0} (not reachable)'.format(storagerouter.name))
GenericController._logger.info('Arakoon collapse finished')
def delete_volume(vdisk, vpool, loop_device=None, root_client=None, wait=True):
"""
Delete a volume
:param vdisk: Virtual disk to delete
:param vpool: vPool which hosts the Virtual Disk
:param loop_device: Loop device where volume is mounted on
:param root_client: SSHClient object
:param wait: Wait for the volume to be deleted from model
:return: None
"""
location = GeneralVDisk.get_filesystem_location(vpool=vpool,
vdisk_name=vdisk.name)
if root_client is None:
root_client = SSHClient('127.0.0.1', username='******')
if loop_device is not None:
root_client.run('umount /dev/{0}'.format(loop_device))
root_client.run('losetup -d /dev/{0}'.format(loop_device))
root_client.dir_delete('/mnt/{0}'.format(loop_device))
root_client.file_delete(location)
if wait is True:
counter = 0
timeout = 60
volume_name = os.path.basename(location).replace('-flat.vmdk', '').replace('.raw', '')
while True and counter < timeout:
time.sleep(1)
vdisks = GeneralVDisk.get_vdisk_by_name(name=volume_name)
if vdisks is None:
break
counter += 1
if counter == timeout:
raise RuntimeError('Disk {0} was not deleted from model after {1} seconds'.format(volume_name, timeout))
def get_configuration(vpool_guid):
"""
Retrieve the running storagedriver configuration for the vPool
:param vpool_guid: Guid of the vPool to retrieve running configuration for
:return: Dictionary with configuration
"""
vpool = VPool(vpool_guid)
if not vpool.storagedrivers or not vpool.storagedrivers[0].storagerouter:
return {}
client = None
for sd in vpool.storagedrivers:
try:
client = SSHClient(sd.storagerouter)
client.run('pwd')
break
except UnableToConnectException:
client = None
pass
if client is None:
raise RuntimeError('Could not find an online storage router to retrieve vPool configuration from')
storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.guid, vpool.storagedrivers[0].storagedriver_id)
storagedriver_config.load()
dtl = storagedriver_config.configuration.get('failovercache', {})
file_system = storagedriver_config.configuration.get('filesystem', {})
volume_router = storagedriver_config.configuration.get('volume_router', {})
volume_manager = storagedriver_config.configuration.get('volume_manager', {})
dtl_mode = file_system.get('fs_dtl_mode', StorageDriverClient.VOLDRV_DTL_ASYNC)
dedupe_mode = volume_manager.get('read_cache_default_mode', StorageDriverClient.VOLDRV_CONTENT_BASED)
dtl_transport = dtl.get('failovercache_transport', StorageDriverClient.VOLDRV_DTL_TRANSPORT_TCP)
cache_strategy = volume_manager.get('read_cache_default_behaviour', StorageDriverClient.VOLDRV_CACHE_ON_READ)
sco_multiplier = volume_router.get('vrouter_sco_multiplier', 1024)
dtl_config_mode = file_system.get('fs_dtl_config_mode', StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE)
tlog_multiplier = volume_manager.get('number_of_scos_in_tlog', 20)
non_disposable_sco_factor = volume_manager.get('non_disposable_scos_factor', 12)
sco_size = sco_multiplier * 4 / 1024 # SCO size is in MiB ==> SCO multiplier * cluster size (4 KiB by default)
write_buffer = tlog_multiplier * sco_size * non_disposable_sco_factor
dtl_mode = StorageDriverClient.REVERSE_DTL_MODE_MAP[dtl_mode]
dtl_enabled = dtl_config_mode == StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE
if dtl_enabled is False:
dtl_mode = StorageDriverClient.FRAMEWORK_DTL_NO_SYNC
return {'sco_size': sco_size,
'dtl_mode': dtl_mode,
'dedupe_mode': StorageDriverClient.REVERSE_DEDUPE_MAP[dedupe_mode],
'dtl_enabled': dtl_enabled,
'write_buffer': write_buffer,
'dtl_transport': StorageDriverClient.REVERSE_DTL_TRANSPORT_MAP[dtl_transport],
'cache_strategy': StorageDriverClient.REVERSE_CACHE_MAP[cache_strategy],
'tlog_multiplier': tlog_multiplier}
def validate(*args, **kwargs):
# check if alba backend exists or not
if kwargs['cluster_basedir'] and kwargs['storagerouter_ip']:
client = SSHClient(kwargs['storagerouter_ip'], username='******')
if client.dir_exists(kwargs['cluster_basedir']):
return func(*args, **kwargs)
else:
raise DirectoryNotFoundError("Required base_dir `{0}` not found on storagerouter `{1}`"
.format(kwargs['cluster_basedir'], kwargs['storagerouter_ip']))
else:
raise AttributeError("Missing parameter(s): cluster_basedir or storagerouter_ip")
def add_extra_node(**kwargs):
"""
Add extra node hook
:param kwargs: Extra parameters
:return: None
"""
ip = kwargs['cluster_ip']
license_contents = []
for lic in LicenseList.get_licenses():
license_contents.append(lic.hash)
client = SSHClient(ip)
client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents)))
def mountpoint_exists(name, storagerouter_guid):
"""
Checks whether a given mount point for a vPool exists
:param name: Name of the mount point to check
:type name: str
:param storagerouter_guid: Guid of the StorageRouter on which to check for mount point existence
:type storagerouter_guid: str
:return: True if mount point not in use else False
:rtype: bool
"""
client = SSHClient(StorageRouter(storagerouter_guid))
return client.dir_exists(directory='/mnt/{0}'.format(name))
def install_plugins():
"""
(Re)load plugins
"""
if ServiceManager.has_service('ovs-watcher-framework',
SSHClient('127.0.0.1', username='******')):
# If the watcher is running, 'ovs setup' was executed and we need to restart everything to load
# the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed
from ovs.dal.lists.storagerouterlist import StorageRouterList
clients = []
try:
for storagerouter in StorageRouterList.get_storagerouters():
clients.append(SSHClient(storagerouter, username='******'))
except UnableToConnectException:
raise RuntimeError('Not all StorageRouters are reachable')
for client in clients:
for service_name in ['watcher-framework', 'memcached']:
ServiceManager.stop_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(
service_name, client=client) is False:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError(
'Could not stop service: {0}'.format(service_name))
for client in clients:
for service_name in ['memcached', 'watcher-framework']:
ServiceManager.start_service(service_name, client=client)
wait = 30
while wait > 0:
if ServiceManager.get_service_status(
service_name, client=client) is True:
break
time.sleep(1)
wait -= 1
if wait == 0:
raise RuntimeError(
'Could not start service: {0}'.format(
service_name))
from ovs.dal.helpers import Migration
Migration.migrate()
from ovs.lib.helpers.toolbox import Toolbox
ip = System.get_my_storagerouter().ip
functions = Toolbox.fetch_hooks('plugin', 'postinstall')
for function in functions:
function(ip=ip)
def restart_cluster(cluster_name, master_ip, filesystem):
"""
Execute a restart sequence (Executed after arakoon and/or alba package upgrade)
:param cluster_name: Name of the cluster to restart
:type cluster_name: str
:param master_ip: IP of one of the cluster nodes
:type master_ip: str
:param filesystem: Indicates whether the configuration should be on the filesystem or in a configuration cluster
:type filesystem: bool
:return: None
"""
ArakoonInstaller._logger.debug(
'Restart sequence for {0} via {1}'.format(cluster_name, master_ip))
config = ArakoonClusterConfig(cluster_name, filesystem)
config.load_config(master_ip)
arakoon_client = ArakoonInstaller.build_client(config)
root_client = SSHClient(master_ip, username='******')
all_clients = [
SSHClient(node.ip, username='******') for node in config.nodes
]
if len(config.nodes) <= 2:
ArakoonInstaller._logger.debug(
' Insufficient nodes in cluster {0}. Full restart'.format(
cluster_name))
for function in [ArakoonInstaller.stop, ArakoonInstaller.start]:
for client in all_clients:
function(cluster_name, client)
ArakoonInstaller.wait_for_cluster(cluster_name, master_ip,
filesystem)
else:
ArakoonInstaller._logger.debug(
' Sufficient nodes in cluster {0}. Sequential restart'.format(
cluster_name))
for client in all_clients:
ArakoonInstaller.stop(cluster_name, client)
ArakoonInstaller.start(cluster_name, client)
ArakoonInstaller._logger.debug(
' Restarted node {0} on cluster {1}'.format(
client.ip, cluster_name))
ArakoonInstaller.wait_for_cluster(cluster_name, master_ip,
filesystem)
ArakoonInstaller.start(cluster_name, root_client)
ArakoonInstaller.wait_for_cluster(cluster_name, master_ip, filesystem)
arakoon_client.set(ArakoonInstaller.INTERNAL_CONFIG_KEY,
config.export_ini())
ArakoonInstaller._logger.debug(
'Restart sequence for {0} via {1} completed'.format(
cluster_name, master_ip))
def restart(cls, storagerouters, logger=LOGGER):
"""
Restarts a list of storagerouters
:param storagerouters: list of storagerouters
:param logger: logging instance
:return: None
"""
for storagerouter in storagerouters:
logger.info("Restarting ovs-workers on {0}".format(
storagerouter.ip))
client = SSHClient(str(storagerouter.ip),
username='******',
cached=False)
client.run(['systemctl', 'restart', 'ovs-workers.service'])
def adjust_for_reroute(storagerouter, start_port=None, end_port=None, trigger_rerout=True, ip_to_block=None, additional_ports=None):
"""
Force edge to reroute. Done by blocking all connections to the volumedriver port
:param storagerouter: storagerouter object of the node to execute the reroute on
:type storagerouter: ovs.dal.hybrids.storagerouter.StorageRouter
:param start_port: port to start blocking
:type start_port: int/str
:param end_port: port to end blocking
:type end_port: int/str
:param trigger_rerout: trigger or unblock the reroute
:type trigger_rerout: bool
:param ip_to_block: ip to block connections on
:type ip_to_block: str
:param additional_ports: additional ports to block outside of the range
:type additional_ports: list[int] / list[str]
"""
if (start_port is None or end_port is None) and ip_to_block is None and additional_ports is None:
raise ValueError('Something to block is required. Be it a range, extra ports or an IP')
ports = []
if start_port is not None and end_port is not None:
if 22 in xrange(int(end_port), int(start_port)): # Avoid blocking ssh
raise ValueError('Port 22 cannot be included in the ports.')
ports.append("{0}:{1}".format(start_port, end_port))
if isinstance(additional_ports, list):
ports.extend([str(item) for item in additional_ports]) # Cast to str
cmds = {}
if trigger_rerout is True:
cmds['input'] = ["iptables", "-I", "INPUT", "1"] # Insert because first rule applies first.
cmds['output'] = ["iptables", "-I", "OUTPUT", "1"]
else:
cmds['input'] = ["iptables", "-D", "INPUT"]
cmds['output'] = ["iptables", "-D", "OUTPUT"]
EdgeTester.LOGGER.debug('Opening {}'.format(','.join(ports)))
if isinstance(ip_to_block, str) or isinstance(ip_to_block, unicode):
ip_extra = ['--source', ip_to_block]
cmds['input'].extend(ip_extra)
cmds['output'].extend(ip_extra)
protocol_rule = ["--protocol", "tcp"]
cmds['input'].extend(protocol_rule)
cmds['output'].extend(protocol_rule)
if len(ports) > 0:
cmds['input'].extend(["--match", "multiport", "--dport", ','.join(ports)])
cmds['output'].extend(["--match", "multiport", "--sport", ','.join(ports)])
action = ["-j", "DROP"]
cmds['input'].extend(action)
cmds['output'].extend(action)
client = SSHClient(storagerouter, username='******')
for rule_key, cmd in cmds.iteritems():
EdgeTester.LOGGER.debug('Executing {0} on {1}'.format(storagerouter.ip, ' '.join(cmd)))
client.run(cmd)
def get_logfiles(albanode_guid, local_storagerouter_guid):
"""
Collects logs, moves them to a web-accessible location and returns log tgz's filename
:param albanode_guid: Alba Node guid to retrieve log files on
:type albanode_guid: str
:param local_storagerouter_guid: Guid of the StorageRouter on which the collect logs was initiated, eg: through the GUI
:type local_storagerouter_guid: str
:return: Name of tgz containing the logs
:rtype: str
"""
web_path = '/opt/OpenvStorage/webapps/frontend/downloads'
alba_node = AlbaNode(albanode_guid)
logfile_name = alba_node.client.get_logs()['filename']
download_url = 'https://{0}:{1}@{2}:{3}/downloads/{4}'.format(
alba_node.username, alba_node.password, alba_node.ip,
alba_node.port, logfile_name)
client = SSHClient(endpoint=StorageRouter(local_storagerouter_guid),
username='******')
client.dir_create(web_path)
client.run([
'wget', download_url, '--directory-prefix', web_path,
'--no-check-certificate'
])
client.run(['chmod', '666', '{0}/{1}'.format(web_path, logfile_name)])
return logfile_name
def configure_vpool_for_host(self, vpool_guid, ip):
if (
self._is_devstack is False and self._is_openstack is False
) or self._cinder_installed is False or self._nova_installed is False:
self._logger.warning(
'Configure vPool: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed'
)
return
vpool = VPool(vpool_guid)
self._logger.debug('configure_vpool {0} started'.format(vpool.name))
# 1. Configure Cinder driver
with remote(ip, [RawConfigParser, open], 'root') as rem:
changed = False
cfg = rem.RawConfigParser()
cfg.read([self._CINDER_CONF])
if not cfg.has_section(vpool.name):
changed = True
cfg.add_section(vpool.name)
cfg.set(vpool.name, "volume_driver",
"cinder.volume.drivers.openvstorage.OVSVolumeDriver")
cfg.set(vpool.name, "volume_backend_name", vpool.name)
cfg.set(vpool.name, "vpool_name", vpool.name)
enabled_backends = []
if cfg.has_option("DEFAULT", "enabled_backends"):
enabled_backends = cfg.get("DEFAULT",
"enabled_backends").split(", ")
if vpool.name not in enabled_backends:
changed = True
enabled_backends.append(vpool.name)
cfg.set("DEFAULT", "enabled_backends",
", ".join(enabled_backends))
if changed is True:
with rem.open(self._CINDER_CONF, "w") as fp:
cfg.write(fp)
# 2. Create volume type
if self.cinder_client and not [
volume_type
for volume_type in self.cinder_client.volume_types.list()
if volume_type.name == vpool.name
]:
volume_type = self.cinder_client.volume_types.create(vpool.name)
volume_type.set_keys(metadata={'volume_backend_name': vpool.name})
# 3. Restart processes
self.client = SSHClient(ip, username='******')
self._restart_processes()
self._logger.debug('configure_vpool {0} completed'.format(vpool.name))
def get_current_memory_usage(storagerouter_ip):
"""
get residential memory usage of a certain storagerouter (through free -m)
:param storagerouter_ip: ip address of a existing storagerouter
:type storagerouter_ip: str
:return: (current usage, max. total usage)
:rtype: tuple
"""
client = SSHClient(storagerouter_ip, username='******')
result = client.run(
"MEM=$(free -m | tr -s ' ' | grep Mem); "
"echo $MEM | cut -d ' ' -f 3; echo $MEM | cut -d ' ' -f 2",
allow_insecure=True).split()
return int(result[0]), int(result[1])
def _umount(mountpoint, client=None):
"""
Unmount the given partition
:param mountpoint: Location where the mountpoint is mounted
:type mountpoint: str
:return:
"""
if client is None:
client = SSHClient(System.get_my_storagerouter(), username='******')
try:
client.run(['umount', mountpoint])
except Exception:
RoleRemover.LOGGER.exception(
'Unable to umount mountpoint {0}'.format(mountpoint))
raise RuntimeError('Could not unmount {0}'.format(mountpoint))
