def _features(self): """ Returns information about installed/available features :return: Dictionary containing edition and available features per component """ try: enterprise = 'enterprise' community = 'community' client = SSHClient(self, username='******') enterprise_regex = re.compile('^(?P<edition>ee-)?(?P<version>.*)$') version = client.run( "volumedriver_fs --version | grep version: | awk '{print $2}'", allow_insecure=True, allow_nonzero=True) volumedriver_version = enterprise_regex.match(version).groupdict() volumedriver_edition = enterprise if volumedriver_version[ 'edition'] == 'ee-' else community volumedriver_version_lv = LooseVersion( volumedriver_version['version']) volumedriver_features = [ feature for feature, version in { 'directory_unlink': ('6.15.0', None) }.iteritems() if volumedriver_version_lv >= LooseVersion(version[0]) and ( version[1] is None or version[1] == volumedriver_edition) ] version = client.run("alba version --terse", allow_insecure=True, allow_nonzero=True) alba_version = enterprise_regex.match(version).groupdict() alba_edition = enterprise if alba_version[ 'edition'] == 'ee-' else community alba_version_lv = LooseVersion(alba_version['version']) alba_features = [ feature for feature, version in { 'cache-quota': ('1.4.4', enterprise), 'block-cache': ('1.4.0', enterprise) }.iteritems() if alba_version_lv >= LooseVersion(version[0]) and ( version[1] is None or version[1] == alba_edition) ] return { 'volumedriver': { 'edition': volumedriver_edition, 'features': volumedriver_features }, 'alba': { 'edition': alba_edition, 'features': alba_features } } except UnableToConnectException: pass except Exception: StorageRouter._logger.exception( 'Could not load feature information') return None
def add_extra_node(**kwargs): ip = kwargs['cluster_ip'] license_contents = [] for lic in LicenseList.get_licenses(): license_contents.append(lic.hash) client = SSHClient(ip) client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents)))
def get_current_memory_usage_of_process(storagerouter_ip, pid): """ get residential memory usage of a certain storagerouter (through /proc/<PID>/status) VmPeak: 8110620 kB VmSize: 3252752 kB VmLck: 0 kB VmPin: 0 kB VmHWM: 4959820 kB VmRSS: 570764 kB VmData: 3019468 kB VmStk: 136 kB VmExe: 12464 kB VmLib: 58852 kB VmPTE: 2644 kB VmPMD: 24 kB VmSwap: 394224 kB :param storagerouter_ip: ip address of a existing storagerouter :type storagerouter_ip: str :param pid: process ID of the process you want to monitor :type pid: int :return: current usage :rtype: str """ client = SSHClient(storagerouter_ip, username='******') return client.run("grep Vm /proc/{0}/status | tr -s ' '".format(pid), allow_insecure=True)
def create_cluster(cluster_name, ip, exclude_ports, plugins=None): """ Creates a cluster """ logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip)) client = SSHClient(ip) base_dir = client.config_read('ovs.arakoon.location').rstrip('/') port_range = client.config_read('ovs.ports.arakoon') ports = System.get_free_ports(port_range, exclude_ports, 2, client) node_name = System.get_my_machine_id(client) config = ArakoonClusterConfig(cluster_name, plugins) if not [node.name for node in config.nodes if node.name == node_name]: config.nodes.append( ArakoonNodeConfig( name=node_name, ip=ip, client_port=ports[0], messaging_port=ports[1], log_dir=ArakoonInstaller.ARAKOON_LOG_DIR.format( cluster_name), home=ArakoonInstaller.ARAKOON_HOME_DIR.format( base_dir, cluster_name), tlog_dir=ArakoonInstaller.ARAKOON_TLOG_DIR.format( base_dir, cluster_name))) ArakoonInstaller._deploy(config) logger.debug('Creating cluster {0} on {1} completed'.format( cluster_name, ip)) return {'client_port': ports[0], 'messaging_port': ports[1]}
def setup(): """ Setup for Arakoon package, will be executed when any test in this package is being executed Make necessary changes before being able to run the tests :return: None """ autotest_config = General.get_config() backend_name = autotest_config.get('backend', 'name') assert backend_name, 'Please fill out a backend name in the autotest.cfg file' backend = GeneralBackend.get_by_name(backend_name) if backend is not None: GeneralAlba.remove_alba_backend(backend.alba_backend) for storagerouter in GeneralStorageRouter.get_masters(): root_client = SSHClient(storagerouter, username='******') if GeneralService.get_service_status(name='ovs-scheduled-tasks', client=root_client) is True: GeneralService.stop_service(name='ovs-scheduled-tasks', client=root_client) storagerouters = GeneralStorageRouter.get_storage_routers() for sr in storagerouters: root_client = SSHClient(sr, username='******') GeneralDisk.add_db_role(sr) for location in TEST_CLEANUP: root_client.run('rm -rf {0}'.format(location)) GeneralAlba.add_alba_backend(backend_name) GeneralArakoon.voldrv_arakoon_checkup()
def system_services_check_test(): """ Verify some system services """ services_to_commands = { "nginx": """ps -efx|grep nginx|grep -v grep""", "rabbitmq-server": """ps -ef|grep rabbitmq-|grep -v grep""", "memcached": """ps -ef|grep memcached|grep -v grep""", "ovs-arakoon-ovsdb": """initctl list| grep ovsdb""", "ovs-snmp": """initctl list| grep ovs-snmp""", "ovs-support-agent": """initctl list| grep support""", "ovs-volumerouter-consumer": """initctl list| grep volumerou""", "ovs-watcher-framework": """initctl list| grep watcher-fr""" } errors = '' services_checked = 'Following services found running:\n' grid_ip = General.get_config().get('main', 'grid_ip') ssh_pass = General.get_config().get('mgmtcenter', 'password') client = SSHClient(grid_ip, username='******', password=ssh_pass) for service_to_check in services_to_commands.iterkeys(): out, err = client.run(services_to_commands[service_to_check], debug=True) if len(err): errors += "Error when trying to run {0}:\n{1}".format(services_to_commands[service_to_check], err) else: if len(out): services_checked += "{0}\n".format(service_to_check) else: errors += "Couldn't find {0} running process\n".format(service_to_check) print services_checked assert len(errors) == 0, "Found the following errors while checking for the system services:{0}\n".format(errors)
def collapse_arakoon(): """ Collapse Arakoon's Tlogs :return: None """ logger.info('Starting arakoon collapse') arakoon_clusters = {} for service in ServiceList.get_services(): if service.type.name in ('Arakoon', 'NamespaceManager', 'AlbaManager'): arakoon_clusters[service.name.replace('arakoon-', '')] = service.storagerouter for cluster, storagerouter in arakoon_clusters.iteritems(): logger.info(' Collapsing cluster {0}'.format(cluster)) client = SSHClient(storagerouter) parser = client.rawconfig_read(PyrakoonStore.ARAKOON_CONFIG_FILE.format(cluster)) nodes = {} for node in parser.get('global', 'cluster').split(','): node = node.strip() nodes[node] = ([parser.get(node, 'ip')], parser.get(node, 'client_port')) config = ArakoonClientConfig(str(cluster), nodes) for node in nodes.keys(): logger.info(' Collapsing node: {0}'.format(node)) client = ArakoonAdminClient(node, config) try: client.collapse_tlogs(2) except: logger.exception('Error during collapsing cluster {0} node {1}'.format(cluster, node)) logger.info('Arakoon collapse finished')
def test_single_node(self): node = sorted(TestArakoonInstaller.nodes.keys())[0] result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, node, '/tmp/db') contents = SSHClient(node).file_read(TestArakoonInstaller.cluster_config_file) expected = TestArakoonInstaller.expected_global.format(TestArakoonInstaller.nodes[node], TestArakoonInstaller.cluster_name) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node) self.assertEqual(contents.strip(), expected.strip())
def deploy_to_slave(master_ip, slave_ip, cluster_name): """ Deploys the configuration file to a slave :param master_ip: IP of the node to deploy from :type master_ip: str :param slave_ip: IP of the slave to deploy to :type slave_ip: str :param cluster_name: Name of the cluster of which to deploy the configuration file :type cluster_name: str :return: None """ EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2}'.format(cluster_name, master_ip, slave_ip)) master_client = SSHClient(master_ip, username='******') slave_client = SSHClient(slave_ip, username='******') current_cluster = [] for item in master_client.run('etcdctl member list').splitlines(): info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict() current_cluster.append('{0}={1}'.format(info['name'], info['peer'])) EtcdInstaller._setup_proxy(','.join(current_cluster), slave_client, cluster_name, force=True) EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2} completed'.format(cluster_name, master_ip, slave_ip))
def shrink_cluster(remaining_node_ip, deleted_node_ip, cluster_name): """ Removes a node from a cluster, the old node will become a slave :param cluster_name: The name of the cluster to shrink :param deleted_node_ip: The ip of the node that should be deleted :param remaining_node_ip: The ip of a remaining node """ logger.debug('Shrinking cluster "{0}" from {1}'.format(cluster_name, deleted_node_ip)) current_client = SSHClient(remaining_node_ip, username='******') if not EtcdInstaller._is_healty(cluster_name, current_client): raise RuntimeError('Cluster "{0}" unhealthy, aborting shrink'.format(cluster_name)) old_client = SSHClient(deleted_node_ip, username='******') node_name = System.get_my_machine_id(old_client) node_id = None for item in current_client.run('etcdctl member list').splitlines(): info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict() if info['name'] == node_name: node_id = info['id'] if node_id is None: raise RuntimeError('Could not locate {0} in the cluster'.format(deleted_node_ip)) current_client.run('etcdctl member remove {0}'.format(node_id)) EtcdInstaller.deploy_to_slave(remaining_node_ip, deleted_node_ip, cluster_name) EtcdInstaller.wait_for_cluster(cluster_name, current_client) logger.debug('Shrinking cluster "{0}" from {1} completed'.format(cluster_name, deleted_node_ip))
def create_cluster(cluster_name, ip, exclude_ports, base_dir, plugins=None): """ Creates a cluster """ logger.debug("Creating cluster {0} on {1}".format(cluster_name, ip)) client = SSHClient(ip) base_dir = base_dir.rstrip("/") port_range = client.config_read("ovs.ports.arakoon") ports = System.get_free_ports(port_range, exclude_ports, 2, client) node_name = System.get_my_machine_id(client) config = ArakoonClusterConfig(cluster_name, plugins) if not [node.name for node in config.nodes if node.name == node_name]: config.nodes.append( ArakoonNodeConfig( name=node_name, ip=ip, client_port=ports[0], messaging_port=ports[1], log_dir=ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name), home=ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name), tlog_dir=ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name), ) ) ArakoonInstaller._deploy(config) logger.debug("Creating cluster {0} on {1} completed".format(cluster_name, ip)) return {"client_port": ports[0], "messaging_port": ports[1]}
def delete_volume(vdisk, vpool, loop_device=None, root_client=None, wait=True): """ Delete a volume :param vdisk: Virtual disk to delete :param vpool: vPool which hosts the Virtual Disk :param loop_device: Loop device where volume is mounted on :param root_client: SSHClient object :param wait: Wait for the volume to be deleted from model :return: None """ location = GeneralVDisk.get_filesystem_location(vpool=vpool, vdisk_name=vdisk.name) if root_client is None: root_client = SSHClient('127.0.0.1', username='******') if loop_device is not None: GeneralVDisk.disconnect_volume(loop_device, root_client) root_client.file_delete(location) if wait is True: counter = 0 timeout = 60 volume_name = '/' + os.path.basename(location) while True and counter < timeout: time.sleep(1) vdisks = VDiskList.get_by_devicename_and_vpool(volume_name, vpool) if vdisks is None: break counter += 1 if counter == timeout: raise RuntimeError('Disk {0} was not deleted from model after {1} seconds'.format(volume_name, timeout))
def setUp(cls): for node in TestArakoonInstaller.nodes: client = SSHClient(node) client.dir_delete('/tmp/db') client.dir_delete('/tmp/cfg') client.dir_create('/tmp/db') client.dir_create('/tmp/cfg')
def deploy_to_slave(master_ip, slave_ip, cluster_name, client_port=DEFAULT_CLIENT_PORT): """ Deploys the configuration file to a slave :param master_ip: IP of the node to deploy from :type master_ip: str :param slave_ip: IP of the slave to deploy to :type slave_ip: str :param cluster_name: Name of the cluster of which to deploy the configuration file :type cluster_name: str :param client_port: Port to be used by client :type client_port: int :return: None """ EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2}'.format(cluster_name, master_ip, slave_ip)) master_client = SSHClient(master_ip, username='******') slave_client = SSHClient(slave_ip, username='******') current_cluster = [] list_command = ['etcdctl', 'member', 'list'] if client_port != EtcdInstaller.DEFAULT_CLIENT_PORT: list_command = ['etcdctl', '--peers={0}:{1}'.format(master_ip, client_port), 'member', 'list'] for item in master_client.run(list_command).splitlines(): info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict() current_cluster.append('{0}={1}'.format(info['name'], info['peer'])) EtcdInstaller._setup_proxy(','.join(current_cluster), slave_client, cluster_name, force=True) EtcdInstaller._logger.debug(' Setting up proxy "{0}" from {1} to {2} completed'.format(cluster_name, master_ip, slave_ip))
def config_files_check_test(): """ Verify some configuration files """ issues_found = '' config_keys = { "/ovs/framework/memcache", "/ovs/arakoon/ovsdb/config" } for key_to_check in config_keys: if not Configuration.exists(key_to_check, raw=True): issues_found += "Couldn't find {0}\n".format(key_to_check) config_files = { "rabbitmq.config": "/etc/rabbitmq/rabbitmq.config", } grid_ip = General.get_config().get('main', 'grid_ip') ssh_pass = General.get_config().get('mgmtcenter', 'password') client = SSHClient(grid_ip, username='******', password=ssh_pass) for config_file_to_check in config_files.iterkeys(): if not client.file_exists(config_files[config_file_to_check]): issues_found += "Couldn't find {0}\n".format(config_file_to_check) assert issues_found == '',\ "Found the following issues while checking for the config files:{0}\n".format(issues_found)
def teardown(): """ Teardown for Arakoon package, will be executed when all started tests in this package have ended Removal actions of possible things left over after the test-run :return: None """ autotest_config = General.get_config() backend_name = autotest_config.get('backend', 'name') backend = GeneralBackend.get_by_name(backend_name) if backend is not None: GeneralAlba.remove_alba_backend(backend.alba_backend) for storagerouter in GeneralStorageRouter.get_masters(): root_client = SSHClient(storagerouter, username='******') if GeneralService.get_service_status(name='ovs-scheduled-tasks', client=root_client) is False: GeneralService.start_service(name='ovs-scheduled-tasks', client=root_client) for location in TEST_CLEANUP: root_client.run('rm -rf {0}'.format(location)) for key in KEY_CLEANUP: if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True): EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def shrink_vpool(storage_driver): """ Remove a Storage Driver from a vPool :param storage_driver: Storage Driver to remove from the vPool :return: None """ vpool = storage_driver.vpool if GeneralHypervisor.get_hypervisor_type() == "VMWARE": root_client = SSHClient(storage_driver.storagerouter, username="******") if storage_driver.mountpoint in General.get_mountpoints(root_client): root_client.run(["umount", "storage_driver.mountpoint"]) task_result = GeneralVPool.api.execute_post_action( component="vpools", guid=vpool.guid, action="shrink_vpool", data={"storagerouter_guid": storage_driver.storagerouter.guid}, wait=True, timeout=GeneralVPool.TIMEOUT_ADD_VPOOL, ) if task_result[0] is not True: raise RuntimeError( "Storage Driver with ID {0} was not successfully removed from vPool {1}".format( storage_driver.storagedriver_id, vpool.name ), task_result, ) return GeneralVPool.get_vpool_by_name(vpool_name=vpool.name)
def restart_cluster_add(cluster_name, current_ips, new_ip): """ Execute a (re)start sequence after adding a new node to a cluster. """ logger.debug('Restart sequence (add) for {0}'.format(cluster_name)) logger.debug('Current ips: {0}'.format(', '.join(current_ips))) logger.debug('New ip: {0}'.format(new_ip)) logger.debug('Catching up new node {0} for cluster {1}'.format( new_ip, cluster_name)) with Remote(new_ip, [ArakoonManagementEx], 'ovs') as remote: cluster = remote.ArakoonManagementEx().getCluster(cluster_name) cluster.catchup_node() logger.debug( 'Catching up new node {0} for cluster {1} completed'.format( new_ip, cluster_name)) threshold = 2 if new_ip in current_ips else 1 for ip in current_ips: if ip == new_ip: continue client = SSHClient(ip, username='******') ArakoonInstaller.stop(cluster_name, client=client) ArakoonInstaller.start(cluster_name, client=client) logger.debug(' Restarted node {0} for cluster {1}'.format( client.ip, cluster_name)) if len(current_ips ) > threshold: # A two node cluster needs all nodes running ArakoonInstaller.wait_for_cluster(cluster_name) new_client = SSHClient(new_ip, username='******') ArakoonInstaller.start(cluster_name, client=new_client) ArakoonInstaller.wait_for_cluster(cluster_name) logger.debug('Started node {0} for cluster {1}'.format( new_ip, cluster_name))
def system_services_check_test(): """ Verify some system services """ services_to_commands = { "nginx": """ps -efx|grep nginx|grep -v grep""", "rabbitmq-server": """ps -ef|grep rabbitmq-|grep -v grep""", "memcached": """ps -ef|grep memcached|grep -v grep""", } errors = '' services_checked = 'Following services found running:\n' grid_ip = General.get_config().get('main', 'grid_ip') ssh_pass = General.get_config().get('mgmtcenter', 'password') client = SSHClient(grid_ip, username='******', password=ssh_pass) for service_to_check in services_to_commands.iterkeys(): out, err = client.run(services_to_commands[service_to_check], debug=True, allow_insecure=True, return_stderr=True) if len(err): errors += "Error when trying to run {0}:\n{1}".format(services_to_commands[service_to_check], err) else: if len(out): services_checked += "{0}\n".format(service_to_check) else: errors += "Couldn't find {0} running process\n".format(service_to_check) for non_running_service in GeneralSystem.list_non_running_ovs_services(grid_ip): errors += str(non_running_service) assert len(errors) == 0,\ "Found the following errors while checking for the system services:{0}\n".format(errors)
def set_rdma_capability(storagerouter_guid): """ Check if the StorageRouter has been reconfigured to be able to support RDMA :param storagerouter_guid: Guid of the StorageRouter to check and set :type storagerouter_guid: str :return: None :rtype: NoneType """ storagerouter = StorageRouter(storagerouter_guid) client = SSHClient(storagerouter, username='******') rdma_capable = False with remote(client.ip, [os], username='******') as rem: for root, dirs, files in rem.os.walk('/sys/class/infiniband'): for directory in dirs: ports_dir = '/'.join([root, directory, 'ports']) if not rem.os.path.exists(ports_dir): continue for sub_root, sub_dirs, _ in rem.os.walk(ports_dir): if sub_root != ports_dir: continue for sub_directory in sub_dirs: state_file = '/'.join( [sub_root, sub_directory, 'state']) if rem.os.path.exists(state_file): if 'ACTIVE' in client.run(['cat', state_file]): rdma_capable = True storagerouter.rdma_capable = rdma_capable storagerouter.save()
def shrink_cluster(remaining_node_ip, ip_to_remove, cluster_name, offline_node_ips=None): """ Removes a node from a cluster, the old node will become a slave :param cluster_name: The name of the cluster to shrink :param ip_to_remove: The ip of the node that should be removed from the cluster :param remaining_node_ip: The ip of a remaining node in the cluster :param offline_node_ips: IPs of offline nodes """ logger.debug('Shrinking cluster "{0}" from {1}'.format(cluster_name, ip_to_remove)) current_client = SSHClient(remaining_node_ip, username='******') if not EtcdInstaller._is_healty(cluster_name, current_client): raise RuntimeError('Cluster "{0}" unhealthy, aborting shrink'.format(cluster_name)) node_id = None for item in current_client.run('etcdctl member list').splitlines(): info = re.search(EtcdInstaller.MEMBER_REGEX, item).groupdict() if EtcdInstaller.CLIENT_URL.format(ip_to_remove) == info['client']: node_id = info['id'] if node_id is not None: current_client.run('etcdctl member remove {0}'.format(node_id)) if ip_to_remove not in offline_node_ips: EtcdInstaller.deploy_to_slave(remaining_node_ip, ip_to_remove, cluster_name, force=True) EtcdInstaller.wait_for_cluster(cluster_name, current_client) logger.debug('Shrinking cluster "{0}" from {1} completed'.format(cluster_name, ip_to_remove))
def __init__(self, host='127.0.0.1', login='******', passwd=None): logger.debug('Init libvirt') self.states = { libvirt.VIR_DOMAIN_NOSTATE: 'NO STATE', libvirt.VIR_DOMAIN_RUNNING: 'RUNNING', libvirt.VIR_DOMAIN_BLOCKED: 'BLOCKED', libvirt.VIR_DOMAIN_PAUSED: 'PAUSED', libvirt.VIR_DOMAIN_SHUTDOWN: 'SHUTDOWN', libvirt.VIR_DOMAIN_SHUTOFF: 'TURNEDOFF', libvirt.VIR_DOMAIN_CRASHED: 'CRASHED' } pattern = re.compile( r"^(?<!\S)((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\b|\.\b){7}(?!\S)$") if pattern.match(host): self.host = host else: raise ValueError("{0} is not a valid ip.".format(host)) self.host = host self.login = login self.streams = {} self.ssh_client = SSHClient(host, username=login, password=passwd) self._conn = self.connect(login, host) # Enable event registering libvirt.virEventRegisterDefaultImpl() logger.debug('Init complete')
def restart_cluster(cluster_name, master_ip): """ Execute a restart sequence (Executed after arakoon and/or alba package upgrade) """ logger.debug('Restart sequence for {0} via {1}'.format( cluster_name, master_ip)) client = SSHClient(master_ip) config = ArakoonClusterConfig(cluster_name) config.load_config(client) all_clients = [ SSHClient(node.ip) for node in config.nodes if node.ip != master_ip ] + [client] if len(config.nodes) <= 2: logger.debug( ' Insufficient nodes in cluster {0}. Full restart'.format( cluster_name)) for function in [ArakoonInstaller.stop, ArakoonInstaller.start]: for client in all_clients: function(cluster_name, client) ArakoonInstaller.wait_for_cluster(cluster_name) else: logger.debug( ' Sufficient nodes in cluster {0}. Sequential restart'.format( cluster_name)) for client in all_clients: ArakoonInstaller.stop(cluster_name, client) ArakoonInstaller.start(cluster_name, client) logger.debug(' Restarted node {0} on cluster {1}'.format( client.ip, cluster_name)) ArakoonInstaller.wait_for_cluster(cluster_name) logger.debug('Restart sequence for {0} via {1} completed'.format( cluster_name, master_ip))
def has_cluster(ip, cluster_name): logger.debug('Checking whether {0} has cluster "{1}" running'.format(ip, cluster_name)) client = SSHClient(ip, username='******') try: return client.run('etcdctl member list').strip() != '' except CalledProcessError: return False
def write_to_volume(vdisk=None, vpool=None, location=None, count=1024, bs='1M', input_type='random', root_client=None): """ Write some data to a file :param vdisk: Virtual disk to write on :param vpool: vPool which hosts the Virtual Disk :param location: Absolute path to file :param count: amount of blocks to write :param bs: Size of the blocks to write :param input_type: Type of input (null, zero, random) :param root_client: SSHClient object :return: None """ if location is None and (vdisk is None or vpool is None): raise ValueError('vDisk and vPool must be provided if no location has been provided') if location is None: location = GeneralVDisk.get_filesystem_location(vpool=vpool, vdisk_name=vdisk.name) if root_client is None: root_client = SSHClient('127.0.0.1', username='******') if input_type not in ('null', 'zero', 'random'): raise ValueError('Invalid input type provided') if General.check_file_is_link(location, root_client.ip, root_client.username, root_client.password): print "Writing to {0}".format(root_client.file_read_link(location)) else: if not root_client.file_exists(location): raise ValueError('File {0} does not exist on Storage Router {1}'.format(location, root_client.ip)) if not isinstance(count, int) or count < 1: raise ValueError('Count must be an integer > 0') root_client.run('dd conv=notrunc if=/dev/{0} of={1} bs={2} count={3}'.format(input_type, location, bs, count))
def unpartition_disk(disk, partitions=None, wait=True): """ Return disk to RAW state :param disk: Disk DAL object :param partitions: Partitions DAL object list :return: None """ if partitions is None: partitions = disk.partitions else: for partition in partitions: if partition not in disk.partitions: raise RuntimeError('Partition {0} does not belong to disk {1}'.format(partition.mountpoint, disk.name)) if len(disk.partitions) == 0: return root_client = SSHClient(disk.storagerouter, username='******') for partition in partitions: General.unmount_partition(root_client, partition) root_client.run(['parted', '-s', '/dev/' + disk.name, 'mklabel', 'gpt']) GeneralStorageRouter.sync_with_reality(disk.storagerouter) counter = 0 timeout = 60 while counter < timeout: time.sleep(1) disk = GeneralDisk.get_disk(guid=disk.guid) if len(disk.partitions) == 0: break counter += 1 if counter == timeout: raise RuntimeError('Removing partitions failed for disk:\n {0} '.format(disk.name))
def extend_cluster(master_ip, new_ip, cluster_name, exclude_ports, base_dir): """ Extends a cluster to a given new node """ logger.debug("Extending cluster {0} from {1} to {2}".format(cluster_name, master_ip, new_ip)) client = SSHClient(master_ip) config = ArakoonClusterConfig(cluster_name) config.load_config(client) client = SSHClient(new_ip) base_dir = base_dir.rstrip("/") port_range = client.config_read("ovs.ports.arakoon") ports = System.get_free_ports(port_range, exclude_ports, 2, client) node_name = System.get_my_machine_id(client) if not [node.name for node in config.nodes if node.name == node_name]: config.nodes.append( ArakoonNodeConfig( name=node_name, ip=new_ip, client_port=ports[0], messaging_port=ports[1], log_dir=ArakoonInstaller.ARAKOON_LOG_DIR.format(cluster_name), home=ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name), tlog_dir=ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name), ) ) ArakoonInstaller._deploy(config) logger.debug("Extending cluster {0} from {1} to {2} completed".format(cluster_name, master_ip, new_ip)) return {"client_port": ports[0], "messaging_port": ports[1]}
def _clean(): volatile = VolatileFactory.get_client() persistent = PersistentFactory.get_client() # noinspection PyProtectedMember volatile._clean() # noinspection PyProtectedMember persistent._clean() # noinspection PyProtectedMember SSHClient._clean() # noinspection PyProtectedMember SystemdMock._clean() # noinspection PyProtectedMember MDSClient._clean() # noinspection PyProtectedMember Decorators._clean() # noinspection PyProtectedMember MockedSSHClient._clean() # noinspection PyProtectedMember StorageRouterClient._clean() Logger._logs = {} DataList._test_hooks = {} Toolbox._function_pointers = {} # Clean underlying persistent store Configuration.get_client()._clean() for file_name in glob.glob( ArakoonClusterConfig.CONFIG_FILE.format('unittest*')): os.remove(file_name) for full_path in glob.glob(DalHelper.UNITTEST_DIR.format('*')): shutil.rmtree(full_path) return volatile, persistent
def setUp(cls): for node in TestArakoonInstaller.nodes: client = SSHClient(node) root_client = SSHClient(node, username='******') root_client.dir_delete('/tmp/db') root_client.dir_create('/tmp/db') client.dir_delete(TestArakoonInstaller.cluster_config_path) client.dir_create(TestArakoonInstaller.cluster_config_path)
def is_datastore_available(self, mountpoint): """ Check if datastore is available :param mountpoint: Mountpoint of the NFS datastore """ if self.ssh_client is None: self.ssh_client = SSHClient(self.host, username='******') return self.ssh_client.run("[ -d {0} ] && echo 'yes' || echo 'no'".format(mountpoint)) == 'yes'
def migrate(previous_version, master_ips=None, extra_ips=None): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at verison 3 it will execute two steps: - 1 > 2 - 2 > 3 :param previous_version: The previous version from which to start the migration. :param master_ips: IP addresses of the MASTER nodes :param extra_ips: IP addresses of the EXTRA nodes """ working_version = previous_version # Version 1 introduced: # - Flexible SSD layout if working_version < 1: from ovs.extensions.generic.configuration import Configuration if Configuration.exists('ovs.arakoon'): Configuration.delete('ovs.arakoon', remove_root=True) Configuration.set('ovs.core.ovsdb', '/opt/OpenvStorage/db') working_version = 1 # Version 2 introduced: # - Registration if working_version < 2: import time from ovs.extensions.generic.configuration import Configuration if not Configuration.exists('ovs.core.registered'): Configuration.set('ovs.core.registered', False) Configuration.set('ovs.core.install_time', time.time()) working_version = 2 # Version 3 introduced: # - New arakoon clients if working_version < 3: from ovs.extensions.db.arakoon import ArakoonInstaller reload(ArakoonInstaller) from ovs.extensions.db.arakoon import ArakoonInstaller from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.generic.configuration import Configuration if master_ips is not None: for ip in master_ips: client = SSHClient(ip) if client.dir_exists(ArakoonInstaller.ArakoonInstaller.ARAKOON_CONFIG_DIR): for cluster_name in client.dir_list(ArakoonInstaller.ArakoonInstaller.ARAKOON_CONFIG_DIR): try: ArakoonInstaller.ArakoonInstaller.deploy_cluster(cluster_name, ip) except: pass if Configuration.exists('ovs.core.storage.persistent'): Configuration.set('ovs.core.storage.persistent', 'pyrakoon') working_version = 3 return working_version
def remove_from_slave(master_ip, slave_ip, cluster_name): """ Removes everything related to a given cluster from the slave """ client = SSHClient(master_ip) config = ArakoonClusterConfig(cluster_name) config.load_config(client) client = SSHClient(slave_ip) config.delete_config(client)
def is_devstack_installed(): """ Check if OpenStack or DevStack is installed :return: True if installed """ client = SSHClient('127.0.0.1', username='******') is_openstack = ServiceManager.has_service(OSManager.get_openstack_cinder_service_name(), client) is_devstack = 'stack' in str(client.run('ps aux | grep SCREEN | grep stack | grep -v grep || true'), allow_insecure=True) return is_openstack or is_devstack
def remove_mds_service(mds_service, vpool, reload_config): """ Removes an MDS service :param mds_service: The MDS service to remove :param vpool: The vPool for which the MDS service will be removed :param reload_config: If True, the volumedriver's updated configuration will be reloaded """ if len(mds_service.vdisks_guids) > 0: raise RuntimeError('Cannot remove MDSService that is still serving disks') storagerouter = mds_service.service.storagerouter client = SSHClient(storagerouter) mdsservice_type = ServiceTypeList.get_by_name('MetadataServer') # Clean up model directories_to_clean = [] for sd_partition in mds_service.storagedriver_partitions: directories_to_clean.append(sd_partition.path) sd_partition.delete() mds_service.delete() mds_service.service.delete() # Generate new mds_nodes section mds_nodes = [] for service in mdsservice_type.services: if service.storagerouter_guid == storagerouter.guid: mds_service = service.mds_service if mds_service.vpool_guid == vpool.guid: sdp = [sd_partition.path for sd_partition in mds_service.storagedriver_partitions if sd_partition.role == DiskPartition.ROLES.DB] mds_nodes.append({'host': service.storagerouter.ip, 'port': service.ports[0], 'db_directory': sdp[0], 'scratch_directory': sdp[0]}) # Generate the correct section in the Storage Driver's configuration storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.name) storagedriver_config.load(client) storagedriver_config.clean() # Clean out obsolete values storagedriver_config.configure_metadata_server(mds_nodes=mds_nodes) storagedriver_config.save(client, reload_config=reload_config) tries = 5 while tries > 0: try: root_client = SSHClient(storagerouter, username='******') root_client.dir_delete(directories=directories_to_clean, follow_symlinks=True) for dir_name in directories_to_clean: logger.debug('Recursively removed {0}'.format(dir_name)) break except Exception: time.sleep(5) logger.debug('Waiting for the MDS service to go down...') tries -= 1 if tries == 0: raise
def test_single_node(self): base_port = Configuration.get('ovs.ports.arakoon')[0] cluster = 'one' node = sorted(TestArakoonInstaller.nodes.keys())[0] ArakoonInstaller.create_cluster(cluster, node, []) contents = SSHClient(node).file_read(self._get_config_path(cluster)) expected = TestArakoonInstaller.expected_global.format(cluster, TestArakoonInstaller.nodes[node]) expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) self.assertEqual(contents.strip(), expected.strip())
def deploy_to_slave(master_ip, slave_ip, cluster_name): """ Deploys the configuration file to a slave """ client = SSHClient(master_ip) config = ArakoonClusterConfig(cluster_name) config.load_config(client) client = SSHClient(slave_ip) config.write_config(client)
def get_mds_storagedriver_config_set(vpool, check_online=False): """ Builds a configuration for all StorageRouters from a given VPool with following goals: * Primary MDS is the local one * All slaves are on different hosts * Maximum `mds.safety` nodes are returned The configuration returned is the default configuration used by the volumedriver of which in normal use-cases only the 1st entry is used, because at volume creation time, the volumedriver needs to create 1 master MDS During ensure_safety, we actually create/set the MDS slaves for each volume :param vpool: vPool to get storagedriver configuration for :param check_online: Check whether the storage routers are actually responsive """ mds_per_storagerouter = {} mds_per_load = {} for storagedriver in vpool.storagedrivers: storagerouter = storagedriver.storagerouter if check_online is True: try: client = SSHClient(storagerouter) client.run('pwd') except UnableToConnectException: continue mds_service, load = MDSServiceController.get_preferred_mds(storagerouter, vpool, include_load=True) mds_per_storagerouter[storagerouter] = {'host': storagerouter.ip, 'port': mds_service.service.ports[0]} if load not in mds_per_load: mds_per_load[load] = [] mds_per_load[load].append(storagerouter) safety = EtcdConfiguration.get('/ovs/framework/storagedriver|mds_safety') config_set = {} for storagerouter, ip_info in mds_per_storagerouter.iteritems(): primary_failure_domain = storagerouter.primary_failure_domain secondary_failure_domain = storagerouter.secondary_failure_domain config_set[storagerouter.guid] = [ip_info] for load in sorted(mds_per_load): if len(config_set[storagerouter.guid]) >= safety: break other_storagerouters = mds_per_load[load] random.shuffle(other_storagerouters) for other_storagerouter in other_storagerouters: if len(config_set[storagerouter.guid]) >= safety: break if other_storagerouter != storagerouter and other_storagerouter in primary_failure_domain.primary_storagerouters: config_set[storagerouter.guid].append(mds_per_storagerouter[other_storagerouter]) if secondary_failure_domain is not None: for load in sorted(mds_per_load): if len(config_set[storagerouter.guid]) >= safety: break other_storagerouters = mds_per_load[load] random.shuffle(other_storagerouters) for other_storagerouter in other_storagerouters: if len(config_set[storagerouter.guid]) >= safety: break if other_storagerouter != storagerouter and other_storagerouter in secondary_failure_domain.primary_storagerouters: config_set[storagerouter.guid].append(mds_per_storagerouter[other_storagerouter]) return config_set
def get_configuration(vpool_guid): """ Retrieve the running storagedriver configuration for the vPool :param vpool_guid: Guid of the vPool to retrieve running configuration for :return: Dictionary with configuration """ vpool = VPool(vpool_guid) if not vpool.storagedrivers or not vpool.storagedrivers[0].storagerouter: return {} client = None for sd in vpool.storagedrivers: try: client = SSHClient(sd.storagerouter) client.run("pwd") break except UnableToConnectException: client = None pass if client is None: raise RuntimeError("Could not find an online storage router to retrieve vPool configuration from") storagedriver_config = StorageDriverConfiguration("storagedriver", vpool.name) storagedriver_config.load(client) dtl = storagedriver_config.configuration.get("failovercache", {}) file_system = storagedriver_config.configuration.get("filesystem", {}) volume_router = storagedriver_config.configuration.get("volume_router", {}) volume_manager = storagedriver_config.configuration.get("volume_manager", {}) dtl_mode = file_system.get("fs_dtl_mode", StorageDriverClient.VOLDRV_DTL_ASYNC) dedupe_mode = volume_manager.get("read_cache_default_mode", StorageDriverClient.VOLDRV_CONTENT_BASED) dtl_transport = dtl.get("failovercache_transport", StorageDriverClient.VOLDRV_DTL_TRANSPORT_TCP) cache_strategy = volume_manager.get("read_cache_default_behaviour", StorageDriverClient.VOLDRV_CACHE_ON_READ) sco_multiplier = volume_router.get("vrouter_sco_multiplier", 1024) dtl_config_mode = file_system.get("fs_dtl_config_mode", StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE) tlog_multiplier = volume_manager.get("number_of_scos_in_tlog", 20) non_disposable_sco_factor = volume_manager.get("non_disposable_scos_factor", 12) sco_size = sco_multiplier * 4 / 1024 # SCO size is in MiB ==> SCO multiplier * cluster size (4 KiB by default) write_buffer = tlog_multiplier * sco_size * non_disposable_sco_factor dtl_mode = StorageDriverClient.REVERSE_DTL_MODE_MAP[dtl_mode] dtl_enabled = dtl_config_mode == StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE if dtl_enabled is False: dtl_mode = StorageDriverClient.FRAMEWORK_DTL_NO_SYNC return { "sco_size": sco_size, "dtl_mode": dtl_mode, "dedupe_mode": StorageDriverClient.REVERSE_DEDUPE_MAP[dedupe_mode], "dtl_enabled": dtl_enabled, "write_buffer": write_buffer, "dtl_transport": StorageDriverClient.REVERSE_DTL_TRANSPORT_MAP[dtl_transport], "cache_strategy": StorageDriverClient.REVERSE_CACHE_MAP[cache_strategy], "tlog_multiplier": tlog_multiplier, }
def get_configuration(vpool_guid): """ Retrieve the running storagedriver configuration for the vPool :param vpool_guid: Guid of the vPool to retrieve running configuration for :return: Dictionary with configuration """ vpool = VPool(vpool_guid) if not vpool.storagedrivers or not vpool.storagedrivers[0].storagerouter: return {} client = None for sd in vpool.storagedrivers: try: client = SSHClient(sd.storagerouter) client.run('pwd') break except UnableToConnectException: client = None pass if client is None: raise RuntimeError('Could not find an online storage router to retrieve vPool configuration from') storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.guid, vpool.storagedrivers[0].storagedriver_id) storagedriver_config.load() dtl = storagedriver_config.configuration.get('distributed_transaction_log', {}) file_system = storagedriver_config.configuration.get('filesystem', {}) volume_router = storagedriver_config.configuration.get('volume_router', {}) volume_manager = storagedriver_config.configuration.get('volume_manager', {}) dtl_mode = file_system.get('fs_dtl_mode', StorageDriverClient.VOLDRV_DTL_ASYNC) dedupe_mode = volume_manager.get('read_cache_default_mode', StorageDriverClient.VOLDRV_CONTENT_BASED) cluster_size = volume_manager.get('default_cluster_size', 4096) / 1024 dtl_transport = dtl.get('dtl_transport', StorageDriverClient.VOLDRV_DTL_TRANSPORT_TCP) cache_strategy = volume_manager.get('read_cache_default_behaviour', StorageDriverClient.VOLDRV_CACHE_ON_READ) sco_multiplier = volume_router.get('vrouter_sco_multiplier', 1024) dtl_config_mode = file_system.get('fs_dtl_config_mode', StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE) tlog_multiplier = volume_manager.get('number_of_scos_in_tlog', 20) non_disposable_sco_factor = volume_manager.get('non_disposable_scos_factor', 12) sco_size = sco_multiplier * cluster_size / 1024 # SCO size is in MiB ==> SCO multiplier * cluster size (4 KiB by default) write_buffer = tlog_multiplier * sco_size * non_disposable_sco_factor dtl_mode = StorageDriverClient.REVERSE_DTL_MODE_MAP[dtl_mode] dtl_enabled = dtl_config_mode == StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE if dtl_enabled is False: dtl_mode = StorageDriverClient.FRAMEWORK_DTL_NO_SYNC return {'sco_size': sco_size, 'dtl_mode': dtl_mode, 'dedupe_mode': StorageDriverClient.REVERSE_DEDUPE_MAP[dedupe_mode], 'dtl_enabled': dtl_enabled, 'cluster_size': cluster_size, 'write_buffer': write_buffer, 'dtl_transport': StorageDriverClient.REVERSE_DTL_TRANSPORT_MAP[dtl_transport], 'cache_strategy': StorageDriverClient.REVERSE_CACHE_MAP[cache_strategy], 'tlog_multiplier': tlog_multiplier}
def run_test_fuse(cls, storagedriver, disk_amount, write_amount, logger=LOGGER): """ Deploy and run a small io test using the FUSE interface :param storagedriver: chosen storagedriver for testing :param disk_amount: amount of disks to deploy and write/read to :param write_amount: amount of data to parse for writing/reading :param logger: logging instance :return: """ vpool = storagedriver.vpool client = SSHClient(storagedriver.storagerouter, username='******') vdisk_info = {} try: for vdisk_number in xrange(disk_amount): vdisk_name = '{0}{1}-fuse'.format(cls.PREFIX, vdisk_number) disk_location = "/mnt/{0}/{1}.raw".format( vpool.name, vdisk_name) logger.info("Truncating vdisk {0} on {1}:{2}".format( vdisk_name, storagedriver.storage_ip, vpool.name)) client.run( ["truncate", "-s", str(cls.VDISK_SIZE), disk_location]) vdisk = cls._get_vdisk('{0}.raw'.format(vdisk_name), vpool.name) vdisk_info[disk_location] = vdisk fio_configuration = { 'io_size': write_amount, 'configuration': (0, 100) } DataWriter.write_data_fio(client, fio_configuration, file_locations=vdisk_info.keys(), screen=False, loop_screen=False) fio_configuration = { 'io_size': write_amount, 'configuration': (100, 0) } DataWriter.write_data_fio(client, fio_configuration, file_locations=vdisk_info.keys(), screen=False, loop_screen=False) except Exception as ex: logger.error( 'An exception occur while testing edge+blktap: {0}'.format( str(ex))) raise finally: for vdisk in vdisk_info.values(): VDiskRemover.remove_vdisk_by_name(vdisk.devicename, vdisk.vpool.name)
def execute_command_on_node(host, command, password=None): """ Execute a command on a specific host :param host: Host to execute command on :param command: Command to execute :param password: Password used to login on host :return: Output of command """ cl = SSHClient(host, username='******', password=password) return cl.run(command)
def update_components(components): """ Initiate the update through commandline for all StorageRouters This is called upon by the API :return: None """ components = [component.strip() for component in components] root_client = SSHClient(endpoint=System.get_my_storagerouter(), username='******') root_client.run(['ovs', 'update', ','.join(components)])
def collapse_arakoon(): """ Collapse Arakoon's Tlogs :return: None """ from ovs_extensions.generic.toolbox import ExtensionsToolbox GenericController._logger.info('Arakoon collapse started') cluster_info = [] storagerouters = StorageRouterList.get_storagerouters() if os.environ.get('RUNNING_UNITTESTS') != 'True': cluster_info = [('cacc', storagerouters[0])] cluster_names = [] for service in ServiceList.get_services(): if service.is_internal is True and service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON, ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR): cluster = ExtensionsToolbox.remove_prefix(service.name, 'arakoon-') if cluster in cluster_names and cluster not in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST]: continue cluster_names.append(cluster) cluster_info.append((cluster, service.storagerouter)) workload = {} cluster_config_map = {} for cluster, storagerouter in cluster_info: GenericController._logger.debug(' Collecting info for cluster {0}'.format(cluster)) ip = storagerouter.ip if cluster in [ARAKOON_NAME, ARAKOON_NAME_UNITTEST] else None try: config = ArakoonClusterConfig(cluster_id=cluster, source_ip=ip) cluster_config_map[cluster] = config except: GenericController._logger.exception(' Retrieving cluster information on {0} for {1} failed'.format(storagerouter.ip, cluster)) continue for node in config.nodes: if node.ip not in workload: workload[node.ip] = {'node_id': node.name, 'clusters': []} workload[node.ip]['clusters'].append((cluster, ip)) for storagerouter in storagerouters: try: if storagerouter.ip not in workload: continue node_workload = workload[storagerouter.ip] client = SSHClient(storagerouter) for cluster, ip in node_workload['clusters']: try: GenericController._logger.debug(' Collapsing cluster {0} on {1}'.format(cluster, storagerouter.ip)) client.run(['arakoon', '--collapse-local', node_workload['node_id'], '2', '-config', cluster_config_map[cluster].external_config_path]) GenericController._logger.debug(' Collapsing cluster {0} on {1} completed'.format(cluster, storagerouter.ip)) except: GenericController._logger.exception(' Collapsing cluster {0} on {1} failed'.format(cluster, storagerouter.ip)) except UnableToConnectException: GenericController._logger.error(' Could not collapse any cluster on {0} (not reachable)'.format(storagerouter.name)) GenericController._logger.info('Arakoon collapse finished')
def delete_volume(vdisk, vpool, loop_device=None, root_client=None, wait=True): """ Delete a volume :param vdisk: Virtual disk to delete :param vpool: vPool which hosts the Virtual Disk :param loop_device: Loop device where volume is mounted on :param root_client: SSHClient object :param wait: Wait for the volume to be deleted from model :return: None """ location = GeneralVDisk.get_filesystem_location(vpool=vpool, vdisk_name=vdisk.name) if root_client is None: root_client = SSHClient('127.0.0.1', username='******') if loop_device is not None: root_client.run('umount /dev/{0}'.format(loop_device)) root_client.run('losetup -d /dev/{0}'.format(loop_device)) root_client.dir_delete('/mnt/{0}'.format(loop_device)) root_client.file_delete(location) if wait is True: counter = 0 timeout = 60 volume_name = os.path.basename(location).replace('-flat.vmdk', '').replace('.raw', '') while True and counter < timeout: time.sleep(1) vdisks = GeneralVDisk.get_vdisk_by_name(name=volume_name) if vdisks is None: break counter += 1 if counter == timeout: raise RuntimeError('Disk {0} was not deleted from model after {1} seconds'.format(volume_name, timeout))
def get_configuration(vpool_guid): """ Retrieve the running storagedriver configuration for the vPool :param vpool_guid: Guid of the vPool to retrieve running configuration for :return: Dictionary with configuration """ vpool = VPool(vpool_guid) if not vpool.storagedrivers or not vpool.storagedrivers[0].storagerouter: return {} client = None for sd in vpool.storagedrivers: try: client = SSHClient(sd.storagerouter) client.run('pwd') break except UnableToConnectException: client = None pass if client is None: raise RuntimeError('Could not find an online storage router to retrieve vPool configuration from') storagedriver_config = StorageDriverConfiguration('storagedriver', vpool.guid, vpool.storagedrivers[0].storagedriver_id) storagedriver_config.load() dtl = storagedriver_config.configuration.get('failovercache', {}) file_system = storagedriver_config.configuration.get('filesystem', {}) volume_router = storagedriver_config.configuration.get('volume_router', {}) volume_manager = storagedriver_config.configuration.get('volume_manager', {}) dtl_mode = file_system.get('fs_dtl_mode', StorageDriverClient.VOLDRV_DTL_ASYNC) dedupe_mode = volume_manager.get('read_cache_default_mode', StorageDriverClient.VOLDRV_CONTENT_BASED) dtl_transport = dtl.get('failovercache_transport', StorageDriverClient.VOLDRV_DTL_TRANSPORT_TCP) cache_strategy = volume_manager.get('read_cache_default_behaviour', StorageDriverClient.VOLDRV_CACHE_ON_READ) sco_multiplier = volume_router.get('vrouter_sco_multiplier', 1024) dtl_config_mode = file_system.get('fs_dtl_config_mode', StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE) tlog_multiplier = volume_manager.get('number_of_scos_in_tlog', 20) non_disposable_sco_factor = volume_manager.get('non_disposable_scos_factor', 12) sco_size = sco_multiplier * 4 / 1024 # SCO size is in MiB ==> SCO multiplier * cluster size (4 KiB by default) write_buffer = tlog_multiplier * sco_size * non_disposable_sco_factor dtl_mode = StorageDriverClient.REVERSE_DTL_MODE_MAP[dtl_mode] dtl_enabled = dtl_config_mode == StorageDriverClient.VOLDRV_DTL_AUTOMATIC_MODE if dtl_enabled is False: dtl_mode = StorageDriverClient.FRAMEWORK_DTL_NO_SYNC return {'sco_size': sco_size, 'dtl_mode': dtl_mode, 'dedupe_mode': StorageDriverClient.REVERSE_DEDUPE_MAP[dedupe_mode], 'dtl_enabled': dtl_enabled, 'write_buffer': write_buffer, 'dtl_transport': StorageDriverClient.REVERSE_DTL_TRANSPORT_MAP[dtl_transport], 'cache_strategy': StorageDriverClient.REVERSE_CACHE_MAP[cache_strategy], 'tlog_multiplier': tlog_multiplier}
def validate(*args, **kwargs): # check if alba backend exists or not if kwargs['cluster_basedir'] and kwargs['storagerouter_ip']: client = SSHClient(kwargs['storagerouter_ip'], username='******') if client.dir_exists(kwargs['cluster_basedir']): return func(*args, **kwargs) else: raise DirectoryNotFoundError("Required base_dir `{0}` not found on storagerouter `{1}`" .format(kwargs['cluster_basedir'], kwargs['storagerouter_ip'])) else: raise AttributeError("Missing parameter(s): cluster_basedir or storagerouter_ip")
def add_extra_node(**kwargs): """ Add extra node hook :param kwargs: Extra parameters :return: None """ ip = kwargs['cluster_ip'] license_contents = [] for lic in LicenseList.get_licenses(): license_contents.append(lic.hash) client = SSHClient(ip) client.file_write('/opt/OpenvStorage/config/licenses', '{0}\n'.format('\n'.join(license_contents)))
def mountpoint_exists(name, storagerouter_guid): """ Checks whether a given mount point for a vPool exists :param name: Name of the mount point to check :type name: str :param storagerouter_guid: Guid of the StorageRouter on which to check for mount point existence :type storagerouter_guid: str :return: True if mount point not in use else False :rtype: bool """ client = SSHClient(StorageRouter(storagerouter_guid)) return client.dir_exists(directory='/mnt/{0}'.format(name))
def install_plugins(): """ (Re)load plugins """ if ServiceManager.has_service('ovs-watcher-framework', SSHClient('127.0.0.1', username='******')): # If the watcher is running, 'ovs setup' was executed and we need to restart everything to load # the plugin. In the other case, the plugin will be loaded once 'ovs setup' is executed from ovs.dal.lists.storagerouterlist import StorageRouterList clients = [] try: for storagerouter in StorageRouterList.get_storagerouters(): clients.append(SSHClient(storagerouter, username='******')) except UnableToConnectException: raise RuntimeError('Not all StorageRouters are reachable') for client in clients: for service_name in ['watcher-framework', 'memcached']: ServiceManager.stop_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status( service_name, client=client) is False: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError( 'Could not stop service: {0}'.format(service_name)) for client in clients: for service_name in ['memcached', 'watcher-framework']: ServiceManager.start_service(service_name, client=client) wait = 30 while wait > 0: if ServiceManager.get_service_status( service_name, client=client) is True: break time.sleep(1) wait -= 1 if wait == 0: raise RuntimeError( 'Could not start service: {0}'.format( service_name)) from ovs.dal.helpers import Migration Migration.migrate() from ovs.lib.helpers.toolbox import Toolbox ip = System.get_my_storagerouter().ip functions = Toolbox.fetch_hooks('plugin', 'postinstall') for function in functions: function(ip=ip)
def restart_cluster(cluster_name, master_ip, filesystem): """ Execute a restart sequence (Executed after arakoon and/or alba package upgrade) :param cluster_name: Name of the cluster to restart :type cluster_name: str :param master_ip: IP of one of the cluster nodes :type master_ip: str :param filesystem: Indicates whether the configuration should be on the filesystem or in a configuration cluster :type filesystem: bool :return: None """ ArakoonInstaller._logger.debug( 'Restart sequence for {0} via {1}'.format(cluster_name, master_ip)) config = ArakoonClusterConfig(cluster_name, filesystem) config.load_config(master_ip) arakoon_client = ArakoonInstaller.build_client(config) root_client = SSHClient(master_ip, username='******') all_clients = [ SSHClient(node.ip, username='******') for node in config.nodes ] if len(config.nodes) <= 2: ArakoonInstaller._logger.debug( ' Insufficient nodes in cluster {0}. Full restart'.format( cluster_name)) for function in [ArakoonInstaller.stop, ArakoonInstaller.start]: for client in all_clients: function(cluster_name, client) ArakoonInstaller.wait_for_cluster(cluster_name, master_ip, filesystem) else: ArakoonInstaller._logger.debug( ' Sufficient nodes in cluster {0}. Sequential restart'.format( cluster_name)) for client in all_clients: ArakoonInstaller.stop(cluster_name, client) ArakoonInstaller.start(cluster_name, client) ArakoonInstaller._logger.debug( ' Restarted node {0} on cluster {1}'.format( client.ip, cluster_name)) ArakoonInstaller.wait_for_cluster(cluster_name, master_ip, filesystem) ArakoonInstaller.start(cluster_name, root_client) ArakoonInstaller.wait_for_cluster(cluster_name, master_ip, filesystem) arakoon_client.set(ArakoonInstaller.INTERNAL_CONFIG_KEY, config.export_ini()) ArakoonInstaller._logger.debug( 'Restart sequence for {0} via {1} completed'.format( cluster_name, master_ip))
def restart(cls, storagerouters, logger=LOGGER): """ Restarts a list of storagerouters :param storagerouters: list of storagerouters :param logger: logging instance :return: None """ for storagerouter in storagerouters: logger.info("Restarting ovs-workers on {0}".format( storagerouter.ip)) client = SSHClient(str(storagerouter.ip), username='******', cached=False) client.run(['systemctl', 'restart', 'ovs-workers.service'])
def adjust_for_reroute(storagerouter, start_port=None, end_port=None, trigger_rerout=True, ip_to_block=None, additional_ports=None): """ Force edge to reroute. Done by blocking all connections to the volumedriver port :param storagerouter: storagerouter object of the node to execute the reroute on :type storagerouter: ovs.dal.hybrids.storagerouter.StorageRouter :param start_port: port to start blocking :type start_port: int/str :param end_port: port to end blocking :type end_port: int/str :param trigger_rerout: trigger or unblock the reroute :type trigger_rerout: bool :param ip_to_block: ip to block connections on :type ip_to_block: str :param additional_ports: additional ports to block outside of the range :type additional_ports: list[int] / list[str] """ if (start_port is None or end_port is None) and ip_to_block is None and additional_ports is None: raise ValueError('Something to block is required. Be it a range, extra ports or an IP') ports = [] if start_port is not None and end_port is not None: if 22 in xrange(int(end_port), int(start_port)): # Avoid blocking ssh raise ValueError('Port 22 cannot be included in the ports.') ports.append("{0}:{1}".format(start_port, end_port)) if isinstance(additional_ports, list): ports.extend([str(item) for item in additional_ports]) # Cast to str cmds = {} if trigger_rerout is True: cmds['input'] = ["iptables", "-I", "INPUT", "1"] # Insert because first rule applies first. cmds['output'] = ["iptables", "-I", "OUTPUT", "1"] else: cmds['input'] = ["iptables", "-D", "INPUT"] cmds['output'] = ["iptables", "-D", "OUTPUT"] EdgeTester.LOGGER.debug('Opening {}'.format(','.join(ports))) if isinstance(ip_to_block, str) or isinstance(ip_to_block, unicode): ip_extra = ['--source', ip_to_block] cmds['input'].extend(ip_extra) cmds['output'].extend(ip_extra) protocol_rule = ["--protocol", "tcp"] cmds['input'].extend(protocol_rule) cmds['output'].extend(protocol_rule) if len(ports) > 0: cmds['input'].extend(["--match", "multiport", "--dport", ','.join(ports)]) cmds['output'].extend(["--match", "multiport", "--sport", ','.join(ports)]) action = ["-j", "DROP"] cmds['input'].extend(action) cmds['output'].extend(action) client = SSHClient(storagerouter, username='******') for rule_key, cmd in cmds.iteritems(): EdgeTester.LOGGER.debug('Executing {0} on {1}'.format(storagerouter.ip, ' '.join(cmd))) client.run(cmd)
def get_logfiles(albanode_guid, local_storagerouter_guid): """ Collects logs, moves them to a web-accessible location and returns log tgz's filename :param albanode_guid: Alba Node guid to retrieve log files on :type albanode_guid: str :param local_storagerouter_guid: Guid of the StorageRouter on which the collect logs was initiated, eg: through the GUI :type local_storagerouter_guid: str :return: Name of tgz containing the logs :rtype: str """ web_path = '/opt/OpenvStorage/webapps/frontend/downloads' alba_node = AlbaNode(albanode_guid) logfile_name = alba_node.client.get_logs()['filename'] download_url = 'https://{0}:{1}@{2}:{3}/downloads/{4}'.format( alba_node.username, alba_node.password, alba_node.ip, alba_node.port, logfile_name) client = SSHClient(endpoint=StorageRouter(local_storagerouter_guid), username='******') client.dir_create(web_path) client.run([ 'wget', download_url, '--directory-prefix', web_path, '--no-check-certificate' ]) client.run(['chmod', '666', '{0}/{1}'.format(web_path, logfile_name)]) return logfile_name
def configure_vpool_for_host(self, vpool_guid, ip): if ( self._is_devstack is False and self._is_openstack is False ) or self._cinder_installed is False or self._nova_installed is False: self._logger.warning( 'Configure vPool: No OpenStack nor DevStack installation detected or Cinder and Nova plugins are not installed' ) return vpool = VPool(vpool_guid) self._logger.debug('configure_vpool {0} started'.format(vpool.name)) # 1. Configure Cinder driver with remote(ip, [RawConfigParser, open], 'root') as rem: changed = False cfg = rem.RawConfigParser() cfg.read([self._CINDER_CONF]) if not cfg.has_section(vpool.name): changed = True cfg.add_section(vpool.name) cfg.set(vpool.name, "volume_driver", "cinder.volume.drivers.openvstorage.OVSVolumeDriver") cfg.set(vpool.name, "volume_backend_name", vpool.name) cfg.set(vpool.name, "vpool_name", vpool.name) enabled_backends = [] if cfg.has_option("DEFAULT", "enabled_backends"): enabled_backends = cfg.get("DEFAULT", "enabled_backends").split(", ") if vpool.name not in enabled_backends: changed = True enabled_backends.append(vpool.name) cfg.set("DEFAULT", "enabled_backends", ", ".join(enabled_backends)) if changed is True: with rem.open(self._CINDER_CONF, "w") as fp: cfg.write(fp) # 2. Create volume type if self.cinder_client and not [ volume_type for volume_type in self.cinder_client.volume_types.list() if volume_type.name == vpool.name ]: volume_type = self.cinder_client.volume_types.create(vpool.name) volume_type.set_keys(metadata={'volume_backend_name': vpool.name}) # 3. Restart processes self.client = SSHClient(ip, username='******') self._restart_processes() self._logger.debug('configure_vpool {0} completed'.format(vpool.name))
def get_current_memory_usage(storagerouter_ip): """ get residential memory usage of a certain storagerouter (through free -m) :param storagerouter_ip: ip address of a existing storagerouter :type storagerouter_ip: str :return: (current usage, max. total usage) :rtype: tuple """ client = SSHClient(storagerouter_ip, username='******') result = client.run( "MEM=$(free -m | tr -s ' ' | grep Mem); " "echo $MEM | cut -d ' ' -f 3; echo $MEM | cut -d ' ' -f 2", allow_insecure=True).split() return int(result[0]), int(result[1])
def _umount(mountpoint, client=None): """ Unmount the given partition :param mountpoint: Location where the mountpoint is mounted :type mountpoint: str :return: """ if client is None: client = SSHClient(System.get_my_storagerouter(), username='******') try: client.run(['umount', mountpoint]) except Exception: RoleRemover.LOGGER.exception( 'Unable to umount mountpoint {0}'.format(mountpoint)) raise RuntimeError('Could not unmount {0}'.format(mountpoint))