def _configure_amqp_to_volumedriver(): Toolbox.log(logger=NodeTypeController._logger, messages='Update existing vPools') login = Configuration.get('/ovs/framework/messagequeue|user') password = Configuration.get('/ovs/framework/messagequeue|password') protocol = Configuration.get('/ovs/framework/messagequeue|protocol') uris = [] for endpoint in Configuration.get( '/ovs/framework/messagequeue|endpoints'): uris.append({ 'amqp_uri': '{0}://{1}:{2}@{3}'.format(protocol, login, password, endpoint) }) if Configuration.dir_exists('/ovs/vpools'): for vpool_guid in Configuration.list('/ovs/vpools'): for storagedriver_id in Configuration.list( '/ovs/vpools/{0}/hosts'.format(vpool_guid)): storagedriver_config = StorageDriverConfiguration( vpool_guid, storagedriver_id) storagedriver_config.configure_event_publisher( events_amqp_routing_key=Configuration.get( '/ovs/framework/messagequeue|queues.storagedriver' ), events_amqp_uris=uris) storagedriver_config.save()
def discover_nodes(cls): # type: () -> Dict[str, AlbaNode] """ Discover nodes by querying the config mgmt :return: The discovered nodes, mapped by their guid :rtype: Dict[str, AlbaNode] """ nodes = {} model_node_ids = set(node.node_id for node in AlbaNodeList.get_albanodes()) found_node_ids = set() node_ids_by_type = {} for node_type, base_config_path in { AlbaNode.NODE_TYPES.ASD: ASD_NODE_BASE_PATH, AlbaNode.NODE_TYPES.S3: S3_NODE_BASE_PATH }.iteritems(): if Configuration.dir_exists(base_config_path): node_ids = Configuration.list(base_config_path) node_ids_by_type[node_type] = node_ids for node_type, node_ids in node_ids_by_type.iteritems(): for node_id in node_ids: if node_id not in model_node_ids and node_id not in found_node_ids: node = cls.model_volatile_node(node_id, node_type) nodes[node.guid] = node found_node_ids.add(node.node_id) return nodes
def get_unused_arakoon_metadata_and_claim(cluster_type, locked=True): """ Retrieve arakoon cluster information based on its type :param cluster_type: Type of arakoon cluster (See ServiceType.ARAKOON_CLUSTER_TYPES) :type cluster_type: str :param locked: Execute this in a locked context :type locked: bool :return: Metadata of the arakoon cluster :rtype: dict """ cluster_type = cluster_type.upper() if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES: raise ValueError('Unsupported arakoon cluster type provided. Please choose from {0}'.format(', '.join(ServiceType.ARAKOON_CLUSTER_TYPES))) if not Configuration.dir_exists(ArakoonInstaller.CONFIG_ROOT): return None mutex = volatile_mutex('claim_arakoon_metadata', wait=10) try: if locked is True: mutex.acquire() for cluster_name in Configuration.list(ArakoonInstaller.CONFIG_ROOT): config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=False) config.load_config() arakoon_client = ArakoonInstaller.build_client(config) if arakoon_client.exists(ArakoonInstaller.METADATA_KEY): metadata = json.loads(arakoon_client.get(ArakoonInstaller.METADATA_KEY)) if metadata['cluster_type'] == cluster_type and metadata['in_use'] is False and metadata['internal'] is False: metadata['in_use'] = True arakoon_client.set(ArakoonInstaller.METADATA_KEY, json.dumps(metadata, indent=4)) return metadata finally: if locked is True: mutex.release()
def _configure_arakoon_to_volumedriver(cluster_name): StorageDriverController._logger.info('Update existing vPools') config = ArakoonClusterConfig(cluster_id=cluster_name) arakoon_nodes = [] for node in config.nodes: arakoon_nodes.append({ 'host': node.ip, 'port': node.client_port, 'node_id': node.name }) if Configuration.dir_exists('/ovs/vpools'): for vpool_guid in Configuration.list('/ovs/vpools'): for storagedriver_id in Configuration.list( '/ovs/vpools/{0}/hosts'.format(vpool_guid)): storagedriver_config = StorageDriverConfiguration( 'storagedriver', vpool_guid, storagedriver_id) storagedriver_config.load() storagedriver_config.configure_volume_registry( vregistry_arakoon_cluster_id=cluster_name, vregistry_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.configure_distributed_lock_store( dls_type='Arakoon', dls_arakoon_cluster_id=cluster_name, dls_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.save()
def _revert_vpool_status(vpool, status=VPool.STATUSES.RUNNING, storagedriver=None, client=None, dirs_created=None): """ Remove the vPool being created or revert the vPool being extended :return: None :rtype: NoneType """ vpool.status = status vpool.save() if status == VPool.STATUSES.RUNNING: if len(dirs_created) > 0: try: client.dir_delete(directories=dirs_created) except Exception: StorageRouterController._logger.warning( 'Failed to clean up following directories: {0}'.format( ', '.join(dirs_created))) if storagedriver is not None: for sdp in storagedriver.partitions: sdp.delete() for proxy in storagedriver.alba_proxies: proxy.delete() storagedriver.delete() if len(vpool.storagedrivers) == 0: vpool.delete() if Configuration.dir_exists( key='/ovs/vpools/{0}'.format(vpool.guid)): Configuration.delete( key='/ovs/vpools/{0}'.format(vpool.guid))
def remove_node(node_guid): """ Removes an ALBA node :param node_guid: Guid of the ALBA node to remove :type node_guid: str :return: None """ node = AlbaNode(node_guid) for disk in node.disks: for osd in disk.osds: AlbaNodeController.remove_asd(node_guid=osd.alba_disk.alba_node_guid, asd_id=osd.osd_id, expected_safety=None) AlbaNodeController.remove_disk(node_guid=disk.alba_node_guid, device_alias=disk.aliases[0]) try: for service_name in node.client.list_maintenance_services(): node.client.remove_maintenance_service(service_name) except (requests.ConnectionError, requests.Timeout): AlbaNodeController._logger.exception('Could not connect to node {0} to retrieve the maintenance services'.format(node.guid)) except InvalidCredentialsError: AlbaNodeController._logger.warning('Failed to retrieve the maintenance services for ALBA node {0}'.format(node.node_id)) if Configuration.dir_exists('/ovs/alba/asdnodes/{0}'.format(node.node_id)): Configuration.delete('/ovs/alba/asdnodes/{0}'.format(node.node_id)) node.delete()
def model_albanodes(**kwargs): """ Add all ALBA nodes known to the config platform to the model :param kwargs: Kwargs containing information regarding the node :type kwargs: dict :return: None :rtype: NoneType """ _ = kwargs if Configuration.dir_exists('/ovs/alba/asdnodes'): for node_id in Configuration.list('/ovs/alba/asdnodes'): node = AlbaNodeList.get_albanode_by_node_id(node_id) if node is None: node = AlbaNode() main_config = Configuration.get( '/ovs/alba/asdnodes/{0}/config/main'.format(node_id)) node.type = 'ASD' node.node_id = node_id node.ip = main_config['ip'] node.port = main_config['port'] node.username = main_config['username'] node.password = main_config['password'] node.storagerouter = StorageRouterList.get_by_ip( main_config['ip']) node.save()
def _get_free_ports(client): node_name = System.get_my_machine_id(client) clusters = [] exclude_ports = [] if Configuration.dir_exists(ArakoonInstaller.CONFIG_ROOT): for cluster_name in Configuration.list(ArakoonInstaller.CONFIG_ROOT): config = ArakoonClusterConfig(cluster_name, False) config.load_config() for node in config.nodes: if node.name == node_name: clusters.append(cluster_name) exclude_ports.append(node.client_port) exclude_ports.append(node.messaging_port) ports = System.get_free_ports(Configuration.get('/ovs/framework/hosts/{0}/ports|arakoon'.format(node_name)), exclude_ports, 2, client) ArakoonInstaller._logger.debug(' Loaded free ports {0} based on existing clusters {1}'.format(ports, clusters)) return ports
def _resave_all_config_entries(config_path='/ovs'): """ Recursive functions which checks every config management key if its a directory or not. If not a directory, we retrieve the config and just save it again using the new indentation logic """ for item in Configuration.list(config_path): new_path = config_path + '/' + item print new_path if Configuration.dir_exists(new_path) is True: _resave_all_config_entries(config_path=new_path) else: try: _config = Configuration.get(new_path) Configuration.set(new_path, _config) except: _config = Configuration.get(new_path, raw=True) Configuration.set(new_path, _config, raw=True)
def _configure_amqp_to_volumedriver(): Toolbox.log(logger=NodeTypeController._logger, messages='Update existing vPools') login = Configuration.get('/ovs/framework/messagequeue|user') password = Configuration.get('/ovs/framework/messagequeue|password') protocol = Configuration.get('/ovs/framework/messagequeue|protocol') uris = [] for endpoint in Configuration.get('/ovs/framework/messagequeue|endpoints'): uris.append({'amqp_uri': '{0}://{1}:{2}@{3}'.format(protocol, login, password, endpoint)}) if Configuration.dir_exists('/ovs/vpools'): for vpool_guid in Configuration.list('/ovs/vpools'): for storagedriver_id in Configuration.list('/ovs/vpools/{0}/hosts'.format(vpool_guid)): storagedriver_config = StorageDriverConfiguration('storagedriver', vpool_guid, storagedriver_id) storagedriver_config.load() storagedriver_config.configure_event_publisher(events_amqp_routing_key=Configuration.get('/ovs/framework/messagequeue|queues.storagedriver'), events_amqp_uris=uris) storagedriver_config.save()
def get_unused_arakoon_metadata_and_claim(cluster_type, locked=True): """ Retrieve arakoon cluster information based on its type :param cluster_type: Type of arakoon cluster (See ServiceType.ARAKOON_CLUSTER_TYPES) :type cluster_type: str :param locked: Execute this in a locked context :type locked: bool :return: Metadata of the arakoon cluster :rtype: dict """ cluster_type = cluster_type.upper() if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES: raise ValueError( 'Unsupported arakoon cluster type provided. Please choose from {0}' .format(', '.join(ServiceType.ARAKOON_CLUSTER_TYPES))) if not Configuration.dir_exists(ArakoonInstaller.CONFIG_ROOT): return None mutex = volatile_mutex('claim_arakoon_metadata', wait=10) try: if locked is True: mutex.acquire() for cluster_name in Configuration.list( ArakoonInstaller.CONFIG_ROOT): config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=False) config.load_config() arakoon_client = ArakoonInstaller.build_client(config) if arakoon_client.exists(ArakoonInstaller.METADATA_KEY): metadata = json.loads( arakoon_client.get(ArakoonInstaller.METADATA_KEY)) if metadata['cluster_type'] == cluster_type and metadata[ 'in_use'] is False and metadata[ 'internal'] is False: metadata['in_use'] = True arakoon_client.set(ArakoonInstaller.METADATA_KEY, json.dumps(metadata, indent=4)) return metadata finally: if locked is True: mutex.release()
def _configure_arakoon_to_volumedriver(cluster_name): StorageDriverController._logger.info('Update existing vPools') config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=False) config.load_config() arakoon_nodes = [] for node in config.nodes: arakoon_nodes.append({'host': node.ip, 'port': node.client_port, 'node_id': node.name}) if Configuration.dir_exists('/ovs/vpools'): for vpool_guid in Configuration.list('/ovs/vpools'): for storagedriver_id in Configuration.list('/ovs/vpools/{0}/hosts'.format(vpool_guid)): storagedriver_config = StorageDriverConfiguration('storagedriver', vpool_guid, storagedriver_id) storagedriver_config.load() storagedriver_config.configure_volume_registry(vregistry_arakoon_cluster_id=cluster_name, vregistry_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.configure_distributed_lock_store(dls_type='Arakoon', dls_arakoon_cluster_id=cluster_name, dls_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.save(reload_config=True)
def model_albanodes(**kwargs): """ Add all ALBA nodes known to the config platform to the model :param kwargs: Kwargs containing information regarding the node :type kwargs: dict :return: None """ _ = kwargs if Configuration.dir_exists('/ovs/alba/asdnodes'): for node_id in Configuration.list('/ovs/alba/asdnodes'): node = AlbaNodeList.get_albanode_by_node_id(node_id) if node is None: node = AlbaNode() main_config = Configuration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id)) node.type = 'ASD' node.node_id = node_id node.ip = main_config['ip'] node.port = main_config['port'] node.username = main_config['username'] node.password = main_config['password'] node.storagerouter = StorageRouterList.get_by_ip(main_config['ip']) node.save()
def _get_free_ports(client): node_name = System.get_my_machine_id(client) clusters = [] exclude_ports = [] if Configuration.dir_exists(ArakoonInstaller.CONFIG_ROOT): for cluster_name in Configuration.list( ArakoonInstaller.CONFIG_ROOT): config = ArakoonClusterConfig(cluster_name, False) config.load_config() for node in config.nodes: if node.name == node_name: clusters.append(cluster_name) exclude_ports.append(node.client_port) exclude_ports.append(node.messaging_port) ports = System.get_free_ports( Configuration.get( '/ovs/framework/hosts/{0}/ports|arakoon'.format(node_name)), exclude_ports, 2, client) ArakoonInstaller._logger.debug( ' Loaded free ports {0} based on existing clusters {1}'.format( ports, clusters)) return ports
def revert_vpool(self, status): """ Remove the vPool being created or revert the vPool being extended :param status: Status to put the vPool in :type status: ovs.dal.hybrids.vpool.VPool.STATUSES :return: None :rtype: NoneType """ self.vpool.status = status self.vpool.save() if status == VPool.STATUSES.RUNNING: if self.sr_installer is not None: try: self.sr_installer.root_client.dir_delete( directories=self.sr_installer.created_dirs) except Exception: self._logger.warning( 'Failed to clean up following directories: {0}'.format( ', '.join(self.sr_installer.created_dirs))) if self.sd_installer is not None and self.sd_installer.storagedriver is not None: for sdp in self.sd_installer.storagedriver.partitions: sdp.delete() for proxy in self.sd_installer.storagedriver.alba_proxies: proxy.delete() self.sd_installer.storagedriver.delete() if len(self.vpool.storagedrivers) == 0: self.vpool.delete() if Configuration.dir_exists( key='/ovs/vpools/{0}'.format(self.vpool.guid)): Configuration.delete( key='/ovs/vpools/{0}'.format(self.vpool.guid)) elif status == VPool.STATUSES.FAILURE: # In case of failure status the cluster registry settings have already been adapted, so revert self.configure_cluster_registry( exclude=[self.sd_installer.storagedriver])
def create_cluster(cluster_name, cluster_type, ip, base_dir, plugins=None, locked=True, internal=True, filesystem=False, ports=None): """ Always creates a cluster but marks it's usage according to the internal flag :param cluster_name: Name of the cluster :type cluster_name: str :param cluster_type: Type of the cluster (See ServiceType.ARAKOON_CLUSTER_TYPES) :type cluster_type: str :param ip: IP address of the first node of the new cluster :type ip: str :param base_dir: Base directory that should contain the data and tlogs :type base_dir: str :param plugins: Plugins that should be added to the configuration file :type plugins: list :param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts) :type locked: bool :param internal: Is cluster internally managed by OVS :type internal: bool :param filesystem: Indicates whether the configuration should be on the filesystem or in a configuration cluster :type filesystem: bool :param ports: A list of ports to be used for this cluster's node :type ports: list :return: Ports used by arakoon cluster :rtype: dict """ if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES: raise ValueError( 'Cluster type {0} is not supported. Please choose from {1}'. format(cluster_type, ', '.join(ServiceType.ARAKOON_CLUSTER_TYPES))) client = SSHClient(ip, username=ArakoonInstaller.SSHCLIENT_USER) if filesystem is True: exists = client.file_exists( ArakoonClusterConfig.CONFIG_FILE.format(cluster_name)) else: exists = Configuration.dir_exists( '/ovs/arakoon/{0}'.format(cluster_name)) if exists is True: raise ValueError( 'An Arakoon cluster with name "{0}" already exists'.format( cluster_name)) ArakoonInstaller._logger.debug('Creating cluster {0} on {1}'.format( cluster_name, ip)) node_name = System.get_my_machine_id(client) base_dir = base_dir.rstrip('/') home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format( base_dir, cluster_name) tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format( base_dir, cluster_name) ArakoonInstaller.clean_leftover_arakoon_data(ip, [home_dir, tlog_dir]) port_mutex = None try: if locked is True: from ovs.extensions.generic.volatilemutex import volatile_mutex port_mutex = volatile_mutex( 'arakoon_install_ports_{0}'.format(ip)) port_mutex.acquire(wait=60) if ports is None: ports = ArakoonInstaller._get_free_ports(client) config = ArakoonClusterConfig(cluster_name, filesystem, plugins) config.nodes.append( ArakoonNodeConfig( name=node_name, ip=ip, client_port=ports[0], messaging_port=ports[1], log_sinks=LogHandler.get_sink_path('arakoon_server'), crash_log_sinks=LogHandler.get_sink_path( 'arakoon_server_crash'), home=home_dir, tlog_dir=tlog_dir)) metadata = { 'internal': internal, 'cluster_name': cluster_name, 'cluster_type': cluster_type.upper(), 'in_use': False } ArakoonInstaller._deploy(config, filesystem=filesystem) finally: if port_mutex is not None: port_mutex.release() ArakoonInstaller._logger.debug( 'Creating cluster {0} on {1} completed'.format(cluster_name, ip)) return { 'metadata': metadata, 'client_port': ports[0], 'messaging_port': ports[1] }
def create_cluster(cluster_name, cluster_type, ip, base_dir, plugins=None, locked=True, internal=True, filesystem=False, ports=None): """ Always creates a cluster but marks it's usage according to the internal flag :param cluster_name: Name of the cluster :type cluster_name: str :param cluster_type: Type of the cluster (See ServiceType.ARAKOON_CLUSTER_TYPES) :type cluster_type: str :param ip: IP address of the first node of the new cluster :type ip: str :param base_dir: Base directory that should contain the data and tlogs :type base_dir: str :param plugins: Plugins that should be added to the configuration file :type plugins: dict :param locked: Indicates whether the create should run in a locked context (e.g. to prevent port conflicts) :type locked: bool :param internal: Is cluster internally managed by OVS :type internal: bool :param filesystem: Indicates whether the configuration should be on the filesystem or in a configuration cluster :type filesystem: bool :param ports: A list of ports to be used for this cluster's node :type ports: list :return: Ports used by arakoon cluster :rtype: dict """ if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES: raise ValueError('Cluster type {0} is not supported. Please choose from {1}'.format(cluster_type, ', '.join(ServiceType.ARAKOON_CLUSTER_TYPES))) client = SSHClient(ip, username=ArakoonInstaller.SSHCLIENT_USER) if filesystem is True: exists = client.file_exists(ArakoonClusterConfig.CONFIG_FILE.format(cluster_name)) else: exists = Configuration.dir_exists('/ovs/arakoon/{0}'.format(cluster_name)) if exists is True: raise ValueError('An Arakoon cluster with name "{0}" already exists'.format(cluster_name)) ArakoonInstaller._logger.debug('Creating cluster {0} on {1}'.format(cluster_name, ip)) node_name = System.get_my_machine_id(client) base_dir = base_dir.rstrip('/') home_dir = ArakoonInstaller.ARAKOON_HOME_DIR.format(base_dir, cluster_name) tlog_dir = ArakoonInstaller.ARAKOON_TLOG_DIR.format(base_dir, cluster_name) ArakoonInstaller.clean_leftover_arakoon_data(ip, [home_dir, tlog_dir]) port_mutex = None try: if locked is True: from ovs.extensions.generic.volatilemutex import volatile_mutex port_mutex = volatile_mutex('arakoon_install_ports_{0}'.format(ip)) port_mutex.acquire(wait=60) if ports is None: ports = ArakoonInstaller._get_free_ports(client) config = ArakoonClusterConfig(cluster_name, filesystem, plugins.keys() if plugins is not None else None) config.nodes.append(ArakoonNodeConfig(name=node_name, ip=ip, client_port=ports[0], messaging_port=ports[1], log_sinks=LogHandler.get_sink_path('arakoon_server'), crash_log_sinks=LogHandler.get_sink_path('arakoon_server_crash'), home=home_dir, tlog_dir=tlog_dir)) metadata = {'internal': internal, 'cluster_name': cluster_name, 'cluster_type': cluster_type.upper(), 'in_use': False} service_metadata = ArakoonInstaller._deploy(config=config, filesystem=filesystem, plugins=plugins.values() if plugins is not None else None, delay_service_registration=cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.CFG)[ip] finally: if port_mutex is not None: port_mutex.release() ArakoonInstaller._logger.debug('Creating cluster {0} on {1} completed'.format(cluster_name, ip)) return {'metadata': metadata, 'client_port': ports[0], 'messaging_port': ports[1], 'service_metadata': service_metadata}
def migrate(previous_version, master_ips=None, extra_ips=None): """ Migrates from a given version to the current version. It uses 'previous_version' to be smart wherever possible, but the code should be able to migrate any version towards the expected version. When this is not possible, the code can set a minimum version and raise when it is not met. :param previous_version: The previous version from which to start the migration :type previous_version: float :param master_ips: IP addresses of the MASTER nodes :type master_ips: list or None :param extra_ips: IP addresses of the EXTRA nodes :type extra_ips: list or None """ _ = master_ips, extra_ips working_version = previous_version # From here on, all actual migration should happen to get to the expected state for THIS RELEASE if working_version < ExtensionMigrator.THIS_VERSION: try: from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.dal.lists.vpoollist import VPoolList from ovs.extensions.generic.configuration import Configuration from ovs.extensions.services.servicefactory import ServiceFactory from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.generic.system import System local_machine_id = System.get_my_machine_id() local_ip = Configuration.get( '/ovs/framework/hosts/{0}/ip'.format(local_machine_id)) local_client = SSHClient(endpoint=local_ip, username='******') # Multiple Proxies if local_client.dir_exists( directory= '/opt/OpenvStorage/config/storagedriver/storagedriver' ): local_client.dir_delete(directories=[ '/opt/OpenvStorage/config/storagedriver/storagedriver' ]) # MDS safety granularity on vPool level mds_safety_key = '/ovs/framework/storagedriver' if Configuration.exists(key=mds_safety_key): current_mds_settings = Configuration.get( key=mds_safety_key) for vpool in VPoolList.get_vpools(): vpool_key = '/ovs/vpools/{0}'.format(vpool.guid) if Configuration.dir_exists(key=vpool_key): Configuration.set( key='{0}/mds_config'.format(vpool_key), value=current_mds_settings) Configuration.delete(key=mds_safety_key) # Introduction of edition key if Configuration.get(key=Configuration.EDITION_KEY, default=None) not in [ PackageFactory.EDITION_COMMUNITY, PackageFactory.EDITION_ENTERPRISE ]: for storagerouter in StorageRouterList.get_storagerouters( ): try: Configuration.set( key=Configuration.EDITION_KEY, value=storagerouter.features['alba'] ['edition']) break except: continue except: ExtensionMigrator._logger.exception( 'Error occurred while executing the migration code') # Don't update migration version with latest version, resulting in next migration trying again to execute this code return ExtensionMigrator.THIS_VERSION - 1 return ExtensionMigrator.THIS_VERSION
def list(self, discover=False, ip=None, node_id=None): """ Lists all available ALBA Nodes :param discover: If True and IP provided, return list of single ALBA node, If True and no IP provided, return all ALBA nodes else return modeled ALBA nodes :type discover: bool :param ip: IP of ALBA node to retrieve :type ip: str :param node_id: ID of the ALBA node :type node_id: str """ if discover is False and (ip is not None or node_id is not None): raise HttpNotAcceptableException(error_description='Discover is mutually exclusive with IP and nodeID', error='invalid_data') if (ip is None and node_id is not None) or (ip is not None and node_id is None): raise HttpNotAcceptableException(error_description='Both IP and nodeID need to be specified', error='invalid_data') if discover is False: return AlbaNodeList.get_albanodes() if ip is not None: node = AlbaNode(volatile=True) node.ip = ip node.type = 'ASD' node.node_id = node_id node.port = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id)) node.username = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id)) node.password = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id)) data = node.client.get_metadata() if data['_success'] is False and data['_error'] == 'Invalid credentials': raise HttpNotAcceptableException(error_description='Invalid credentials', error='invalid_data') if data['node_id'] != node_id: raise HttpNotAcceptableException(error_description='Unexpected node identifier. {0} vs {1}'.format(data['node_id'], node_id), error='invalid_data') node_list = DataList(AlbaNode, {}) node_list._executed = True node_list._guids = [node.guid] node_list._objects = {node.guid: node} node_list._data = {node.guid: {'guid': node.guid, 'data': node._data}} return node_list nodes = {} model_node_ids = [node.node_id for node in AlbaNodeList.get_albanodes()] found_node_ids = [] asd_node_ids = [] if Configuration.dir_exists('/ovs/alba/asdnodes'): asd_node_ids = Configuration.list('/ovs/alba/asdnodes') for node_id in asd_node_ids: node = AlbaNode(volatile=True) node.type = 'ASD' node.node_id = node_id node.ip = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|ip'.format(node_id)) node.port = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|port'.format(node_id)) node.username = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|username'.format(node_id)) node.password = Configuration.get('/ovs/alba/asdnodes/{0}/config/main|password'.format(node_id)) if node.node_id not in model_node_ids and node.node_id not in found_node_ids: nodes[node.guid] = node found_node_ids.append(node.node_id) node_list = DataList(AlbaNode, {}) node_list._executed = True node_list._guids = nodes.keys() node_list._objects = nodes node_list._data = dict([(node.guid, {'guid': node.guid, 'data': node._data}) for node in nodes.values()]) return node_list
def validate_alba_backend_sanity_without_claimed_disks(alba_backend): """ Validate whether the ALBA backend is configured correctly :param alba_backend: ALBA backend :return: None """ # Attribute validation assert alba_backend.available is True,\ 'ALBA backend {0} is not available'.format(alba_backend.backend.name) assert len(alba_backend.presets) >= 1,\ 'No preset found for ALBA backend {0}'.format(alba_backend.backend.name) assert len([default for default in alba_backend.presets if default['is_default'] is True]) == 1,\ 'Could not find default preset for backend {0}'.format(alba_backend.backend.name) assert alba_backend.backend.backend_type.code == 'alba',\ 'Backend type for ALBA backend is {0}'.format(alba_backend.backend.backend_type.code) assert alba_backend.backend.status == 'RUNNING',\ 'Status for ALBA backend is {0}'.format(alba_backend.backend.status) # Validate ABM and NSM services storagerouters = GeneralStorageRouter.get_storage_routers() storagerouters_with_db_role = [sr for sr in storagerouters if GeneralStorageRouter.has_roles(storagerouter=sr, roles='DB') is True and sr.node_type == 'MASTER'] assert len(alba_backend.abm_services) == len(storagerouters_with_db_role),\ 'Not enough ABM services found' assert len(alba_backend.nsm_services) == len(storagerouters_with_db_role),\ 'Not enough NSM services found' # Validate ALBA backend configuration structure alba_backend_key = '/ovs/alba/backends' assert Configuration.dir_exists(key=alba_backend_key) is True,\ 'Configuration does not contain key {0}'.format(alba_backend_key) actual_config_keys = [key for key in Configuration.list(alba_backend_key)] expected_config_keys = ['global_gui_error_interval', alba_backend.guid, 'default_nsm_hosts'] optional_config_keys = ['verification_factor'] expected_keys_amount = 0 for optional_key in optional_config_keys: if optional_key in actual_config_keys: expected_keys_amount += 1 for expected_key in expected_config_keys: if not re.match(Toolbox.regex_guid, expected_key): expected_keys_amount += 1 assert expected_key in actual_config_keys,\ 'Key {0} was not found in tree {1}'.format(expected_key, alba_backend_key) for actual_key in list(actual_config_keys): if re.match(Toolbox.regex_guid, actual_key): actual_config_keys.remove(actual_key) # Remove all alba backend keys assert len(actual_config_keys) == expected_keys_amount,\ 'Another key was added to the {0} tree'.format(alba_backend_key) this_alba_backend_key = '{0}/{1}'.format(alba_backend_key, alba_backend.guid) actual_keys = [key for key in Configuration.list(this_alba_backend_key)] expected_keys = ['maintenance'] assert actual_keys == expected_keys,\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_keys, expected_keys) maintenance_key = '{0}/maintenance'.format(this_alba_backend_key) actual_keys = [key for key in Configuration.list(maintenance_key)] expected_keys = ['nr_of_agents', 'config'] assert set(actual_keys) == set(expected_keys),\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_keys, expected_keys) # @TODO: Add validation for config values # Validate ASD node configuration structure alba_nodes = GeneralAlba.get_alba_nodes() assert len(alba_nodes) > 0,\ 'Could not find any ALBA nodes in the model' alba_node_key = '/ovs/alba/asdnodes' actual_keys = [key for key in Configuration.list(alba_node_key)] assert len(alba_nodes) == len(actual_keys),\ 'Amount of ALBA nodes in model: {0} >< amount of ALBA nodes in configuration: {1}.'.format(len(alba_nodes), len(actual_keys)) for alba_node in alba_nodes: assert alba_node.node_id in actual_keys,\ 'ALBA node with ID {0} not present in configuration'.format(alba_node.node_id) actual_asdnode_keys = [key for key in Configuration.list('{0}/{1}'.format(alba_node_key, alba_node.node_id))] expected_asdnode_keys = ['config', 'services'] assert actual_asdnode_keys == expected_asdnode_keys,\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_asdnode_keys, expected_asdnode_keys) actual_config_keys = [key for key in Configuration.list('{0}/{1}/config'.format(alba_node_key, alba_node.node_id))] expected_config_keys = ['main', 'network'] assert set(actual_config_keys) == set(expected_config_keys),\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_config_keys, expected_config_keys) # @TODO: Add validation for main and network values # Validate Arakoon configuration structure arakoon_abm_key = '/ovs/arakoon/{0}/config'.format(alba_backend.abm_services[0].service.name).replace('arakoon-', '') arakoon_nsm_key = '/ovs/arakoon/{0}/config'.format(alba_backend.nsm_services[0].service.name).replace('arakoon-', '') assert Configuration.exists(key=arakoon_abm_key, raw=True) is True,\ 'Configuration key {0} does not exist'.format(arakoon_abm_key) assert Configuration.exists(key=arakoon_nsm_key, raw=True) is True,\ 'Configuration key {0} does not exist'.format(arakoon_nsm_key) # @TODO: Add validation for config values # Validate maintenance agents actual_amount_agents = len([service for node_services in [alba_node.client.list_maintenance_services() for alba_node in alba_nodes] for service in node_services]) expected_amount_agents = 1 assert actual_amount_agents == expected_amount_agents,\ 'Amount of maintenance agents is incorrect. Found {0} - Expected {1}'.format(actual_amount_agents, expected_amount_agents) # Validate arakoon services machine_ids = [sr.machine_id for sr in storagerouters_with_db_role] abm_service_name = alba_backend.abm_services[0].service.name nsm_service_name = alba_backend.nsm_services[0].service.name for storagerouter in storagerouters_with_db_role: root_client = SSHClient(endpoint=storagerouter, username='******') for service_name in [abm_service_name, nsm_service_name]: assert GeneralService.has_service(name=service_name, client=root_client) is True,\ 'Service {0} not deployed on Storage Router {1}'.format(service_name, storagerouter.name) exitcode, output = GeneralService.get_service_status(name=service_name, client=root_client) assert exitcode is True,\ 'Service {0} not running on Storage Router {1} - {2}'.format(service_name, storagerouter.name, output) out, err, _ = General.execute_command('arakoon --who-master -config {0}'.format(Configuration.get_configuration_path('/ovs/arakoon/{0}/config'.format(abm_service_name.replace('arakoon-', ''))))) assert out.strip() in machine_ids,\ 'Arakoon master is {0}, but should be 1 of "{1}"'.format(out.strip(), ', '.join(machine_ids))