def save(self, client=None, reload_config=True): """ Saves the configuration to a given file, optionally a remote one :param client: If provided, save remote configuration :param reload_config: Reload the running Storage Driver configuration """ Configuration.set(self.key, json.dumps(self.configuration, indent=4), raw=True) if self.config_type == 'storagedriver' and reload_config is True: if len(self.dirty_entries) > 0: if client is None: self._logger.info('Applying local storagedriver configuration changes') changes = LocalStorageRouterClient(self.remote_path).update_configuration(self.remote_path) else: self._logger.info('Applying storagedriver configuration changes on {0}'.format(client.ip)) with remote(client.ip, [LocalStorageRouterClient]) as rem: changes = copy.deepcopy(rem.LocalStorageRouterClient(self.remote_path).update_configuration(self.remote_path)) for change in changes: if change['param_name'] not in self.dirty_entries: raise RuntimeError('Unexpected configuration change: {0}'.format(change['param_name'])) self._logger.info('Changed {0} from "{1}" to "{2}"'.format(change['param_name'], change['old_value'], change['new_value'])) self.dirty_entries.remove(change['param_name']) self._logger.info('Changes applied') if len(self.dirty_entries) > 0: self._logger.warning('Following changes were not applied: {0}'.format(', '.join(self.dirty_entries))) else: self._logger.debug('No need to apply changes, nothing changed') self.is_new = False self.dirty_entries = []
def is_service_internally_managed(service): """ Validate whether the service is internally or externally managed :param service: Service to verify :type service: str :return: True if internally managed, False otherwise :rtype: bool """ if service not in ['memcached', 'rabbitmq']: raise ValueError('Can only check memcached or rabbitmq') service_name_map = {'memcached': 'memcache', 'rabbitmq': 'messagequeue'}[service] config_key = '/ovs/framework/{0}'.format(service_name_map) if not Configuration.exists(key=config_key): return True if not Configuration.exists(key='{0}|metadata'.format(config_key)): raise ValueError('Not all required keys ({0}) for {1} are present in the configuration management'.format(config_key, service)) metadata = Configuration.get('{0}|metadata'.format(config_key)) if 'internal' not in metadata: raise ValueError('Internal flag not present in metadata for {0}.\nPlease provide a key: {1} and value "metadata": {{"internal": True/False}}'.format(service, config_key)) internal = metadata['internal'] if internal is False: if not Configuration.exists(key='{0}|endpoints'.format(config_key)): raise ValueError('Externally managed {0} cluster must have "endpoints" information\nPlease provide a key: {1} and value "endpoints": [<ip:port>]'.format(service, config_key)) endpoints = Configuration.get(key='{0}|endpoints'.format(config_key)) if not isinstance(endpoints, list) or len(endpoints) == 0: raise ValueError('The endpoints for {0} cannot be empty and must be a list'.format(service)) return internal
def setUpClass(cls): ArakoonInstaller.ARAKOON_CONFIG_DIR = '/tmp/cfg' ArakoonInstaller.ARAKOON_CONFIG_FILE = '/tmp/cfg/{0}/{0}.cfg' TestArakoonInstaller.expected_global = '[global]\ncluster_id = {0}\ncluster = {1}\nplugins = \n\n' TestArakoonInstaller.expected_base = '[{0}]\nname = {0}\nip = {1}\nclient_port = {2}\nmessaging_port = {3}\ntlog_compression = snappy\nlog_level = info\nlog_dir = /var/log/arakoon/one\nhome = /tmp/db/arakoon/one\ntlog_dir = /tmp/db/tlogs/one\nfsync = true\n\n' # System def _get_my_machine_id(_client): return TestArakoonInstaller.nodes[_client.ip] System.get_my_machine_id = staticmethod(_get_my_machine_id) # Configuration def _get(key): if key == 'ovs.core.storage.persistent': return 'arakoon' c = PersistentFactory.get_client() if c.exists(key): return c.get(key) return None def _get_int(key): return int(Configuration.get(key)) def _set(key, value): c = PersistentFactory.get_client() c.set(key, value) Configuration.get = staticmethod(_get) Configuration.get_int = staticmethod(_get_int) Configuration.set = staticmethod(_set) Configuration.set('ovs.ports.arakoon', [22000]) Configuration.set('ovs.arakoon.location', '/tmp/db')
def remove_node(node_guid): """ Removes an ALBA node :param node_guid: Guid of the ALBA node to remove :type node_guid: str :return: None """ node = AlbaNode(node_guid) for disk in node.disks: for osd in disk.osds: AlbaNodeController.remove_asd(node_guid=osd.alba_disk.alba_node_guid, asd_id=osd.osd_id, expected_safety=None) AlbaNodeController.remove_disk(node_guid=disk.alba_node_guid, device_alias=disk.aliases[0]) try: for service_name in node.client.list_maintenance_services(): node.client.remove_maintenance_service(service_name) except (requests.ConnectionError, requests.Timeout): AlbaNodeController._logger.exception('Could not connect to node {0} to retrieve the maintenance services'.format(node.guid)) except InvalidCredentialsError: AlbaNodeController._logger.warning('Failed to retrieve the maintenance services for ALBA node {0}'.format(node.node_id)) if Configuration.dir_exists('/ovs/alba/asdnodes/{0}'.format(node.node_id)): Configuration.delete('/ovs/alba/asdnodes/{0}'.format(node.node_id)) node.delete()
def update_configuration(self, path): """ Update configuration mock """ from ovs.extensions.generic.configuration import Configuration if path != self.path: raise RuntimeError( 'Unexpected path passed. Not an issue, but unexpected. This (unittest) code might need to be adapted.' ) main_key = Configuration.extract_key_from_path(path) current_content = LocalStorageRouterClient.configurations.get( main_key, {}) new_content = json.loads(Configuration.get(main_key, raw=True)) changes = [] for section_key, section in new_content.iteritems(): current_section = current_content.get(section_key, {}) for key, value in section.iteritems(): current_value = current_section.get(key) if current_section.get(key) != value: changes.append({ 'param_name': key, 'old_value': current_value, 'new_value': value }) LocalStorageRouterClient.configurations[main_key] = new_content return changes
def get_support_info(): """ Returns support information for the entire cluster :return: Support information :rtype: dict """ celery_scheduling = Configuration.get( key='/ovs/framework/scheduling/celery', default={}) stats_monkey_disabled = 'ovs.stats_monkey.run_all' in celery_scheduling and celery_scheduling[ 'ovs.stats_monkey.run_all'] is None stats_monkey_disabled &= 'alba.stats_monkey.run_all' in celery_scheduling and celery_scheduling[ 'alba.stats_monkey.run_all'] is None return { 'cluster_id': Configuration.get(key='/ovs/framework/cluster_id'), 'stats_monkey': not stats_monkey_disabled, 'support_agent': Configuration.get(key='/ovs/framework/support|support_agent'), 'remote_access': Configuration.get(key='ovs/framework/support|remote_access'), 'stats_monkey_config': Configuration.get(key='ovs/framework/monitoring/stats_monkey', default={}) }
def get_client(client_type=None): """ Returns a volatile storage client """ if not hasattr(VolatileFactory, 'store') or VolatileFactory.store is None: if client_type is None: client_type = Configuration.get('ovs.core.storage.volatile') VolatileFactory.store = None if client_type == 'memcache': from ovs.extensions.storage.volatile.memcachestore import MemcacheStore memcache_servers = list() memcache_config = RawConfigParser() memcache_config.read(os.path.join(Configuration.get('ovs.core.cfgdir'), 'memcacheclient.cfg')) nodes = [node.strip() for node in memcache_config.get('main', 'nodes').split(',')] nodes.sort() for node in nodes: location = memcache_config.get(node, 'location') memcache_servers.append(location) VolatileFactory.store = MemcacheStore(memcache_servers) if client_type == 'default': from ovs.extensions.storage.volatile.dummystore import DummyVolatileStore VolatileFactory.store = DummyVolatileStore() if VolatileFactory.store is None: raise RuntimeError('Invalid client_type specified') return VolatileFactory.store
def migrate(master_ips=None, extra_ips=None): """ Executes all migrations. It keeps track of an internal "migration version" which is always increasing by one :param master_ips: IP addresses of the MASTER nodes :param extra_ips: IP addresses of the EXTRA nodes """ machine_id = System.get_my_machine_id() key = '/ovs/framework/hosts/{0}/versions'.format(machine_id) data = Configuration.get(key) if Configuration.exists(key) else {} migrators = [] path = '/'.join([os.path.dirname(__file__), 'migration']) for filename in os.listdir(path): if os.path.isfile('/'.join([path, filename])) and filename.endswith('.py'): name = filename.replace('.py', '') module = imp.load_source(name, '/'.join([path, filename])) for member in inspect.getmembers(module): if inspect.isclass(member[1]) and member[1].__module__ == name and 'object' in [base.__name__ for base in member[1].__bases__]: migrators.append((member[1].identifier, member[1].migrate)) end_version = 0 for identifier, method in migrators: base_version = data[identifier] if identifier in data else 0 version = method(base_version, master_ips, extra_ips) if version > end_version: end_version = version data[identifier] = end_version Configuration.set(key, data)
def _revert_vpool_status(vpool, status=VPool.STATUSES.RUNNING, storagedriver=None, client=None, dirs_created=None): """ Remove the vPool being created or revert the vPool being extended :return: None :rtype: NoneType """ vpool.status = status vpool.save() if status == VPool.STATUSES.RUNNING: if len(dirs_created) > 0: try: client.dir_delete(directories=dirs_created) except Exception: StorageRouterController._logger.warning( 'Failed to clean up following directories: {0}'.format( ', '.join(dirs_created))) if storagedriver is not None: for sdp in storagedriver.partitions: sdp.delete() for proxy in storagedriver.alba_proxies: proxy.delete() storagedriver.delete() if len(vpool.storagedrivers) == 0: vpool.delete() if Configuration.dir_exists( key='/ovs/vpools/{0}'.format(vpool.guid)): Configuration.delete( key='/ovs/vpools/{0}'.format(vpool.guid))
def get_unused_arakoon_metadata_and_claim(cluster_type, locked=True): """ Retrieve arakoon cluster information based on its type :param cluster_type: Type of arakoon cluster (See ServiceType.ARAKOON_CLUSTER_TYPES) :type cluster_type: str :param locked: Execute this in a locked context :type locked: bool :return: Metadata of the arakoon cluster :rtype: dict """ cluster_type = cluster_type.upper() if cluster_type not in ServiceType.ARAKOON_CLUSTER_TYPES: raise ValueError('Unsupported arakoon cluster type provided. Please choose from {0}'.format(', '.join(ServiceType.ARAKOON_CLUSTER_TYPES))) if not Configuration.dir_exists(ArakoonInstaller.CONFIG_ROOT): return None mutex = volatile_mutex('claim_arakoon_metadata', wait=10) try: if locked is True: mutex.acquire() for cluster_name in Configuration.list(ArakoonInstaller.CONFIG_ROOT): config = ArakoonClusterConfig(cluster_id=cluster_name, filesystem=False) config.load_config() arakoon_client = ArakoonInstaller.build_client(config) if arakoon_client.exists(ArakoonInstaller.METADATA_KEY): metadata = json.loads(arakoon_client.get(ArakoonInstaller.METADATA_KEY)) if metadata['cluster_type'] == cluster_type and metadata['in_use'] is False and metadata['internal'] is False: metadata['in_use'] = True arakoon_client.set(ArakoonInstaller.METADATA_KEY, json.dumps(metadata, indent=4)) return metadata finally: if locked is True: mutex.release()
def override_scheduletasks(configuration): """ Override the scheduled tasks crontab with your own confguration :param configuration: configuration to override scheduled tasks :type configuration: dict :return: """ service_name = 'ovs-watcher-framework' Configuration.set(CelerySetup.SCHEDULED_TASK_CFG, configuration) fetched_cfg = Configuration.get(CelerySetup.SCHEDULED_TASK_CFG, configuration) if cmp(fetched_cfg, configuration) == 0: # restart ovs-watcher-framework on all nodes for sr_ip in StoragerouterHelper.get_storagerouter_ips(): client = SSHClient(sr_ip, username='******') service_manager = ServiceFactory.get_manager() try: service_manager.restart_service(service_name, client) except: return False CelerySetup.LOGGER.info( "Successfully restarted all `{0}` services!".format( service_name)) return True else: CelerySetup.LOGGER.warning( "`{0}` config is `{1}` but should be `{2}`".format( CelerySetup.SCHEDULED_TASK_CFG, fetched_cfg, configuration)) return False
def get_client(client_type=None): """ Returns a persistent storage client :param client_type: Type of store client """ if not hasattr(PersistentFactory, 'store') or PersistentFactory.store is None: if os.environ.get('RUNNING_UNITTESTS') == 'True': client_type = 'dummy' if client_type is None: client_type = Configuration.get( '/ovs/framework/stores|persistent') PersistentFactory.store = None if client_type in ['pyrakoon', 'arakoon']: from ovs.extensions.storage.persistent.pyrakoonstore import PyrakoonStore PersistentFactory.store = PyrakoonStore( str( Configuration.get( '/ovs/framework/arakoon_clusters|ovsdb'))) if client_type == 'dummy': from ovs.extensions.storage.persistent.dummystore import DummyPersistentStore PersistentFactory.store = DummyPersistentStore() if PersistentFactory.store is None: raise RuntimeError('Invalid client_type specified') return PersistentFactory.store
def _clean(): volatile = VolatileFactory.get_client() persistent = PersistentFactory.get_client() # noinspection PyProtectedMember volatile._clean() # noinspection PyProtectedMember persistent._clean() # noinspection PyProtectedMember SSHClient._clean() # noinspection PyProtectedMember SystemdMock._clean() # noinspection PyProtectedMember MDSClient._clean() # noinspection PyProtectedMember Decorators._clean() # noinspection PyProtectedMember MockedSSHClient._clean() # noinspection PyProtectedMember StorageRouterClient._clean() Logger._logs = {} DataList._test_hooks = {} Toolbox._function_pointers = {} # Clean underlying persistent store Configuration.get_client()._clean() for file_name in glob.glob( ArakoonClusterConfig.CONFIG_FILE.format('unittest*')): os.remove(file_name) for full_path in glob.glob(DalHelper.UNITTEST_DIR.format('*')): shutil.rmtree(full_path) return volatile, persistent
def _configure_arakoon_to_volumedriver(cluster_name): StorageDriverController._logger.info('Update existing vPools') config = ArakoonClusterConfig(cluster_id=cluster_name) arakoon_nodes = [] for node in config.nodes: arakoon_nodes.append({ 'host': node.ip, 'port': node.client_port, 'node_id': node.name }) if Configuration.dir_exists('/ovs/vpools'): for vpool_guid in Configuration.list('/ovs/vpools'): for storagedriver_id in Configuration.list( '/ovs/vpools/{0}/hosts'.format(vpool_guid)): storagedriver_config = StorageDriverConfiguration( 'storagedriver', vpool_guid, storagedriver_id) storagedriver_config.load() storagedriver_config.configure_volume_registry( vregistry_arakoon_cluster_id=cluster_name, vregistry_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.configure_distributed_lock_store( dls_type='Arakoon', dls_arakoon_cluster_id=cluster_name, dls_arakoon_cluster_nodes=arakoon_nodes) storagedriver_config.save()
def migrate(previous_version): """ Migrates from any version to any version, running all migrations required If previous_version is for example 0 and this script is at verison 3 it will execute two steps: - 1 > 2 - 2 > 3 @param previous_version: The previous version from which to start the migration. """ working_version = previous_version # Version 1 introduced: # - Flexible SSD layout if working_version < 1: from ovs.extensions.generic.configuration import Configuration if Configuration.exists('ovs.arakoon'): Configuration.delete('ovs.arakoon', remove_root=True) Configuration.set('ovs.core.ovsdb', '/opt/OpenvStorage/db') working_version = 1 # Version 2 introduced: # - Registration if working_version < 2: import time from ovs.extensions.generic.configuration import Configuration if not Configuration.exists('ovs.core.registered'): Configuration.set('ovs.core.registered', False) Configuration.set('ovs.core.install_time', time.time()) working_version = 2 return working_version
def setUp(self): """ (Re)Sets the stores on every test """ self.persistent = DalHelper.setup(fake_sleep=True)[1] Configuration.set('/ovs/framework/arakoon_clusters|voldrv', 'voldrv') Configuration.set('/ovs/framework/rdma', False)
def __init__(self): """ Initializes the client """ self._enable_support = Configuration.get('ovs.support.enablesupport') self.interval = int(Configuration.get('ovs.support.interval')) self._url = 'https://monitoring.openvstorage.com/api/support/heartbeat/'
def setUp(self): """ (Re)Sets the stores on every test """ # Some modules rely on this key, which is loaded during imp.load_source in _get_functions() Configuration.set(key=Configuration.EDITION_KEY, value=PackageFactory.EDITION_ENTERPRISE)
def __init__(self, source, name=None): """ Initializes the logger """ parent_invoker = inspect.stack()[1] if not __file__.startswith(parent_invoker[1]) or parent_invoker[3] != "get": raise RuntimeError( "Cannot invoke instance from outside this class. Please use LogHandler.get(source, name=None) instead" ) if name is None: name = Configuration.get("ovs.logging.default_name") log_filename = LogHandler.load_path(source) formatter = logging.Formatter( "%(asctime)s - [%(process)s] - [%(levelname)s] - [{0}] - [%(name)s] - %(message)s".format(source) ) handler = logging.FileHandler(log_filename) handler.setFormatter(formatter) self.logger = logging.getLogger(name) self.logger.propagate = True self.logger.setLevel(getattr(logging, Configuration.get("ovs.logging.level"))) self.logger.addHandler(handler)
def __init__(self): self._config_corefile = os.path.join( Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-core.conf') self._config_exportfile = os.path.join( Configuration.get('ovs.core.cfgdir'), 'templates', 'ganesha-export.conf')
def model_alba_node(node_id, node_type, ip=None): # type: (str, str, Optional[str]) -> AlbaNode """ Models a non-existing AlbaNode :param node_id: ID of the node :type node_id: str :param node_type: Type of the node :type node_type: str :param ip: IP of the node :type ip: str :return: The modeled node :rtype: AlbaNode """ node = AlbaNode() node.type = node_type node.node_id = node_id config_path = AlbaNode.CONFIG_LOCATIONS[node_type].format( node_id) # type str node.ip = ip or Configuration.get(os.path.join(config_path, 'main|ip')) node.port = Configuration.get(os.path.join(config_path, 'main|port')) node.username = Configuration.get( os.path.join(config_path, 'main|username')) node.password = Configuration.get( os.path.join(config_path, 'main|password')) node.storagerouter = StorageRouterList.get_by_ip(node.ip) return node
def model_albanodes(**kwargs): """ Add all ALBA nodes known to the config platform to the model :param kwargs: Kwargs containing information regarding the node :type kwargs: dict :return: None :rtype: NoneType """ _ = kwargs if Configuration.dir_exists('/ovs/alba/asdnodes'): for node_id in Configuration.list('/ovs/alba/asdnodes'): node = AlbaNodeList.get_albanode_by_node_id(node_id) if node is None: node = AlbaNode() main_config = Configuration.get( '/ovs/alba/asdnodes/{0}/config/main'.format(node_id)) node.type = 'ASD' node.node_id = node_id node.ip = main_config['ip'] node.port = main_config['port'] node.username = main_config['username'] node.password = main_config['password'] node.storagerouter = StorageRouterList.get_by_ip( main_config['ip']) node.save()
def discover_nodes(cls): # type: () -> Dict[str, AlbaNode] """ Discover nodes by querying the config mgmt :return: The discovered nodes, mapped by their guid :rtype: Dict[str, AlbaNode] """ nodes = {} model_node_ids = set(node.node_id for node in AlbaNodeList.get_albanodes()) found_node_ids = set() node_ids_by_type = {} for node_type, base_config_path in { AlbaNode.NODE_TYPES.ASD: ASD_NODE_BASE_PATH, AlbaNode.NODE_TYPES.S3: S3_NODE_BASE_PATH }.iteritems(): if Configuration.dir_exists(base_config_path): node_ids = Configuration.list(base_config_path) node_ids_by_type[node_type] = node_ids for node_type, node_ids in node_ids_by_type.iteritems(): for node_id in node_ids: if node_id not in model_node_ids and node_id not in found_node_ids: node = cls.model_volatile_node(node_id, node_type) nodes[node.guid] = node found_node_ids.add(node.node_id) return nodes
def __init__(self, vpool_guid, storagedriver_id): """ Initializes the class """ _log_level = LOG_LEVEL_MAPPING[OVSLogger( 'extensions').getEffectiveLevel()] # noinspection PyCallByClass,PyTypeChecker storagerouterclient.Logger.setupLogging( OVSLogger.load_path('storagerouterclient'), _log_level) # noinspection PyArgumentList storagerouterclient.Logger.enableLogging() self._key = '/ovs/vpools/{0}/hosts/{1}/config'.format( vpool_guid, storagedriver_id) self._logger = OVSLogger('extensions') self._dirty_entries = [] self.remote_path = Configuration.get_configuration_path( self._key).strip('/') # Load configuration if Configuration.exists(self._key): self.configuration = Configuration.get(self._key) self.config_missing = False else: self.configuration = {} self.config_missing = True self._logger.debug( 'Could not find config {0}, a new one will be created'.format( self._key))
def _get_store_info(cls): client_type = Configuration.get('/ovs/framework/stores|persistent') if client_type not in ['pyrakoon', 'arakoon']: raise RuntimeError('Configured client type {0} is not implemented'.format(client_type)) cluster = Configuration.get('/ovs/framework/arakoon_clusters|ovsdb') contents = Configuration.get('/ovs/arakoon/{0}/config'.format(cluster), raw=True) return {'cluster': cluster, 'configuration': contents}
def delete_config(cluster_name): """ Remove the configuration entry for arakoon cluster_name :param cluster_name: Name of the arakoon cluster :return: None """ config_key = GeneralArakoon.CONFIG_KEY.format(cluster_name) if Configuration.exists(config_key, raw=True): Configuration.delete(os.path.dirname(config_key))
def load_path(source): log_filename = '{0}/{1}.log'.format( Configuration.get('ovs.logging.path'), LogHandler.targets[source] if source in LogHandler.targets else Configuration.get('ovs.logging.default_file') ) if not os.path.exists(log_filename): open(log_filename, 'a').close() os.chmod(log_filename, 0o666) return log_filename
def get_path(binary_name): config_location = 'ovs.path.{0}'.format(binary_name) path = Configuration.get(config_location) if not path: try: path = check_output('which {0}'.format(binary_name), shell=True).strip() Configuration.set(config_location, path) except CalledProcessError: return None return path
def write_config(self, ip=None): """ Writes the configuration down to in the format expected by Arakoon """ contents = self.export_ini() if self.filesystem is False: Configuration.set(self.config_path, contents, raw=True) else: client = self._load_client(ip) client.file_write(self.config_path, contents)
def _assert_set_get(self, a, b): for set_data, get_data in zip(a, b): set_data_type, set_key, set_value, raw = set_data get_data_type, get_key, get_value, raw = get_data Configuration.set(set_key, set_value, raw=raw) set_get_value = Configuration.get(set_key, raw=raw) get_get_value = Configuration.get(get_key, raw=raw) self.assertIsInstance(set_get_value, set_data_type) self.assertTrue(Configuration.get(set_key, raw=raw)) # Make sure every key and value is set as should be self.assertEquals(get_get_value, get_value) # Make sure every value is as predefined
def unregister_service(node_name, service_name): """ Un-register the metadata of a service from the configuration management :param node_name: Name of the node on which to un-register the service :type node_name: str :param service_name: Name of the service to clean from the configuration management :type service_name: str :return: None """ Configuration.delete(key='/ovs/framework/hosts/{0}/services/{1}'.format(node_name, Toolbox.remove_prefix(service_name, 'ovs-')))
def setup(**kwargs): """ Execute several actions before starting a new UnitTest :param kwargs: Additional key word arguments """ if kwargs.get('fake_sleep', False) is True: fakesleep.monkey_patch() volatile, persistent = DalHelper._clean() Configuration.initialize() return volatile, persistent
def load(self): """ Loads the configuration from a given file, optionally a remote one """ self.configuration = {} if Configuration.exists(self.key): self.is_new = False self.configuration = json.loads(Configuration.get(self.key, raw=True)) else: self._logger.debug('Could not find config {0}, a new one will be created'.format(self.key)) self.dirty_entries = []
def delete_config(self, ip=None): """ Deletes a configuration file """ if self.filesystem is False: key = self.config_path if Configuration.exists(key, raw=True): Configuration.delete(key, raw=True) else: client = self._load_client(ip) client.file_delete(self.config_path)
def config_set(self, key, value): if self.is_local is True: from ovs.extensions.generic.configuration import Configuration Configuration.set(key, value) else: write = """ import sys, json sys.path.append('/opt/OpenvStorage') from ovs.extensions.generic.configuration import Configuration Configuration.set('{0}', json.loads('{1}')) """.format(key, json.dumps(value)) self.run('python -c """{0}"""'.format(write))
def _set_mds_safety(vpool, safety=None, checkup=False, logger=LOGGER): if safety is None: safety = len(StoragerouterHelper.get_storagerouters()) if safety <= 0: raise ValueError('Safety should be at least 1.') logger.debug('Setting the safety to {} and {} checkup'.format(safety, 'will' if checkup is True else 'false')) storagedriver_config = Configuration.get('/ovs/vpools/{0}/mds_config'.format(vpool.guid)) current_safety = storagedriver_config current_safety['mds_safety'] = safety Configuration.set('/ovs/framework/storagedriver', current_safety) if checkup is True: MDSServiceController.mds_checkup()
def config_set(self, key, value): if self.is_local is True: from ovs.extensions.generic.configuration import Configuration Configuration.set(key, value) else: write = """ import sys, json sys.path.append('/opt/OpenvStorage') from ovs.extensions.generic.configuration import Configuration Configuration.set('{0}', json.loads('{1}')) """.format(key, json.dumps(value).replace('"', '\\"')) self.run('python -c """{0}"""'.format(write))
def register_service(node_name, service_metadata): """ Register the metadata of the service to the configuration management :param node_name: Name of the node on which the service is running :type node_name: str :param service_metadata: Metadata of the service :type service_metadata: dict :return: None """ service_name = service_metadata['SERVICE_NAME'] Configuration.set(key='/ovs/framework/hosts/{0}/services/{1}'.format(node_name, Toolbox.remove_prefix(service_name, 'ovs-')), value=service_metadata)
def create(self): """ Prepares a new Storagedriver for a given vPool and Storagerouter :return: None :rtype: NoneType """ if self.sr_installer is None: raise RuntimeError('No StorageRouterInstaller instance found') machine_id = System.get_my_machine_id(client=self.sr_installer.root_client) port_range = Configuration.get('/ovs/framework/hosts/{0}/ports|storagedriver'.format(machine_id)) storagerouter = self.sr_installer.storagerouter with volatile_mutex('add_vpool_get_free_ports_{0}'.format(machine_id), wait=30): model_ports_in_use = [] for sd in StorageDriverList.get_storagedrivers(): if sd.storagerouter_guid == storagerouter.guid: model_ports_in_use += sd.ports.values() for proxy in sd.alba_proxies: model_ports_in_use.append(proxy.service.ports[0]) ports = System.get_free_ports(selected_range=port_range, exclude=model_ports_in_use, amount=4 + self.sr_installer.requested_proxies, client=self.sr_installer.root_client) vpool = self.vp_installer.vpool vrouter_id = '{0}{1}'.format(vpool.name, machine_id) storagedriver = StorageDriver() storagedriver.name = vrouter_id.replace('_', ' ') storagedriver.ports = {'management': ports[0], 'xmlrpc': ports[1], 'dtl': ports[2], 'edge': ports[3]} storagedriver.vpool = vpool storagedriver.cluster_ip = Configuration.get('/ovs/framework/hosts/{0}/ip'.format(machine_id)) storagedriver.storage_ip = self.storage_ip storagedriver.mountpoint = '/mnt/{0}'.format(vpool.name) storagedriver.description = storagedriver.name storagedriver.storagerouter = storagerouter storagedriver.storagedriver_id = vrouter_id storagedriver.save() # ALBA Proxies proxy_service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ALBA_PROXY) for proxy_id in xrange(self.sr_installer.requested_proxies): service = Service() service.storagerouter = storagerouter service.ports = [ports[4 + proxy_id]] service.name = 'albaproxy_{0}_{1}'.format(vpool.name, proxy_id) service.type = proxy_service_type service.save() alba_proxy = AlbaProxy() alba_proxy.service = service alba_proxy.storagedriver = storagedriver alba_proxy.save() self.storagedriver = storagedriver
def ipmi_check(cls, result_handler): """ :param result_handler: logging object :type result_handler: ovs.extensions.healthcheck.result.HCResults :return: """ for albanode in AlbaNodeList.get_albanodes(): node_id = albanode.node_id ipmi_config_loc = '/ovs/alba/asdnodes/{0}/config/ipmi'.format( node_id) if not Configuration.exists(ipmi_config_loc): result_handler.skip( 'No IPMI info found on AlbaNode with ID {0}'.format( node_id)) continue ipmi_config = Configuration.get(ipmi_config_loc) ip = ipmi_config.get('ip') try: controller = IPMIController( ip=ip, username=ipmi_config.get('username'), password=ipmi_config.get('password'), client=SSHClient(System.get_my_storagerouter())) except: result_handler.failure( 'IPMI settings are not valid for AlbaNode with ID {0}'. format(node_id)) continue try: status = controller.status_node().get(ip) if status == IPMIController.IPMI_POWER_ON: result_handler.success( 'IPMI AlbaNode with ID {0} status is POWER ON'.format( node_id)) elif status == IPMIController.IPMI_POWER_OFF: result_handler.warning( 'IPMI AlbaNode with ID {0} status is POWER OFF'.format( node_id)) except IPMITimeOutException as ex: result_handler.failure( "IPMI AlbaNode with ID {0} timed out: '{1}'".format( node_id, ex)) except IPMICallException as ex: result_handler.failure( "IPMI AlbaNode with ID {0} call failed: '{1}'".format( node_id, ex)) except Exception: msg = 'Could not retrieve info through IPMI for AlbaNode with ID {0}'.format( node_id) cls.logger.exception(msg) result_handler.exception(msg)
def change_config(storagedriver, config): """ Change the config of the volumedriver and reload the config. Restart will be triggered if no vDisk are running on the volumedriver. :param storagedriver: StorageDriver object :type storagedriver: StorageDriver :param config: Volumedriver config :type config: dict :return: """ service_manager = ServiceFactory.get_manager() config_key = '/ovs/vpools/{0}/hosts/{1}/config'.format( storagedriver.vpool.guid, storagedriver.name) current_config = Configuration.get(config_key) if 'volume_manager' in config: volume_manager = current_config['volume_manager'] for key, value in config['volume_manager'].iteritems(): volume_manager[key] = value if 'backend_connection_manager' in config: backend_connection_manager = current_config[ 'backend_connection_manager'] for key, value in config['backend_connection_manager'].iteritems(): if key == 'proxy': for current_config_key, current_config_value in backend_connection_manager.iteritems( ): if current_config_key.isdigit(): for proxy_key, proxy_config in config[ 'backend_connection_manager'][ 'proxy'].iteritems(): current_config_value[proxy_key] = proxy_config else: backend_connection_manager[key] = value StoragedriverHelper.LOGGER.info("New config: {0}".format( json.dumps(current_config, indent=4))) Configuration.set(config_key, json.dumps(current_config, indent=4), raw=True) client = SSHClient(storagedriver.storagerouter, 'root') service_name = 'ovs-volumedriver_{0}'.format(storagedriver.vpool.name) if len(storagedriver.vdisks_guids) == 0: StoragedriverHelper.LOGGER.info( "Restarting service: {0}".format(service_name)) service_manager.restart_service(service_name, client) else: StoragedriverHelper.LOGGER.info( "Not restarting service: {0}, amount of vdisks: {1}".format( service_name, len(storagedriver.vdisks_guids)))
def get_heartbeat_data(): """ Returns heartbeat data """ data = {'cid': Configuration.get('ovs.support.cid'), 'nid': Configuration.get('ovs.support.nid'), 'metadata': {}, 'errors': []} try: # Versions data['metadata']['versions'] = PackageManager.get_versions() except Exception, ex: data['errors'].append(str(ex))
def validate_alba_backend_removal(alba_backend_info): """ Validate whether the backend has been deleted properly alba_backend_info should be a dictionary containing: - guid - name - maintenance_service_names :param alba_backend_info: Information about the backend :return: None """ Toolbox.verify_required_params(actual_params=alba_backend_info, required_params={'name': (str, None), 'guid': (str, Toolbox.regex_guid), 'maintenance_service_names': (list, None)}, exact_match=True) alba_backend_guid = alba_backend_info['guid'] alba_backend_name = alba_backend_info['name'] backend = GeneralBackend.get_by_name(alba_backend_name) assert backend is None,\ 'Still found a backend in the model with name {0}'.format(alba_backend_name) # Validate services removed from model for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.ALBA_MGR): assert service.name != '{0}-abm'.format(alba_backend_name),\ 'An AlbaManager service has been found with name {0}'.format(alba_backend_name) for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.NS_MGR): assert service.name.startswith('{0}-nsm_'.format(alba_backend_name)) is False,\ 'An NamespaceManager service has been found with name {0}'.format(alba_backend_name) # Validate ALBA backend configuration structure alba_backend_key = '/ovs/alba/backends' actual_configuration_keys = [key for key in Configuration.list(alba_backend_key)] assert alba_backend_guid not in actual_configuration_keys,\ 'Configuration still contains an entry in {0} with guid {1}'.format(alba_backend_key, alba_backend_guid) # Validate Arakoon configuration structure arakoon_keys = [key for key in Configuration.list('/ovs/arakoon') if key.startswith(alba_backend_name)] assert len(arakoon_keys) == 0,\ 'Configuration still contains configurations for clusters: {0}'.format(', '.join(arakoon_keys)) # Validate services for storagerouter in GeneralStorageRouter.get_storage_routers(): root_client = SSHClient(endpoint=storagerouter, username='******') maintenance_services = alba_backend_info['maintenance_service_names'] abm_arakoon_service_name = 'ovs-arakoon-{0}-abm'.format(alba_backend_name) nsm_arakoon_service_name = 'ovs-arakoon-{0}-nsm_0'.format(alba_backend_name) for service_name in [abm_arakoon_service_name, nsm_arakoon_service_name] + maintenance_services: assert GeneralService.has_service(name=service_name, client=root_client) is False,\ 'Service {0} still deployed on Storage Router {1}'.format(service_name, storagerouter.name)
def get_config(cluster_name): """ Retrieve the configuration for given cluster :param cluster_name: Name of the cluster :return: RawConfigParser object """ config_key = GeneralArakoon.CONFIG_KEY.format(cluster_name) if not Configuration.exists(config_key, raw=True): raise ValueError('Unknown arakoon cluster_name {0} provided'.format(cluster_name)) voldrv_config = Configuration.get(config_key, raw=True) parser = RawConfigParser() parser.readfp(StringIO(voldrv_config)) return parser
def get_cluster_name(internal_name): """ Retrieve the name of the cluster :param internal_name: Name as known by the framework :type internal_name: str :return: Name known by user :rtype: str """ config_key = '/ovs/framework/arakoon_clusters' if Configuration.exists(config_key): cluster_info = Configuration.get(config_key) if internal_name in cluster_info: return cluster_info[internal_name] if internal_name not in ['ovsdb', 'voldrv']: return internal_name
def register(node_id): """ Adds a Node with a given node_id to the model :param node_id: ID of the ALBA node :type node_id: str :return: None """ node = AlbaNodeList.get_albanode_by_node_id(node_id) if node is None: main_config = Configuration.get('/ovs/alba/asdnodes/{0}/config/main'.format(node_id)) node = AlbaNode() node.ip = main_config['ip'] node.port = main_config['port'] node.username = main_config['username'] node.password = main_config['password'] node.storagerouter = StorageRouterList.get_by_ip(main_config['ip']) data = node.client.get_metadata() if data['_success'] is False and data['_error'] == 'Invalid credentials': raise RuntimeError('Invalid credentials') if data['node_id'] != node_id: AlbaNodeController._logger.error('Unexpected node_id: {0} vs {1}'.format(data['node_id'], node_id)) raise RuntimeError('Unexpected node identifier') node.node_id = node_id node.type = 'ASD' node.save() AlbaController.checkup_maintenance_agents.delay()
def _presets(self): """ Returns the policies active on the node """ if len(self.abm_services) == 0: return [] # No ABM services yet, so backend not fully installed yet asds = {} if self.scaling != AlbaBackend.SCALINGS.GLOBAL: for node in AlbaNodeList.get_albanodes(): asds[node.node_id] = 0 for disk in self.local_stack[node.node_id].values(): for asd_info in disk['asds'].values(): if asd_info['status'] in ['claimed', 'warning']: asds[node.node_id] += 1 config = Configuration.get_configuration_path('/ovs/arakoon/{0}-abm/config'.format(self.name)) presets = AlbaCLI.run(command='list-presets', config=config) preset_dict = {} for preset in presets: preset_dict[preset['name']] = preset if 'in_use' not in preset: preset['in_use'] = True if 'is_default' not in preset: preset['is_default'] = False preset['is_available'] = False preset['policies'] = [tuple(policy) for policy in preset['policies']] preset['policy_metadata'] = {} active_policy = None for policy in preset['policies']: is_available = False available_disks = 0 if self.scaling != AlbaBackend.SCALINGS.GLOBAL: available_disks += sum(min(asds[node], policy[3]) for node in asds) if self.scaling != AlbaBackend.SCALINGS.LOCAL: available_disks += sum(self.local_summary['devices'].values()) if available_disks >= policy[2]: if active_policy is None: active_policy = policy is_available = True preset['policy_metadata'][policy] = {'is_active': False, 'in_use': False, 'is_available': is_available} preset['is_available'] |= is_available if active_policy is not None: preset['policy_metadata'][active_policy]['is_active'] = True for namespace in self.ns_data: if namespace['namespace']['state'] != 'active': continue policy_usage = namespace['statistics']['bucket_count'] preset = preset_dict[namespace['namespace']['preset_name']] for usage in policy_usage: upolicy = tuple(usage[0]) # Policy as reported to be "in use" for cpolicy in preset['policies']: # All configured policies if upolicy[0] == cpolicy[0] and upolicy[1] == cpolicy[1] and upolicy[3] <= cpolicy[3]: preset['policy_metadata'][cpolicy]['in_use'] = True break for preset in presets: preset['policies'] = [str(policy) for policy in preset['policies']] for key in preset['policy_metadata'].keys(): preset['policy_metadata'][str(key)] = preset['policy_metadata'][key] del preset['policy_metadata'][key] return presets
def config_files_check_test(): """ Verify some configuration files """ issues_found = '' config_keys = { "/ovs/framework/memcache", "/ovs/arakoon/ovsdb/config" } for key_to_check in config_keys: if not Configuration.exists(key_to_check, raw=True): issues_found += "Couldn't find {0}\n".format(key_to_check) config_files = { "rabbitmq.config": "/etc/rabbitmq/rabbitmq.config", } grid_ip = General.get_config().get('main', 'grid_ip') ssh_pass = General.get_config().get('mgmtcenter', 'password') client = SSHClient(grid_ip, username='******', password=ssh_pass) for config_file_to_check in config_files.iterkeys(): if not client.file_exists(config_files[config_file_to_check]): issues_found += "Couldn't find {0}\n".format(config_file_to_check) assert issues_found == '',\ "Found the following issues while checking for the config files:{0}\n".format(issues_found)
def configure_avahi(client, node_name, node_type, logger): """ Configure Avahi :param client: Client on which to configure avahi :type client: ovs.extensions.generic.sshclient.SSHClient :param node_name: Name of the node to set in Avahi :type node_name: str :param node_type: Type of the node ('master' or 'extra') :type node_type: str :param logger: Logger object used for logging :type logger: ovs.log.log_handler.LogHandler :return: None """ cluster_name = Configuration.get('/ovs/framework/cluster_name') Toolbox.log(logger=logger, messages='Announcing service') client.file_write(NodeTypeController.avahi_filename, """<?xml version="1.0" standalone='no'?> <!--*-nxml-*--> <!DOCTYPE service-group SYSTEM "avahi-service.dtd"> <!-- $Id$ --> <service-group> <name replace-wildcards="yes">ovs_cluster_{0}_{1}_{3}</name> <service> <type>_ovs_{2}_node._tcp</type> <port>443</port> </service> </service-group>""".format(cluster_name, node_name, node_type, client.ip.replace('.', '_'))) Toolbox.change_service_state(client, 'avahi-daemon', 'restart', NodeTypeController._logger)
def update_preset(alba_backend_guid, name, policies): """ Updates policies for an existing preset to Alba :param alba_backend_guid: Guid of the ALBA backend :type alba_backend_guid: str :param name: Name of backend :type name: str :param policies: New policy list to be sent to alba :type policies: list :return: None """ # VALIDATIONS AlbaPresetController._validate_policies_param(policies=policies) alba_backend = AlbaBackend(alba_backend_guid) if name not in [preset['name'] for preset in alba_backend.presets]: raise RuntimeError('Could not find a preset with name {0} for ALBA Backend {1}'.format(name, alba_backend.name)) # UPDATE PRESET AlbaPresetController._logger.debug('Updating preset {0} with policies {1}'.format(name, policies)) config = Configuration.get_configuration_path(ArakoonInstaller.CONFIG_KEY.format(AlbaController.get_abm_cluster_name(alba_backend=alba_backend))) temp_config_file = tempfile.mktemp() with open(temp_config_file, 'wb') as data_file: data_file.write(json.dumps({'policies': policies})) data_file.flush() AlbaCLI.run(command='update-preset', config=config, named_params={'input-url': temp_config_file}, extra_params=[name]) alba_backend.invalidate_dynamics() os.remove(temp_config_file)
def _get_arakoon_clusters(cls, result_handler): """ Retrieves all Arakoon clusters registered in this OVSCluster :param result_handler: Logging object :type result_handler: ovs.extensions.healthcheck.result.HCResults :return: Dict with the Arakoon cluster types as key and list with dicts which contain cluster names and pyrakoon clients :rtype: dict(str, list[dict]) """ result_handler.info('Fetching available arakoon clusters.', add_to_result=False) arakoon_clusters = {} for cluster_name in list(Configuration.list('/ovs/arakoon')) + ['cacc']: # Determine Arakoon type is_cacc = cluster_name == 'cacc' arakoon_config = ArakoonClusterConfig(cluster_id=cluster_name, load_config=not is_cacc) if is_cacc is True: with open(Configuration.CACC_LOCATION) as config_file: contents = config_file.read() arakoon_config.read_config(contents=contents) try: arakoon_client = ArakoonInstaller.build_client(arakoon_config) except (ArakoonNoMaster, ArakoonNoMasterResult) as ex: result_handler.failure('Unable to find a master for Arakoon cluster {0}. (Message: {1})'.format(cluster_name, str(ex)), code=ErrorCodes.master_none) except Exception as ex: msg = 'Unable to connect to Arakoon cluster {0}. (Message: {1})'.format(cluster_name, str(ex)) result_handler.exception(msg, code=ErrorCodes.unhandled_exception) cls.logger.exception(msg) continue metadata = json.loads(arakoon_client.get(ArakoonInstaller.METADATA_KEY)) cluster_type = metadata['cluster_type'] if cluster_type not in arakoon_clusters: arakoon_clusters[cluster_type] = [] arakoon_clusters[cluster_type].append({'cluster_name': cluster_name, 'client': arakoon_client, 'config': arakoon_config}) return arakoon_clusters
def test_multi_node(self): base_port = Configuration.get('ovs.ports.arakoon')[0] cluster = 'one' nodes = sorted(TestArakoonInstaller.nodes.keys()) nodes = dict((node, SSHClient(node)) for node in nodes) first_node = nodes.keys()[0] ArakoonInstaller.create_cluster(cluster, first_node, []) for node in nodes[1:]: ArakoonInstaller.extend_cluster(first_node, node, cluster, []) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes)) for node in nodes: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node, client in nodes.iteritems(): contents = client.file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip()) ArakoonInstaller.shrink_cluster(nodes[1], first_node, cluster) expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:])) for node in nodes.keys()[1:]: expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1) expected = expected.strip() for node, client in nodes.iteritems(): if node == first_node: continue contents = client.file_read(self._get_config_path(cluster)) self.assertEqual(contents.strip(), expected.strip())