def merge(self, dest_index: str, src_index: str, keys: list = None): """ Merges the content of src_index and dest_index file Parameters: dst_index - Destination Index, to this index resulted values will be merged src_index - Source Index, From which new keys (and related values) are picked up for merging keys - optional parameter, Only these keys (and related values) from src_index will be merged. """ if src_index not in self._cache.keys(): raise ConfError(errors.ERR_NOT_INITIALIZED, "config index %s is "\ "not loaded", src_index) if dest_index not in self._cache.keys(): raise ConfError(errors.ERR_NOT_INITIALIZED, "config index %s is "\ "not loaded", dest_index) if keys is None: keys = self._cache[src_index].get_keys() else: for key in keys: if not self._cache[src_index].get(key): raise ConfError(errno.ENOENT, "%s is not present in %s", \ key, src_index) self._merge(dest_index, src_index, keys)
def load(self, index: str, kvs_url: str, **kwargs): """ Loads the config from KV Store Parameters: index: Identifier for the config loaded from the KV Store kv_store: KV Store (Conf Backend) overwrite: When False, it throws exception if index already exists. Default: False callback: Callback for the config changes in the KV Store. """ overwrite = False for key, val in kwargs.items(): if key == 'overwrite': overwrite = True elif key == 'callback': self._callbacks[index] = val else: raise ConfError(errno.EINVAL, "Invalid parameter %s", key) if index in self._cache.keys() and not overwrite: raise ConfError(errno.EINVAL, "conf index %s already exists", index) kv_store = KvStoreFactory.get_instance(kvs_url, self._delim) self._cache[index] = ConfCache(kv_store, self._delim)
def compare(self, index1: str, index2: str): """ Compares two configs and returns difference Parameters: index1 : Conf Index 1 index2 : Conf Index 2 Return Value: Returns three lists : New keys, deleted keys, Updated keys """ if index1 not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index1) if index2 not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index2) key_list1 = self._cache[index1].get_keys() key_list2 = self._cache[index2].get_keys() deleted_keys = list(set(key_list1).difference(key_list2)) new_keys = list(set(key_list2).difference(key_list1)) updated_keys = list( filter( lambda key: key not in deleted_keys and self._cache[index1]. get(key) != self._cache[index2].get(key), key_list1)) return new_keys, deleted_keys, updated_keys
def copy(self, src_index: str, dst_index: str, key_list: list = None, recurse: bool = True): """ Copies one config domain to the other and saves Parameters: src_index Source Index dst_index Destination Index """ if src_index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", src_index) if dst_index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", dst_index) if key_list is None: if recurse: key_list = self._cache[src_index].get_keys(key_index=True) else: key_list = self._cache[src_index].get_keys(key_index=False) for key in key_list: self._cache[dst_index].set(key, self._cache[src_index].get(key))
def set(args): """ Set Key Value """ kv_delim = '=' if args.kv_delim == None else args.kv_delim if len(kv_delim) > 1 or kv_delim not in [':', '>', '.', '|', '/', '=']: raise ConfError(errno.EINVAL, "invalid delim %s", kv_delim) kv_list = args.args[0].split(';') for kv in kv_list: try: key, val = kv.split(kv_delim, 1) except: raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv) Conf.set(ConfCli._index, key, val) Conf.save(ConfCli._index)
def save(self, index: str): """ Saves the given index configuration onto KV Store """ if index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index) self._cache[index].dump()
def get_storage_path(key, none_allowed: bool = False): """Get the config file path.""" path = Conf.get(CortxConf._cluster_index, f'cortx>common>storage>{key}') if not none_allowed: if not path: raise ConfError(errno.EINVAL, "Invalid key %s", key) return path
def get_config_value(index, key): value = Conf.get(index, key) if not value: raise ConfError( errno.EINVAL, "Config validation failure. \ No value found for key: '%s' in input config.", key) return value
def delete(self, index: str, key: str): """ Delets a given key from the config """ if index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index) self._cache[index].delete(key)
def get_keys(args) -> list: """ Returns list of keys present in store """ key_index = 'true' if args.key_index == None else args.key_index.lower().strip() key_index = True if key_index == 'true' else False if key_index == 'false' else None if key_index == None: raise ConfError(errno.EINVAL, "invalid key_index value %s", key_index) return Conf.get_keys(ConfCli._index, key_index=key_index)
def copy(self, src_index: str, key_list: list = None): """Copy src_index config into CORTX confstore file.""" try: Conf.copy(src_index, self._conf_idx, key_list) except (AssertionError, ConfError) as e: raise ConfError( errno.EINVAL, f'Error occurred while copying config into confstore. {e}')
def __init__(self, kv_store: KvStore, delim='>'): if len(delim) > 1: raise ConfError(errno.EINVAL, "invalid delim %s", delim) self._delim = delim self._dirty = False self._kv_store = kv_store self._data = None self.load()
def validate(self, stage): if stage == "post_install": PkgV().validate('rpms', ['consul']) elif stage == "config": keys = [ f"server_node>{Conf.machine_id}>network>data>private_interfaces", f"server_node>{Conf.machine_id}>network>data>private_fqdn", "cortx>software>consul>config_path", "cortx>software>consul>data_path", ] for key in keys: value = Conf.get(self.index, key) if not value: raise ConfError( errno.EINVAL, "Consul Setup config validation falied. %s key not found", key) max_retry = 3 server_node_fqdns = [ Conf.get( self.index, f"server_node>{machine_id}>network>data>private_fqdn") for machine_id in Conf.get(self.index, "server_node").keys() ] for i in range(max_retry): try: NetworkV().validate("connectivity", server_node_fqdns) break except VError: if i == (max_retry - 1): raise time.sleep(0.5) elif stage == "cleanup": keys = [ "cortx>software>consul>config_path", "cortx>software>consul>data_path", ] for key in keys: value = Conf.get(self.index, key) if not value: raise ConfError( errno.EINVAL, "Consul Setup config validation falied. %s key not found", key)
def set(self, key: str, val: str): """Save key-value in CORTX confstore.""" try: Conf.set(self._conf_idx, key, val) Conf.save(self._conf_idx) except (AssertionError, ConfError) as e: raise ConfError( errno.EINVAL, f'Error occurred while adding key {key} and value {val}' f' in confstore. {e}')
def set(args): """ Set Key Value """ kv_list = args.args[0].split(';') for kv in kv_list: try: key, val = kv.split('=') except: raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv) Conf.set(ConfCli._index, key, val) Conf.save(ConfCli._index)
def load(self, index: str, kvs_url: str, **kwargs): """ Loads the config from KV Store Parameters: index: Identifier for the config loaded from the KV Store kv_store: KV Store (Conf Backend) fail_reload: When True, and if index already exists, load() throws exception. When True, and if index do not exists, load() succeeds. When false, irrespective of index status, load() succeeds Default: True skip_reload: When True, it skips reloading a index configuration by overriding fail_reload Default: False callback: Callback for the config changes in the KV Store. """ fail_reload = True skip_reload = False recurse = True for key, val in kwargs.items(): if key == 'fail_reload': fail_reload = val elif key == 'skip_reload': skip_reload = val elif key == 'callback': self._callbacks[index] = val elif key == 'recurse': if val not in [True, False]: raise ConfError(errno.EINVAL, "Invalid value for recurse %s", val) recurse = val else: raise ConfError(errno.EINVAL, "Invalid parameter %s", key) if index in self._cache.keys(): if skip_reload: return if fail_reload: raise ConfError(errno.EINVAL, "conf index %s already exists", index) kv_store = KvStoreFactory.get_instance(kvs_url, self._delim) self._cache[index] = ConfCache(kv_store, self._delim, recurse=recurse)
def __init__(self, delim='>'): """ ConfStore will be initialized at the time of load delim is used to split key into hierarchy, e.g. "k1>2" or "k1.k2" """ if len(delim) > 1 or delim not in [':', '>', '.', '|', ';', '/']: raise ConfError(errno.EINVAL, "invalid delim %s", delim) self._delim = delim self._cache = {} self._callbacks = {}
def init(cls, cluster_conf): """Initiallize DbConf.""" Conf.load(cls._cluster_index, cluster_conf, skip_reload=True) cls._get_consul_netloc() if cls._consul_host is None or cls._consul_port is None: raise ConfError( errno.ENOENT, f'Consul KV configuration not found in {cluster_conf}') Conf.load( cls._db_index, f"consul://{cls._consul_host}:{cls._consul_port}/utils_db_conf", skip_reload=True)
def get(self, index: str, key: str, default_val: str = None, **filters): """ Obtain value for the given configuration Paraeters: index Configuration Domain ID where config key values are stored key Configuration key. This can take two forms 1. "xyz" - Top Level Key 2. "x.y.z" - Key 'z' under x and y. Nested Structure. default_val Default Value Return Value: Return type will be dict or string based of key """ if index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index) if key is None: raise ConfError( errno.EINVAL, "can't able to find config key " "%s in loaded config", key) val = self._cache[index].get(key, **filters) return default_val if val is None else val
def set(self, index: str, key: str, val): """ Sets the value into the DB for the given index, key Parameters: index Configuration Domain ID where config key values are stored key Configuration key. This can take two forms 1. "xyz" - Top Level Key 2. "x.y.z" - Key 'z' under x and y. Nested Structure. val Value to be set. Can be string or dict """ if index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index) self._cache[index].set(key, val)
def set_kvs(self, kvs: list): """ Parameters: kvs - List of KV tuple, e.g. [('k1','v1'),('k2','v2')] Where, k1, k2 - is full key path till the leaf key. """ for key, val in kvs: try: Conf.set(self._conf_idx, key, val) except (AssertionError, ConfError) as e: raise ConfError( errno.EINVAL, f'Error occurred while adding key {key} and value {val}' f' in confstore. {e}') Conf.save(self._conf_idx)
def get(args) -> str: """ Obtain value for the given keys """ params = args.args key_list = params[0].split(';') n_keys = len(key_list) def_val_list = list(None for i in range(0, n_keys)) if len(params) > 1: def_val_list = params[1].split(';') if len(def_val_list) != n_keys: raise ConfError(errno.EINVAL, "No. of default values, dont match no. of keys") val_list = [] for i in range(0, n_keys): val = Conf.get(ConfCli._index, key_list[i], def_val_list[i]) val_list.append(val) format_type = 'json' if args.format == None else args.format return Format.dump(val_list, format_type)
def merge(args): """ merges source conf file into dest. conf file. """ src_index = 'src_index' dest_index = ConfCli._index ConfCli.load(args.src_url, src_index) if not args.keys: # no keys provided keys = Conf.get_keys(src_index) # getting src file keys else: keys = args.keys[0].split(';') src_keys = Conf.get_keys(src_index) for key in keys: if key not in src_keys: raise ConfError(errno.ENOENT, "%s is not present in %s", \ key, args.src_url) Conf.merge(dest_index, src_index, keys) Conf.save(dest_index)
def setUpClass(cls, \ cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'): """Setup test class.""" if TestConfStore._cluster_conf_path: cls.cluster_conf_path = TestConfStore._cluster_conf_path else: cls.cluster_conf_path = cluster_conf_path for index_url in load_index_url(): index = index_url[0] url = index_url[1] if index not in TestConfStore.indexes: cls.indexes.append(index) if 'consul' in index.lower(): Conf.load('config', cls.cluster_conf_path, skip_reload=True) endpoint_url = Conf.get('config', url) if endpoint_url is not None and 'http' in endpoint_url: url = endpoint_url.replace('http', 'consul') else: raise ConfError(errno.EINVAL, "Invalid consul endpoint key %s", url) load_config(index, url)
def get_cluster_conf_path(): if CortxConf._cluster_conf is None: raise ConfError(errno.ENOENT, "Path for config file, cluster.conf, not provided") return CortxConf._cluster_conf
def delete(self, index: str, key: str, force: bool = False): """Delets a given key from the config.""" if index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index) return self._cache[index].delete(key, force)
def get_data(self, index: str): """ Obtains entire config for given index """ if index not in self._cache.keys(): raise ConfError(errno.EINVAL, "config index %s is not loaded", index) return self._cache[index].get_data()