def merge(self, dest_index: str, src_index: str, keys: list = None):
"""
Merges the content of src_index and dest_index file
Parameters:
dst_index - Destination Index, to this index resulted values will be
merged
src_index - Source Index, From which new keys (and related values) are
picked up for merging
keys - optional parameter, Only these keys (and related values) from
src_index will be merged.
"""
if src_index not in self._cache.keys():
raise ConfError(errors.ERR_NOT_INITIALIZED, "config index %s is "\
"not loaded", src_index)
if dest_index not in self._cache.keys():
raise ConfError(errors.ERR_NOT_INITIALIZED, "config index %s is "\
"not loaded", dest_index)
if keys is None:
keys = self._cache[src_index].get_keys()
else:
for key in keys:
if not self._cache[src_index].get(key):
raise ConfError(errno.ENOENT, "%s is not present in %s", \
key, src_index)
self._merge(dest_index, src_index, keys)
def load(self, index: str, kvs_url: str, **kwargs):
"""
Loads the config from KV Store
Parameters:
index: Identifier for the config loaded from the KV Store
kv_store: KV Store (Conf Backend)
overwrite: When False, it throws exception if index already exists.
Default: False
callback: Callback for the config changes in the KV Store.
"""
overwrite = False
for key, val in kwargs.items():
if key == 'overwrite':
overwrite = True
elif key == 'callback':
self._callbacks[index] = val
else:
raise ConfError(errno.EINVAL, "Invalid parameter %s", key)
if index in self._cache.keys() and not overwrite:
raise ConfError(errno.EINVAL, "conf index %s already exists",
index)
kv_store = KvStoreFactory.get_instance(kvs_url, self._delim)
self._cache[index] = ConfCache(kv_store, self._delim)
def compare(self, index1: str, index2: str):
"""
Compares two configs and returns difference
Parameters:
index1 : Conf Index 1
index2 : Conf Index 2
Return Value:
Returns three lists : New keys, deleted keys, Updated keys
"""
if index1 not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index1)
if index2 not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index2)
key_list1 = self._cache[index1].get_keys()
key_list2 = self._cache[index2].get_keys()
deleted_keys = list(set(key_list1).difference(key_list2))
new_keys = list(set(key_list2).difference(key_list1))
updated_keys = list(
filter(
lambda key: key not in deleted_keys and self._cache[index1].
get(key) != self._cache[index2].get(key), key_list1))
return new_keys, deleted_keys, updated_keys
def copy(self,
src_index: str,
dst_index: str,
key_list: list = None,
recurse: bool = True):
"""
Copies one config domain to the other and saves
Parameters:
src_index Source Index
dst_index Destination Index
"""
if src_index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
src_index)
if dst_index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
dst_index)
if key_list is None:
if recurse:
key_list = self._cache[src_index].get_keys(key_index=True)
else:
key_list = self._cache[src_index].get_keys(key_index=False)
for key in key_list:
self._cache[dst_index].set(key, self._cache[src_index].get(key))
def set(args):
""" Set Key Value """
kv_delim = '=' if args.kv_delim == None else args.kv_delim
if len(kv_delim) > 1 or kv_delim not in [':', '>', '.', '|', '/', '=']:
raise ConfError(errno.EINVAL, "invalid delim %s", kv_delim)
kv_list = args.args[0].split(';')
for kv in kv_list:
try:
key, val = kv.split(kv_delim, 1)
except:
raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv)
Conf.set(ConfCli._index, key, val)
Conf.save(ConfCli._index)
def save(self, index: str):
""" Saves the given index configuration onto KV Store """
if index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index)
self._cache[index].dump()
def get_storage_path(key, none_allowed: bool = False):
"""Get the config file path."""
path = Conf.get(CortxConf._cluster_index, f'cortx>common>storage>{key}')
if not none_allowed:
if not path:
raise ConfError(errno.EINVAL, "Invalid key %s", key)
return path
def get_config_value(index, key):
value = Conf.get(index, key)
if not value:
raise ConfError(
errno.EINVAL, "Config validation failure. \
No value found for key: '%s' in input config.", key)
return value
def delete(self, index: str, key: str):
""" Delets a given key from the config """
if index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index)
self._cache[index].delete(key)
def get_keys(args) -> list:
""" Returns list of keys present in store """
key_index = 'true' if args.key_index == None else args.key_index.lower().strip()
key_index = True if key_index == 'true' else False if key_index == 'false' else None
if key_index == None:
raise ConfError(errno.EINVAL, "invalid key_index value %s", key_index)
return Conf.get_keys(ConfCli._index, key_index=key_index)
def copy(self, src_index: str, key_list: list = None):
"""Copy src_index config into CORTX confstore file."""
try:
Conf.copy(src_index, self._conf_idx, key_list)
except (AssertionError, ConfError) as e:
raise ConfError(
errno.EINVAL,
f'Error occurred while copying config into confstore. {e}')
def __init__(self, kv_store: KvStore, delim='>'):
if len(delim) > 1:
raise ConfError(errno.EINVAL, "invalid delim %s", delim)
self._delim = delim
self._dirty = False
self._kv_store = kv_store
self._data = None
self.load()
def validate(self, stage):
if stage == "post_install":
PkgV().validate('rpms', ['consul'])
elif stage == "config":
keys = [
f"server_node>{Conf.machine_id}>network>data>private_interfaces",
f"server_node>{Conf.machine_id}>network>data>private_fqdn",
"cortx>software>consul>config_path",
"cortx>software>consul>data_path",
]
for key in keys:
value = Conf.get(self.index, key)
if not value:
raise ConfError(
errno.EINVAL,
"Consul Setup config validation falied. %s key not found",
key)
max_retry = 3
server_node_fqdns = [
Conf.get(
self.index,
f"server_node>{machine_id}>network>data>private_fqdn")
for machine_id in Conf.get(self.index, "server_node").keys()
]
for i in range(max_retry):
try:
NetworkV().validate("connectivity", server_node_fqdns)
break
except VError:
if i == (max_retry - 1):
raise
time.sleep(0.5)
elif stage == "cleanup":
keys = [
"cortx>software>consul>config_path",
"cortx>software>consul>data_path",
]
for key in keys:
value = Conf.get(self.index, key)
if not value:
raise ConfError(
errno.EINVAL,
"Consul Setup config validation falied. %s key not found",
key)
def set(self, key: str, val: str):
"""Save key-value in CORTX confstore."""
try:
Conf.set(self._conf_idx, key, val)
Conf.save(self._conf_idx)
except (AssertionError, ConfError) as e:
raise ConfError(
errno.EINVAL,
f'Error occurred while adding key {key} and value {val}'
f' in confstore. {e}')
def set(args):
""" Set Key Value """
kv_list = args.args[0].split(';')
for kv in kv_list:
try:
key, val = kv.split('=')
except:
raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv)
Conf.set(ConfCli._index, key, val)
Conf.save(ConfCli._index)
def load(self, index: str, kvs_url: str, **kwargs):
"""
Loads the config from KV Store
Parameters:
index: Identifier for the config loaded from the KV Store
kv_store: KV Store (Conf Backend)
fail_reload: When True, and if index already exists, load() throws
exception.
When True, and if index do not exists, load() succeeds.
When false, irrespective of index status, load() succeeds
Default: True
skip_reload: When True, it skips reloading a index configuration by
overriding fail_reload
Default: False
callback: Callback for the config changes in the KV Store.
"""
fail_reload = True
skip_reload = False
recurse = True
for key, val in kwargs.items():
if key == 'fail_reload':
fail_reload = val
elif key == 'skip_reload':
skip_reload = val
elif key == 'callback':
self._callbacks[index] = val
elif key == 'recurse':
if val not in [True, False]:
raise ConfError(errno.EINVAL,
"Invalid value for recurse %s", val)
recurse = val
else:
raise ConfError(errno.EINVAL, "Invalid parameter %s", key)
if index in self._cache.keys():
if skip_reload:
return
if fail_reload:
raise ConfError(errno.EINVAL, "conf index %s already exists",
index)
kv_store = KvStoreFactory.get_instance(kvs_url, self._delim)
self._cache[index] = ConfCache(kv_store, self._delim, recurse=recurse)
def __init__(self, delim='>'):
"""
ConfStore will be initialized at the time of load
delim is used to split key into hierarchy, e.g. "k1>2" or "k1.k2"
"""
if len(delim) > 1 or delim not in [':', '>', '.', '|', ';', '/']:
raise ConfError(errno.EINVAL, "invalid delim %s", delim)
self._delim = delim
self._cache = {}
self._callbacks = {}
def init(cls, cluster_conf):
"""Initiallize DbConf."""
Conf.load(cls._cluster_index, cluster_conf, skip_reload=True)
cls._get_consul_netloc()
if cls._consul_host is None or cls._consul_port is None:
raise ConfError(
errno.ENOENT,
f'Consul KV configuration not found in {cluster_conf}')
Conf.load(
cls._db_index,
f"consul://{cls._consul_host}:{cls._consul_port}/utils_db_conf",
skip_reload=True)
def get(self, index: str, key: str, default_val: str = None, **filters):
"""
Obtain value for the given configuration
Paraeters:
index Configuration Domain ID where config key values are stored
key Configuration key. This can take two forms
1. "xyz" - Top Level Key
2. "x.y.z" - Key 'z' under x and y. Nested Structure.
default_val
Default Value
Return Value:
Return type will be dict or string based of key
"""
if index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index)
if key is None:
raise ConfError(
errno.EINVAL, "can't able to find config key "
"%s in loaded config", key)
val = self._cache[index].get(key, **filters)
return default_val if val is None else val
def set(self, index: str, key: str, val):
"""
Sets the value into the DB for the given index, key
Parameters:
index Configuration Domain ID where config key values are stored
key Configuration key. This can take two forms
1. "xyz" - Top Level Key
2. "x.y.z" - Key 'z' under x and y. Nested Structure.
val Value to be set. Can be string or dict
"""
if index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index)
self._cache[index].set(key, val)
def set_kvs(self, kvs: list):
"""
Parameters:
kvs - List of KV tuple, e.g. [('k1','v1'),('k2','v2')]
Where, k1, k2 - is full key path till the leaf key.
"""
for key, val in kvs:
try:
Conf.set(self._conf_idx, key, val)
except (AssertionError, ConfError) as e:
raise ConfError(
errno.EINVAL,
f'Error occurred while adding key {key} and value {val}'
f' in confstore. {e}')
Conf.save(self._conf_idx)
def get(args) -> str:
""" Obtain value for the given keys """
params = args.args
key_list = params[0].split(';')
n_keys = len(key_list)
def_val_list = list(None for i in range(0, n_keys))
if len(params) > 1:
def_val_list = params[1].split(';')
if len(def_val_list) != n_keys:
raise ConfError(errno.EINVAL,
"No. of default values, dont match no. of keys")
val_list = []
for i in range(0, n_keys):
val = Conf.get(ConfCli._index, key_list[i], def_val_list[i])
val_list.append(val)
format_type = 'json' if args.format == None else args.format
return Format.dump(val_list, format_type)
def merge(args):
""" merges source conf file into dest. conf file. """
src_index = 'src_index'
dest_index = ConfCli._index
ConfCli.load(args.src_url, src_index)
if not args.keys: # no keys provided
keys = Conf.get_keys(src_index) # getting src file keys
else:
keys = args.keys[0].split(';')
src_keys = Conf.get_keys(src_index)
for key in keys:
if key not in src_keys:
raise ConfError(errno.ENOENT, "%s is not present in %s", \
key, args.src_url)
Conf.merge(dest_index, src_index, keys)
Conf.save(dest_index)
def setUpClass(cls, \
cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'):
"""Setup test class."""
if TestConfStore._cluster_conf_path:
cls.cluster_conf_path = TestConfStore._cluster_conf_path
else:
cls.cluster_conf_path = cluster_conf_path
for index_url in load_index_url():
index = index_url[0]
url = index_url[1]
if index not in TestConfStore.indexes:
cls.indexes.append(index)
if 'consul' in index.lower():
Conf.load('config', cls.cluster_conf_path, skip_reload=True)
endpoint_url = Conf.get('config', url)
if endpoint_url is not None and 'http' in endpoint_url:
url = endpoint_url.replace('http', 'consul')
else:
raise ConfError(errno.EINVAL,
"Invalid consul endpoint key %s", url)
load_config(index, url)
def get_cluster_conf_path():
if CortxConf._cluster_conf is None:
raise ConfError(errno.ENOENT, "Path for config file, cluster.conf, not provided")
return CortxConf._cluster_conf
def delete(self, index: str, key: str, force: bool = False):
"""Delets a given key from the config."""
if index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index)
return self._cache[index].delete(key, force)
def get_data(self, index: str):
""" Obtains entire config for given index """
if index not in self._cache.keys():
raise ConfError(errno.EINVAL, "config index %s is not loaded",
index)
return self._cache[index].get_data()
