def update_motr_hare_keys_for_all_nodes(self):
hostname = self.server_node["hostname"]
nodes_info = Conf.get(self._index, 'server_node')
for value in nodes_info.values():
host = value["hostname"]
cvg_count = value["storage"]["cvg_count"]
name = value["name"]
self.logger.info(f"update_motr_hare_keys for {host}\n")
for i in range(int(cvg_count)):
lv_md_name = f"lv_raw_md{i + 1}"
cmd = (f"ssh {host}" f" lvs -o lv_path | grep {lv_md_name}")
res = execute_command(self, cmd)
lv_path = res[0].rstrip("\n")
Conf.set(self._index_motr_hare,
f"server>{name}>cvg[{i}]>m0d[0]>md_seg1",
f"{lv_path.strip()}")
Conf.save(self._index_motr_hare)
for value in nodes_info.values():
if (hostname == value["hostname"]):
continue
else:
host = value["hostname"]
cmd = (f"scp {self._motr_hare_conf}"
f" {host}:{self._motr_hare_conf}")
execute_command(self, cmd)
def test_set_delete_and_get(self):
""" Get a deleted key. """
for index in TestConfStore.indexes:
Conf.set(index, 'K1', 'V1')
Conf.delete(index, 'K1')
val = Conf.get(index, 'K1')
self.assertEqual(val, None)
def _update_rmq_cluster_nodes_in_config(self):
"""
Updates RMQ cluster nodes in sspl config file
"""
Conf.set("sspl", "%s>%s" % (self.SECTION, self.CLUSTER_NODES),
self.requested_nodes)
print("Successfully updated RMQ cluster nodes in config.")
def update_ldap_credentials(self):
"""Update ldap credentials (rootdn, sgiam) to openldap_config file."""
try:
# Load the openldap config file
index_id = 'openldap_config_file_write_index'
Conf.load(index_id, f'yaml://{self.openldap_config_file}')
# get the rootdn credentials from provisoner config file
# set the rootdn credentials in to openldap_config file
ldap_root_user = self.get_confvalue(
self.get_confkey('CONFIG>CONFSTORE_ROOTDN_USER_KEY'))
encrypted_rootdn_pass = self.get_confvalue(
self.get_confkey('CONFIG>CONFSTORE_ROOTDN_PASSWD_KEY'))
if encrypted_rootdn_pass is None:
Log.error('rootdn password cannot be None.')
raise Exception('rootdn password cannot be None.')
Conf.set(index_id, f'{self.rootdn_user_key}', f'{ldap_root_user}')
Conf.set(index_id, f'{self.rootdn_pass_key}',
f'{encrypted_rootdn_pass}')
# save openldap config file
Conf.save(index_id)
except Exception as e:
Log.error(f'update rootdn credentials failed, error: {e}')
raise Exception(f'update rootdn credentials failed, error: {e}')
def process(self):
"""Configure SSPL at prepare stage."""
# Update sspl.conf with provisioner supplied input config copy
Conf.set(
consts.SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url",
consts.global_config_path)
Conf.save(consts.SSPL_CONFIG_INDEX)
def _copy_cluster_map(url_index: str):
cluster_data = Conf.get(url_index, 'server_node')
for _, node_data in cluster_data.items():
hostname = node_data.get('hostname')
node_name = node_data.get('name')
Conf.set('cluster', f'cluster>{node_name}', hostname)
Conf.save('cluster')
def run(self):
"""Prepare confstore in factory.
cortx_setup prepare_confstore
"""
template_file_path = str(CONFSTORE_CLUSTER_FILE.parent /
'factory_confstore_template')
Conf.load('node_config_index', f'json://{CONFSTORE_CLUSTER_FILE}')
StateFunExecuter.execute(
'file.managed',
fun_kwargs=dict(
name=template_file_path,
source=
'salt://components/system/files/factory_confstore_template.j2',
template='jinja'))
template_data = ""
with open(template_file_path, 'r') as f:
template_data = f.read().splitlines()
for data in template_data:
if data:
key, value = data.split("=")
Conf.set('node_config_index', key, value)
Conf.save('node_config_index')
self.logger.debug("Done")
def update_sensor_info(config_index, node_type, enclosure_type):
"""Enable or disable sensor monitoring based on node_type
and enclosure_type.
"""
sensors = dict()
sensors["REALSTORSENSORS"] = "true"
sensors["NODEHWSENSOR"] = "true"
sensors["DISKMONITOR"] = "true"
sensors["SERVICEMONITOR"] = "true"
sensors["RAIDSENSOR"] = "true"
sensors["RAIDINTEGRITYSENSOR"] = "true"
sensors["SASPORTSENSOR"] = "true"
sensors["MEMFAULTSENSOR"] = "true"
sensors["CPUFAULTSENSOR"] = "true"
if enclosure_type and enclosure_type.lower() in ["virtual", "vm", "jbod"]:
sensors["REALSTORSENSORS"] = "false"
if node_type and node_type.lower() in ["virtual", "vm"]:
sensors["NODEHWSENSOR"] = "false"
sensors["SASPORTSENSOR"] = "false"
sensors["MEMFAULTSENSOR"] = "false"
sensors["CPUFAULTSENSOR"] = "false"
sensors["RAIDSENSOR"] = "false"
sensors["RAIDINTEGRITYSENSOR"] = "false"
# Update sensor information in config
for sect, value in sensors.items():
Conf.set(config_index, '%s>monitor' % sect, value)
Conf.save(config_index)
def test_conf_store_merge_02no_key(self):
Conf.set('src_index', 'new_key', 'new_value')
Conf.save('src_index')
Conf.merge('dest_index', 'src_index')
Conf.save('dest_index')
self.assertEqual(Conf.get('dest_index', 'new_key'), 'new_value')
self.assertIn('new_key', Conf.get_keys('dest_index'))
def _create_msg_bus_config(kafka_server_list, port_list):
""" Create the config file required for message bus """
from cortx.utils.conf_store import Conf
with open(r'/etc/cortx/message_bus.conf.new', 'w+') as file:
json.dump({}, file, indent=2)
Conf.load("index", "json:///etc/cortx/message_bus.conf.new")
Conf.set("index", "message_broker>type", "kafka")
for i in range(len(kafka_server_list)):
Conf.set("index", f"message_broker>cluster[{i}]", \
{"server": kafka_server_list[i], "port": port_list[i]})
Conf.save("index")
# copy this conf file as message_bus.conf
cmd = "/bin/mv /etc/cortx/message_bus.conf.new" + \
" /etc/cortx/message_bus.conf"
try:
cmd_proc = SimpleProcess(cmd)
res_op, res_err, res_rc = cmd_proc.run()
if res_rc != 0:
raise SetupError(errno.EIO, \
"/etc/cortx/message_bus.conf file creation failed, \
rc = %d", res_rc)
except Exception as e:
raise SetupError(errno.EIO, \
"/etc/cortx/message_bus.conf file creation failed, %s", e)
return res_rc
def _copy_cluster_map(config_path: str):
Conf.load('cluster', config_path, skip_reload=True)
cluster_data = Conf.get('cluster', 'node')
for _, node_data in cluster_data.items():
hostname = node_data.get('hostname')
node_name = node_data.get('name')
Conf.set('cluster', f'cluster>{node_name}', hostname)
Conf.save('cluster')
def test_conf_store_set(self):
"""
Test by setting the key, value to given index and reading it back.
"""
load_config('set_local', 'json:///tmp/file1.json')
Conf.set('set_local', 'bridge>proxy', 'no')
result_data = Conf.get('set_local', 'bridge>proxy')
self.assertEqual(result_data, 'no')
def test_conf_store_merge_03already_key(self):
Conf.set('dest_index', 'cortx>software>kafka>servers[0]', 'host2.com')
Conf.save('dest_index')
Conf.set('src_index', 'cortx>software>kafka>servers[0]', 'host1.com')
Conf.merge('dest_index', 'src_index')
Conf.save('dest_index')
self.assertEqual('host2.com', Conf.get('dest_index',\
'cortx>software>kafka>servers[0]'))
def test_conf_store_set_value_with_null_index(self):
"""Test by setting the key, value to null index."""
load_config('set_local_4', 'json:///tmp/file1.json')
try:
Conf.set('set_local_4', 'bridge>lte_type[]>test', 'sample')
except Exception as err:
self.assertEqual('Invalid key index for the key lte_type',
err.desc)
def test_get_keys_delete(self):
"""Test get_keys after deletion of a key."""
load_config('get_keys_delete', 'json:///tmp/file1.json')
Conf.set('get_keys_delete', 'bridge>delete>key', 'del_val')
pre_key_list = Conf.get_keys('get_keys_delete')
Conf.delete('get_keys_delete', 'bridge>delete>key')
post_key_list = Conf.get_keys('get_keys_delete')
self.assertTrue(pre_key_list != post_key_list)
def update_sensor_info(config_index):
key = 'monitor'
sensors = dict()
sensors["REALSTORSENSORS"] = "true"
sensors["NODEHWSENSOR"] = "true"
sensors["DISKMONITOR"] = "true"
sensors["SERVICEMONITOR"] = "true"
sensors["RAIDSENSOR"] = "true"
sensors["SASPORTSENSOR"] = "true"
sensors["MEMFAULTSENSOR"] = "true"
sensors["CPUFAULTSENSOR"] = "true"
try:
with open("/etc/machine-id") as f:
machine_id = f.read().strip("\n")
except Exception as err:
raise SetupError(1, "Failed to get machine-id. - %s" % (err))
srvnode = Conf.get(GLOBAL_CONFIG_INDEX,
"cluster>server_nodes>%s" % (machine_id))
enclosure_id = Conf.get(GLOBAL_CONFIG_INDEX,
"cluster>%s>storage>enclosure_id" % (srvnode))
node_key_id = Conf.get(GLOBAL_CONFIG_INDEX,
'cluster>server_nodes>%s' % (machine_id))
storage_type = Conf.get(GLOBAL_CONFIG_INDEX,
'storage>%s>type' % enclosure_id)
if storage_type and storage_type.lower() in ["virtual", "jbod"]:
sensors["REALSTORSENSORS"] = "false"
server_type = Conf.get(GLOBAL_CONFIG_INDEX,
'cluster>%s>node_type' % (node_key_id))
if server_type and server_type.lower() in ["virtual"]:
sensors["NODEHWSENSOR"] = "false"
sensors["SASPORTSENSOR"] = "false"
sensors["MEMFAULTSENSOR"] = "false"
sensors["CPUFAULTSENSOR"] = "false"
sensors["RAIDSENSOR"] = "false"
# Onward LDR_R2, consul will be abstracted out and it won't exit as hard dependeny of SSPL
# Note: SSPL has backward compatibility to LDR_R1 and there consul is a dependency of SSPL.
if SSPL_STORE_TYPE == "consul":
host = os.getenv('CONSUL_HOST', CONSUL_HOST)
port = os.getenv('CONSUL_PORT', CONSUL_PORT)
try:
consul_conn = consul.Consul(host=host, port=port)
for sect, value in sensors.items():
consul_conn.kv.put("sspl/config/%s/monitor" % sect, value)
except Exception as cerror:
print("Error in connecting with consul: {}".format(cerror))
# Update sensor information in config
for sect, value in sensors.items():
Conf.set(config_index, '%s>%s' % (sect, key), value)
Conf.save(config_index)
def test_conf_store_set_index(self):
"""
Test by setting the key, value to given index and reading it back.
"""
load_config('set_local1', 'json:///tmp/file1.json')
Conf.set('set_local', 'bridge>lte_type[2]>test',
{'name': '5g', 'location': 'NY'})
result_data = Conf.get('set_local', 'bridge>lte_type[2]>test>location')
self.assertEqual(result_data, 'NY')
def test_conf_store_by_wrong_value(self):
""" Test by setting the wrong value to given key. """
Conf.set('pro_local', 'studio_location', '=amritsar')
Conf.save('pro_local')
try:
load_config('pro_local1', 'properties:///tmp/example.properties')
except Exception as err:
self.assertEqual('Invalid properties store format %s. %s.',
err.args[1])
def test_conf_store_set_key_with_at(self):
"""
Test by setting the key with at '@' special character in it
& reading it back.
"""
load_config('at_local', 'json:///tmp/file1.json')
Conf.set('at_local', 'bridge>proxy@type', 'cloud')
result_data = Conf.get('at_local', 'bridge>proxy@type')
self.assertEqual(result_data, 'cloud')
def test_conf_store_set_nested_keys(self):
"""
Test by setting the key, value to given nested key structure
and retrieve it.
"""
load_config('set_local_9', 'json:///tmp/file1.json')
Conf.set('set_local_9', 'bridge>nstd>k1>k2>k3>k4>5>6>7', 'okay')
result_data = Conf.get('set_local_9', 'bridge>nstd>k1>k2>k3>k4>5>6>7')
self.assertEqual(result_data, 'okay')
def update_sensor_info():
key = 'monitor'
sensors = dict()
sensors["REALSTORSENSORS"] = "true"
sensors["NODEHWSENSOR"] = "true"
sensors["SYSTEMDWATCHDOG"] = "true"
sensors["RAIDSENSOR"] = "true"
sensors["SASPORTSENSOR"] = "true"
sensors["MEMFAULTSENSOR"] = "true"
sensors["CPUFAULTSENSOR"] = "true"
try:
with open("/etc/machine-id") as f:
node_key_id = f.read().strip("\n")
except Exception as err:
print("Failed to get machine-id. - %s" % (err))
storage_type = Conf.get('global_config', 'storage>enclosure_1>type')
if storage_type and storage_type.lower() in ["virtual", "jbod"]:
sensors["REALSTORSENSORS"] = "false"
server_type = Conf.get('global_config',
'cluster>%s>node_type' % (node_key_id))
if server_type and server_type.lower() in ["virtual"]:
sensors["NODEHWSENSOR"] = "false"
sensors["SASPORTSENSOR"] = "false"
sensors["MEMFAULTSENSOR"] = "false"
sensors["CPUFAULTSENSOR"] = "false"
sensors["RAIDSENSOR"] = "false"
for sect, value in sensors.items():
Conf.set('sspl', '%s>%s' % (sect, key), value)
Conf.save('sspl')
# Onward LDR_R2, consul will be abstracted out and it won't exit as hard dependeny of SSPL
# Note: SSPL has backward compatibility to LDR_R1 and there consul is a dependency of SSPL.
if SSPL_STORE_TYPE == "consul":
host = os.getenv('CONSUL_HOST', CONSUL_HOST)
port = os.getenv('CONSUL_PORT', CONSUL_PORT)
try:
consul_conn = consul.Consul(host=host, port=port)
for sect, value in sensors.items():
consul_conn.kv.put("sspl/config/%s/monitor" % sect, value)
except Exception as cerror:
print("Error in connecting with consul: {}".format(cerror))
# Update sensor information for sspl_test
test_file_config_path = "/opt/seagate/%s/sspl/sspl_test/conf/sspl_tests.conf" % PRODUCT_FAMILY
Conf.load('sspl_test', 'yaml://%s' % test_file_config_path)
for sect, value in sensors.items():
Conf.set('sspl_test', '%s>%s' % (sect, key), value)
Conf.save('sspl_test')
def set(args):
""" Set Key Value """
kv_list = args.args[0].split(';')
for kv in kv_list:
try:
key, val = kv.split('=')
except:
raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv)
Conf.set(ConfCli._index, key, val)
Conf.save(ConfCli._index)
def test_conf_store_set_dict_then_string(self):
"""Test by setting the key, dict value to given index and then try to overwrite it with a string."""
load_config('set_local_3', 'json:///tmp/file1.json')
Conf.set('set_local_3', 'bridge>lte_type[2]>test>nested_test', {
'name': '5g',
'location': 'NY'
})
Conf.set('set_local_3', 'bridge>lte_type[2]>test', 'sample')
result_data = Conf.get('set_local_3', 'bridge>lte_type[2]>test')
self.assertEqual(result_data, 'sample')
def test_json_message_kv_store(self):
"""Tests jsonmessage basic operation."""
index = 'json_message_kv_store_index'
Conf.load(index, 'jsonmessage:{"key1":"val1"}')
self.assertEqual(Conf.get(index, 'key1'), 'val1')
Conf.set(index, 'key2', 'val2')
self.assertEqual(Conf.get_keys(index), ['key1', 'key2'])
Conf.set(index, 'key2>key3>key4', 'val4')
Conf.delete(index, 'key2>key3>key4')
self.assertEqual(Conf.get(index, 'key2>key3>key4'), None)
def test_conf_store_set_with_wrong_key(self):
"""
Test by setting the value to invalid wrong key.
"""
load_config('set_local_6', 'json:///tmp/file1.json')
try:
Conf.set('set_local_6', 'bridge>lte_type[2]>..',
{'name': '5g', 'location': 'NY'})
except Exception as err:
self.assertEqual('Invalid key name ', err.desc)
def _provision_components(cortx_conf_url: str, _conf_idx: str, interfaces: Enum, apply_phase: str):
"""Invoke Mini Provisioners of cluster components."""
node_id, _ = CortxProvisioner._get_node_info(_conf_idx)
num_components = int(Conf.get(_conf_idx, f'node>{node_id}>num_components'))
for interface in interfaces:
for comp_idx in range(0, num_components):
key_prefix = f'node>{node_id}>components[{comp_idx}]'
component_name = Conf.get(_conf_idx, f'{key_prefix}>name')
# Check if RPM exists for the component, if it does exist get the build version
component_version = CortxProvisioner.cortx_release.get_component_version(
component_name)
# Get services.
service_idx = 0
services = []
while (Conf.get(_conf_idx, f'{key_prefix}>services[{service_idx}]') is not None):
services.append(Conf.get(_conf_idx, f'{key_prefix}>services[{service_idx}]'))
service_idx = service_idx + 1
service = 'all' if service_idx == 0 else ','.join(services)
if apply_phase == ProvisionerStages.UPGRADE.value:
version = Conf.get(_conf_idx, f'{key_prefix}>version')
# Skip update for component if it is already updated.
is_updated = CortxProvisioner._is_component_updated(component_name, version)
if is_updated is True:
Log.info(f'{component_name} is already updated with {version} version.')
continue
CortxProvisioner._update_provisioning_status(
_conf_idx, node_id, apply_phase, ProvisionerStatus.PROGRESS.value)
if interface.value == 'upgrade':
# TODO: add --changeset parameter once all components support config upgrade
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf_url} --services {service}")
else:
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf_url} --services {service}")
Log.info(f"{cmd}")
cmd_proc = SimpleProcess(cmd)
_, err, rc = cmd_proc.run()
if rc != 0:
CortxProvisioner._update_provisioning_status(
_conf_idx, node_id, apply_phase, ProvisionerStatus.ERROR.value)
raise CortxProvisionerError(
rc, "%s phase of %s, failed. %s", interface.value,
component_name, err)
# Update version for each component if Provisioning successful.
Conf.set(_conf_idx, f'{key_prefix}>version', component_version)
# TODO: Remove the following code when gconf is completely moved to consul.
CortxProvisioner._load_consul_conf(CortxProvisioner._cortx_gconf_consul_index)
Conf.set(CortxProvisioner._cortx_gconf_consul_index,
f'{key_prefix}>version', component_version)
Conf.save(CortxProvisioner._cortx_gconf_consul_index)
def test_get_keys(self):
""" set keys and get keys """
key_list = ['test_k1', 'test_k2', 'test_k3', 'test_k4']
for index in TestConfStore.indexes:
for key in key_list:
Conf.set(index, key, '#!random_value')
get_key_list = Conf.get_keys(index)
self.assertTrue(
all([
True if key in get_key_list else False for key in key_list
]))
def test_get_keys_starts_with(self):
"""set and get keys which starts with a string. """
key_list = ['swtest_k1', 'swtest_k2', 'swtest_k3', 'swtest_k4']
for index in TestConfStore.indexes:
for key in key_list:
Conf.set(index, key, '#!random_value')
get_key_list = Conf.get_keys(index, starts_with='swtest')
self.assertTrue(
all([
True if key in get_key_list else False for key in key_list
]))
def run(self, key=None, value=None):
"""signature set command execution method.
(Specific only to current(given) node.)
Sets LR signature.
Execution:
`cortx_setup signature set --key key --value value`
"""
try:
# Move this definition to common config file
lr_sign_file = Path("/etc/node-signature.yaml")
index = "signature"
# Remove additional trailing spaces
if value:
value = value.strip()
if not (key and value):
raise ValueError(
"A valid signature is mandatory to set for stamping. "
"Expected Signature params format: --key 'key' --value 'value'."
)
if not (lr_sign_file.exists()
and lr_sign_file.stat().st_size != 0):
self.logger.warning("Node Signature was never set "
"in ConfStore. Setting now.")
else:
# Immutable file. Make it editable first.
_cmd = f"chattr -i {lr_sign_file}"
subprocess.Popen(_cmd, shell=True) # nosec
# This step takes a second to process.
sleep(1)
Conf.load(index, f'yaml://{lr_sign_file}')
Conf.set(index, f'signature>{key}', f'{value}')
Conf.save(index)
# TODO: check if this signature data
# should be written to grains and in what format.
# Make the file immutable to any editing
_cmd = f"chattr +i {lr_sign_file} && lsattr {lr_sign_file}"
subprocess.Popen(_cmd, shell=True) # nosec
return f"Node Stamping done and '{key}' is set as '{value}' in Signature"
except Exception as exc:
raise ValueError("Failed to set node signature: %s" % str(exc))
def set(args):
""" Set Key Value """
kv_delim = '=' if args.kv_delim == None else args.kv_delim
if len(kv_delim) > 1 or kv_delim not in [':', '>', '.', '|', '/', '=']:
raise ConfError(errno.EINVAL, "invalid delim %s", kv_delim)
kv_list = args.args[0].split(';')
for kv in kv_list:
try:
key, val = kv.split(kv_delim, 1)
except:
raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv)
Conf.set(ConfCli._index, key, val)
Conf.save(ConfCli._index)
