def update_motr_hare_keys_for_all_nodes(self): hostname = self.server_node["hostname"] nodes_info = Conf.get(self._index, 'server_node') for value in nodes_info.values(): host = value["hostname"] cvg_count = value["storage"]["cvg_count"] name = value["name"] self.logger.info(f"update_motr_hare_keys for {host}\n") for i in range(int(cvg_count)): lv_md_name = f"lv_raw_md{i + 1}" cmd = (f"ssh {host}" f" lvs -o lv_path | grep {lv_md_name}") res = execute_command(self, cmd) lv_path = res[0].rstrip("\n") Conf.set(self._index_motr_hare, f"server>{name}>cvg[{i}]>m0d[0]>md_seg1", f"{lv_path.strip()}") Conf.save(self._index_motr_hare) for value in nodes_info.values(): if (hostname == value["hostname"]): continue else: host = value["hostname"] cmd = (f"scp {self._motr_hare_conf}" f" {host}:{self._motr_hare_conf}") execute_command(self, cmd)
def test_set_delete_and_get(self): """ Get a deleted key. """ for index in TestConfStore.indexes: Conf.set(index, 'K1', 'V1') Conf.delete(index, 'K1') val = Conf.get(index, 'K1') self.assertEqual(val, None)
def _update_rmq_cluster_nodes_in_config(self): """ Updates RMQ cluster nodes in sspl config file """ Conf.set("sspl", "%s>%s" % (self.SECTION, self.CLUSTER_NODES), self.requested_nodes) print("Successfully updated RMQ cluster nodes in config.")
def update_ldap_credentials(self): """Update ldap credentials (rootdn, sgiam) to openldap_config file.""" try: # Load the openldap config file index_id = 'openldap_config_file_write_index' Conf.load(index_id, f'yaml://{self.openldap_config_file}') # get the rootdn credentials from provisoner config file # set the rootdn credentials in to openldap_config file ldap_root_user = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_ROOTDN_USER_KEY')) encrypted_rootdn_pass = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_ROOTDN_PASSWD_KEY')) if encrypted_rootdn_pass is None: Log.error('rootdn password cannot be None.') raise Exception('rootdn password cannot be None.') Conf.set(index_id, f'{self.rootdn_user_key}', f'{ldap_root_user}') Conf.set(index_id, f'{self.rootdn_pass_key}', f'{encrypted_rootdn_pass}') # save openldap config file Conf.save(index_id) except Exception as e: Log.error(f'update rootdn credentials failed, error: {e}') raise Exception(f'update rootdn credentials failed, error: {e}')
def process(self): """Configure SSPL at prepare stage.""" # Update sspl.conf with provisioner supplied input config copy Conf.set( consts.SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url", consts.global_config_path) Conf.save(consts.SSPL_CONFIG_INDEX)
def _copy_cluster_map(url_index: str): cluster_data = Conf.get(url_index, 'server_node') for _, node_data in cluster_data.items(): hostname = node_data.get('hostname') node_name = node_data.get('name') Conf.set('cluster', f'cluster>{node_name}', hostname) Conf.save('cluster')
def run(self): """Prepare confstore in factory. cortx_setup prepare_confstore """ template_file_path = str(CONFSTORE_CLUSTER_FILE.parent / 'factory_confstore_template') Conf.load('node_config_index', f'json://{CONFSTORE_CLUSTER_FILE}') StateFunExecuter.execute( 'file.managed', fun_kwargs=dict( name=template_file_path, source= 'salt://components/system/files/factory_confstore_template.j2', template='jinja')) template_data = "" with open(template_file_path, 'r') as f: template_data = f.read().splitlines() for data in template_data: if data: key, value = data.split("=") Conf.set('node_config_index', key, value) Conf.save('node_config_index') self.logger.debug("Done")
def update_sensor_info(config_index, node_type, enclosure_type): """Enable or disable sensor monitoring based on node_type and enclosure_type. """ sensors = dict() sensors["REALSTORSENSORS"] = "true" sensors["NODEHWSENSOR"] = "true" sensors["DISKMONITOR"] = "true" sensors["SERVICEMONITOR"] = "true" sensors["RAIDSENSOR"] = "true" sensors["RAIDINTEGRITYSENSOR"] = "true" sensors["SASPORTSENSOR"] = "true" sensors["MEMFAULTSENSOR"] = "true" sensors["CPUFAULTSENSOR"] = "true" if enclosure_type and enclosure_type.lower() in ["virtual", "vm", "jbod"]: sensors["REALSTORSENSORS"] = "false" if node_type and node_type.lower() in ["virtual", "vm"]: sensors["NODEHWSENSOR"] = "false" sensors["SASPORTSENSOR"] = "false" sensors["MEMFAULTSENSOR"] = "false" sensors["CPUFAULTSENSOR"] = "false" sensors["RAIDSENSOR"] = "false" sensors["RAIDINTEGRITYSENSOR"] = "false" # Update sensor information in config for sect, value in sensors.items(): Conf.set(config_index, '%s>monitor' % sect, value) Conf.save(config_index)
def test_conf_store_merge_02no_key(self): Conf.set('src_index', 'new_key', 'new_value') Conf.save('src_index') Conf.merge('dest_index', 'src_index') Conf.save('dest_index') self.assertEqual(Conf.get('dest_index', 'new_key'), 'new_value') self.assertIn('new_key', Conf.get_keys('dest_index'))
def _create_msg_bus_config(kafka_server_list, port_list): """ Create the config file required for message bus """ from cortx.utils.conf_store import Conf with open(r'/etc/cortx/message_bus.conf.new', 'w+') as file: json.dump({}, file, indent=2) Conf.load("index", "json:///etc/cortx/message_bus.conf.new") Conf.set("index", "message_broker>type", "kafka") for i in range(len(kafka_server_list)): Conf.set("index", f"message_broker>cluster[{i}]", \ {"server": kafka_server_list[i], "port": port_list[i]}) Conf.save("index") # copy this conf file as message_bus.conf cmd = "/bin/mv /etc/cortx/message_bus.conf.new" + \ " /etc/cortx/message_bus.conf" try: cmd_proc = SimpleProcess(cmd) res_op, res_err, res_rc = cmd_proc.run() if res_rc != 0: raise SetupError(errno.EIO, \ "/etc/cortx/message_bus.conf file creation failed, \ rc = %d", res_rc) except Exception as e: raise SetupError(errno.EIO, \ "/etc/cortx/message_bus.conf file creation failed, %s", e) return res_rc
def _copy_cluster_map(config_path: str): Conf.load('cluster', config_path, skip_reload=True) cluster_data = Conf.get('cluster', 'node') for _, node_data in cluster_data.items(): hostname = node_data.get('hostname') node_name = node_data.get('name') Conf.set('cluster', f'cluster>{node_name}', hostname) Conf.save('cluster')
def test_conf_store_set(self): """ Test by setting the key, value to given index and reading it back. """ load_config('set_local', 'json:///tmp/file1.json') Conf.set('set_local', 'bridge>proxy', 'no') result_data = Conf.get('set_local', 'bridge>proxy') self.assertEqual(result_data, 'no')
def test_conf_store_merge_03already_key(self): Conf.set('dest_index', 'cortx>software>kafka>servers[0]', 'host2.com') Conf.save('dest_index') Conf.set('src_index', 'cortx>software>kafka>servers[0]', 'host1.com') Conf.merge('dest_index', 'src_index') Conf.save('dest_index') self.assertEqual('host2.com', Conf.get('dest_index',\ 'cortx>software>kafka>servers[0]'))
def test_conf_store_set_value_with_null_index(self): """Test by setting the key, value to null index.""" load_config('set_local_4', 'json:///tmp/file1.json') try: Conf.set('set_local_4', 'bridge>lte_type[]>test', 'sample') except Exception as err: self.assertEqual('Invalid key index for the key lte_type', err.desc)
def test_get_keys_delete(self): """Test get_keys after deletion of a key.""" load_config('get_keys_delete', 'json:///tmp/file1.json') Conf.set('get_keys_delete', 'bridge>delete>key', 'del_val') pre_key_list = Conf.get_keys('get_keys_delete') Conf.delete('get_keys_delete', 'bridge>delete>key') post_key_list = Conf.get_keys('get_keys_delete') self.assertTrue(pre_key_list != post_key_list)
def update_sensor_info(config_index): key = 'monitor' sensors = dict() sensors["REALSTORSENSORS"] = "true" sensors["NODEHWSENSOR"] = "true" sensors["DISKMONITOR"] = "true" sensors["SERVICEMONITOR"] = "true" sensors["RAIDSENSOR"] = "true" sensors["SASPORTSENSOR"] = "true" sensors["MEMFAULTSENSOR"] = "true" sensors["CPUFAULTSENSOR"] = "true" try: with open("/etc/machine-id") as f: machine_id = f.read().strip("\n") except Exception as err: raise SetupError(1, "Failed to get machine-id. - %s" % (err)) srvnode = Conf.get(GLOBAL_CONFIG_INDEX, "cluster>server_nodes>%s" % (machine_id)) enclosure_id = Conf.get(GLOBAL_CONFIG_INDEX, "cluster>%s>storage>enclosure_id" % (srvnode)) node_key_id = Conf.get(GLOBAL_CONFIG_INDEX, 'cluster>server_nodes>%s' % (machine_id)) storage_type = Conf.get(GLOBAL_CONFIG_INDEX, 'storage>%s>type' % enclosure_id) if storage_type and storage_type.lower() in ["virtual", "jbod"]: sensors["REALSTORSENSORS"] = "false" server_type = Conf.get(GLOBAL_CONFIG_INDEX, 'cluster>%s>node_type' % (node_key_id)) if server_type and server_type.lower() in ["virtual"]: sensors["NODEHWSENSOR"] = "false" sensors["SASPORTSENSOR"] = "false" sensors["MEMFAULTSENSOR"] = "false" sensors["CPUFAULTSENSOR"] = "false" sensors["RAIDSENSOR"] = "false" # Onward LDR_R2, consul will be abstracted out and it won't exit as hard dependeny of SSPL # Note: SSPL has backward compatibility to LDR_R1 and there consul is a dependency of SSPL. if SSPL_STORE_TYPE == "consul": host = os.getenv('CONSUL_HOST', CONSUL_HOST) port = os.getenv('CONSUL_PORT', CONSUL_PORT) try: consul_conn = consul.Consul(host=host, port=port) for sect, value in sensors.items(): consul_conn.kv.put("sspl/config/%s/monitor" % sect, value) except Exception as cerror: print("Error in connecting with consul: {}".format(cerror)) # Update sensor information in config for sect, value in sensors.items(): Conf.set(config_index, '%s>%s' % (sect, key), value) Conf.save(config_index)
def test_conf_store_set_index(self): """ Test by setting the key, value to given index and reading it back. """ load_config('set_local1', 'json:///tmp/file1.json') Conf.set('set_local', 'bridge>lte_type[2]>test', {'name': '5g', 'location': 'NY'}) result_data = Conf.get('set_local', 'bridge>lte_type[2]>test>location') self.assertEqual(result_data, 'NY')
def test_conf_store_by_wrong_value(self): """ Test by setting the wrong value to given key. """ Conf.set('pro_local', 'studio_location', '=amritsar') Conf.save('pro_local') try: load_config('pro_local1', 'properties:///tmp/example.properties') except Exception as err: self.assertEqual('Invalid properties store format %s. %s.', err.args[1])
def test_conf_store_set_key_with_at(self): """ Test by setting the key with at '@' special character in it & reading it back. """ load_config('at_local', 'json:///tmp/file1.json') Conf.set('at_local', 'bridge>proxy@type', 'cloud') result_data = Conf.get('at_local', 'bridge>proxy@type') self.assertEqual(result_data, 'cloud')
def test_conf_store_set_nested_keys(self): """ Test by setting the key, value to given nested key structure and retrieve it. """ load_config('set_local_9', 'json:///tmp/file1.json') Conf.set('set_local_9', 'bridge>nstd>k1>k2>k3>k4>5>6>7', 'okay') result_data = Conf.get('set_local_9', 'bridge>nstd>k1>k2>k3>k4>5>6>7') self.assertEqual(result_data, 'okay')
def update_sensor_info(): key = 'monitor' sensors = dict() sensors["REALSTORSENSORS"] = "true" sensors["NODEHWSENSOR"] = "true" sensors["SYSTEMDWATCHDOG"] = "true" sensors["RAIDSENSOR"] = "true" sensors["SASPORTSENSOR"] = "true" sensors["MEMFAULTSENSOR"] = "true" sensors["CPUFAULTSENSOR"] = "true" try: with open("/etc/machine-id") as f: node_key_id = f.read().strip("\n") except Exception as err: print("Failed to get machine-id. - %s" % (err)) storage_type = Conf.get('global_config', 'storage>enclosure_1>type') if storage_type and storage_type.lower() in ["virtual", "jbod"]: sensors["REALSTORSENSORS"] = "false" server_type = Conf.get('global_config', 'cluster>%s>node_type' % (node_key_id)) if server_type and server_type.lower() in ["virtual"]: sensors["NODEHWSENSOR"] = "false" sensors["SASPORTSENSOR"] = "false" sensors["MEMFAULTSENSOR"] = "false" sensors["CPUFAULTSENSOR"] = "false" sensors["RAIDSENSOR"] = "false" for sect, value in sensors.items(): Conf.set('sspl', '%s>%s' % (sect, key), value) Conf.save('sspl') # Onward LDR_R2, consul will be abstracted out and it won't exit as hard dependeny of SSPL # Note: SSPL has backward compatibility to LDR_R1 and there consul is a dependency of SSPL. if SSPL_STORE_TYPE == "consul": host = os.getenv('CONSUL_HOST', CONSUL_HOST) port = os.getenv('CONSUL_PORT', CONSUL_PORT) try: consul_conn = consul.Consul(host=host, port=port) for sect, value in sensors.items(): consul_conn.kv.put("sspl/config/%s/monitor" % sect, value) except Exception as cerror: print("Error in connecting with consul: {}".format(cerror)) # Update sensor information for sspl_test test_file_config_path = "/opt/seagate/%s/sspl/sspl_test/conf/sspl_tests.conf" % PRODUCT_FAMILY Conf.load('sspl_test', 'yaml://%s' % test_file_config_path) for sect, value in sensors.items(): Conf.set('sspl_test', '%s>%s' % (sect, key), value) Conf.save('sspl_test')
def set(args): """ Set Key Value """ kv_list = args.args[0].split(';') for kv in kv_list: try: key, val = kv.split('=') except: raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv) Conf.set(ConfCli._index, key, val) Conf.save(ConfCli._index)
def test_conf_store_set_dict_then_string(self): """Test by setting the key, dict value to given index and then try to overwrite it with a string.""" load_config('set_local_3', 'json:///tmp/file1.json') Conf.set('set_local_3', 'bridge>lte_type[2]>test>nested_test', { 'name': '5g', 'location': 'NY' }) Conf.set('set_local_3', 'bridge>lte_type[2]>test', 'sample') result_data = Conf.get('set_local_3', 'bridge>lte_type[2]>test') self.assertEqual(result_data, 'sample')
def test_json_message_kv_store(self): """Tests jsonmessage basic operation.""" index = 'json_message_kv_store_index' Conf.load(index, 'jsonmessage:{"key1":"val1"}') self.assertEqual(Conf.get(index, 'key1'), 'val1') Conf.set(index, 'key2', 'val2') self.assertEqual(Conf.get_keys(index), ['key1', 'key2']) Conf.set(index, 'key2>key3>key4', 'val4') Conf.delete(index, 'key2>key3>key4') self.assertEqual(Conf.get(index, 'key2>key3>key4'), None)
def test_conf_store_set_with_wrong_key(self): """ Test by setting the value to invalid wrong key. """ load_config('set_local_6', 'json:///tmp/file1.json') try: Conf.set('set_local_6', 'bridge>lte_type[2]>..', {'name': '5g', 'location': 'NY'}) except Exception as err: self.assertEqual('Invalid key name ', err.desc)
def _provision_components(cortx_conf_url: str, _conf_idx: str, interfaces: Enum, apply_phase: str): """Invoke Mini Provisioners of cluster components.""" node_id, _ = CortxProvisioner._get_node_info(_conf_idx) num_components = int(Conf.get(_conf_idx, f'node>{node_id}>num_components')) for interface in interfaces: for comp_idx in range(0, num_components): key_prefix = f'node>{node_id}>components[{comp_idx}]' component_name = Conf.get(_conf_idx, f'{key_prefix}>name') # Check if RPM exists for the component, if it does exist get the build version component_version = CortxProvisioner.cortx_release.get_component_version( component_name) # Get services. service_idx = 0 services = [] while (Conf.get(_conf_idx, f'{key_prefix}>services[{service_idx}]') is not None): services.append(Conf.get(_conf_idx, f'{key_prefix}>services[{service_idx}]')) service_idx = service_idx + 1 service = 'all' if service_idx == 0 else ','.join(services) if apply_phase == ProvisionerStages.UPGRADE.value: version = Conf.get(_conf_idx, f'{key_prefix}>version') # Skip update for component if it is already updated. is_updated = CortxProvisioner._is_component_updated(component_name, version) if is_updated is True: Log.info(f'{component_name} is already updated with {version} version.') continue CortxProvisioner._update_provisioning_status( _conf_idx, node_id, apply_phase, ProvisionerStatus.PROGRESS.value) if interface.value == 'upgrade': # TODO: add --changeset parameter once all components support config upgrade cmd = ( f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}" f" --config {cortx_conf_url} --services {service}") else: cmd = ( f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}" f" --config {cortx_conf_url} --services {service}") Log.info(f"{cmd}") cmd_proc = SimpleProcess(cmd) _, err, rc = cmd_proc.run() if rc != 0: CortxProvisioner._update_provisioning_status( _conf_idx, node_id, apply_phase, ProvisionerStatus.ERROR.value) raise CortxProvisionerError( rc, "%s phase of %s, failed. %s", interface.value, component_name, err) # Update version for each component if Provisioning successful. Conf.set(_conf_idx, f'{key_prefix}>version', component_version) # TODO: Remove the following code when gconf is completely moved to consul. CortxProvisioner._load_consul_conf(CortxProvisioner._cortx_gconf_consul_index) Conf.set(CortxProvisioner._cortx_gconf_consul_index, f'{key_prefix}>version', component_version) Conf.save(CortxProvisioner._cortx_gconf_consul_index)
def test_get_keys(self): """ set keys and get keys """ key_list = ['test_k1', 'test_k2', 'test_k3', 'test_k4'] for index in TestConfStore.indexes: for key in key_list: Conf.set(index, key, '#!random_value') get_key_list = Conf.get_keys(index) self.assertTrue( all([ True if key in get_key_list else False for key in key_list ]))
def test_get_keys_starts_with(self): """set and get keys which starts with a string. """ key_list = ['swtest_k1', 'swtest_k2', 'swtest_k3', 'swtest_k4'] for index in TestConfStore.indexes: for key in key_list: Conf.set(index, key, '#!random_value') get_key_list = Conf.get_keys(index, starts_with='swtest') self.assertTrue( all([ True if key in get_key_list else False for key in key_list ]))
def run(self, key=None, value=None): """signature set command execution method. (Specific only to current(given) node.) Sets LR signature. Execution: `cortx_setup signature set --key key --value value` """ try: # Move this definition to common config file lr_sign_file = Path("/etc/node-signature.yaml") index = "signature" # Remove additional trailing spaces if value: value = value.strip() if not (key and value): raise ValueError( "A valid signature is mandatory to set for stamping. " "Expected Signature params format: --key 'key' --value 'value'." ) if not (lr_sign_file.exists() and lr_sign_file.stat().st_size != 0): self.logger.warning("Node Signature was never set " "in ConfStore. Setting now.") else: # Immutable file. Make it editable first. _cmd = f"chattr -i {lr_sign_file}" subprocess.Popen(_cmd, shell=True) # nosec # This step takes a second to process. sleep(1) Conf.load(index, f'yaml://{lr_sign_file}') Conf.set(index, f'signature>{key}', f'{value}') Conf.save(index) # TODO: check if this signature data # should be written to grains and in what format. # Make the file immutable to any editing _cmd = f"chattr +i {lr_sign_file} && lsattr {lr_sign_file}" subprocess.Popen(_cmd, shell=True) # nosec return f"Node Stamping done and '{key}' is set as '{value}' in Signature" except Exception as exc: raise ValueError("Failed to set node signature: %s" % str(exc))
def set(args): """ Set Key Value """ kv_delim = '=' if args.kv_delim == None else args.kv_delim if len(kv_delim) > 1 or kv_delim not in [':', '>', '.', '|', '/', '=']: raise ConfError(errno.EINVAL, "invalid delim %s", kv_delim) kv_list = args.args[0].split(';') for kv in kv_list: try: key, val = kv.split(kv_delim, 1) except: raise ConfError(errno.EINVAL, "Invalid KV pair %s", kv) Conf.set(ConfCli._index, key, val) Conf.save(ConfCli._index)