def init(cls, component: str, source: str): """ Set the Event Message context Parameters: component Component that generates the IEM. For e.g. 'S3', 'SSPL' source Single character that indicates the type of component. For e.g. H-Hardware, S-Software, F-Firmware, O-OS """ cls._component = component cls._source = source try: Conf.load('cluster', cls._conf_file, skip_reload=True) ids = Conf.get('cluster', 'server_node') cls._site_id = ids['site_id'] cls._rack_id = ids['rack_id'] cls._node_id = ids['node_id'] cls._cluster_id = ids['cluster_id'] except Exception as e: raise EventMessageError(errno.EINVAL, "Invalid config in %s. %s", \ cls._conf_file, e) if cls._component is None: raise EventMessageError(errno.EINVAL, "Invalid component type: %s", \ cls._component) if cls._source not in cls._SOURCE.keys(): raise EventMessageError(errno.EINVAL, "Invalid source type: %s", \ cls._source) cls._producer = MessageProducer(producer_id='event_producer', \ message_type='IEM', method='sync')
def main(argv: dict): try: desc = "CORTX Kafka Setup command" command = Cmd.get_command(desc, argv[1:]) # Get kafka server list from template file kafka_config = 'kafka_config' Conf.load(kafka_config, command.url) kafka_servers = Conf.get(kafka_config, 'cortx>software>kafka>servers') # Get log path and initialise Log cluster_conf = MappedConf(command.cluster_conf) log_dir = cluster_conf.get(CLUSTER_CONF_LOG_KEY) log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}') log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('kafka_setup', log_path, level=log_level, backup_count=5,\ file_size_in_mb=5) rc = command.process(kafka_servers) if rc != 0: raise ValueError(f"Failed to run {argv[1]}") except KafkaSetupError as e: sys.stderr.write("%s\n" % str(e)) Cmd.usage(argv[0]) return e.rc() except Exception as e: sys.stderr.write("error: %s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) Cmd.usage(argv[0]) return errno.EINVAL
def validate(self): if not self.args: raise SetupError( errno.EINVAL, "%s - Argument validation failure. Global config is needed", self.name) if (len(self.args) != 2) or (self.args[0] != "--config"): raise SetupError(errno.EINVAL, "%s - Argument validation failure. Check Usage.", self.name) global_config = self.args[1] Conf.load('global_config', global_config) role = Conf.get('global_config', 'release>setup') if not role: raise SetupError(errno.EINVAL, "%s - validation failure. %s", self.name, "Role not found in %s" % (global_config)) from cortx.sspl.bin.sspl_constants import setups if role not in setups: raise SetupError(errno.EINVAL, "%s - validataion failure. %s", self.name, "Role %s is not supported. Check Usage" % role) product = Conf.get('global_config', 'release>product') if not product: raise SetupError(errno.EINVAL, "%s - validation failure. %s", self.name, "Product not found in %s" % (global_config))
def test_conf_load_skip_reload(self): """Test conf load skip_reload argument.""" Conf.load('skip_index', 'json:///tmp/file1.json') expected_lst = Conf.get_keys('skip_index') Conf.load('skip_index', 'toml:///tmp/document.toml', skip_reload=True) out_lst = Conf.get_keys('skip_index') self.assertTrue(True if expected_lst == out_lst else False)
def main(argv: dict): #fetch all data required for processes conf_url = argv[-1] kafka_config = 'kafka_config' Conf.load(kafka_config, conf_url) kafka_servers = Conf.get(kafka_config, 'cortx>software>kafka>servers') try: desc = "CORTX Kafka Setup command" command = Cmd.get_command(desc, argv[1:]) rc = command.process(kafka_servers) if rc != 0: raise ValueError(f"Failed to run {argv[1]}") except KafkaSetupError as e: sys.stderr.write("%s\n" % str(e)) Cmd.usage(argv[0]) return e.rc() except Exception as e: sys.stderr.write("error: %s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) Cmd.usage(argv[0]) return errno.EINVAL
def __init__(self): """ Initialize a MessageBus and load its configurations """ Conf.load('config_file', 'json:///etc/cortx/cortx.conf', skip_reload=True) # if Log.logger is already initialized by some parent process # the same file will be used to log all the messagebus related # logs, else standard message_bus.log will be used. if not Log.logger: log_level = Conf.get('config_file', 'utils>log_level', 'INFO') Log.init('message_bus', '/var/log/cortx/utils/message_bus', \ level=log_level, backup_count=5, file_size_in_mb=5) try: Conf.load('message_bus', self.conf_file, skip_reload=True) self._broker_conf = Conf.get('message_bus', 'message_broker') broker_type = self._broker_conf['type'] Log.info(f"MessageBus initialized as {broker_type}") except ConfError as e: Log.error(f"MessageBusError: {e.rc} Error while parsing" \ f" configuration file {self.conf_file}. {e}.") raise MessageBusError(e.rc, "Error while parsing " + \ "configuration file %s. %s.", self.conf_file, e) except Exception as e: Log.error(f"MessageBusError: {e.rc} Error while parsing" \ f" configuration file {self.conf_file}. {e}.") raise MessageBusError(errno.ENOENT, "Error while parsing " + \ "configuration file %s. %s.", self.conf_file, e) self._broker = MessageBrokerFactory.get_instance(broker_type, \ self._broker_conf)
def main(argv: list): try: if sys.argv[1] == "post_install": Conf.init(delim='.') Conf.load(const.HA_GLOBAL_INDEX, f"yaml://{const.SOURCE_CONFIG_FILE}") log_path = Conf.get(const.HA_GLOBAL_INDEX, "LOG.path") log_level = Conf.get(const.HA_GLOBAL_INDEX, "LOG.level") Log.init(service_name='ha_setup', log_path=log_path, level=log_level) else: ConfigManager.init("ha_setup") desc = "HA Setup command" command = Cmd.get_command(desc, argv[1:]) command.process() sys.stdout.write( f"Mini Provisioning {sys.argv[1]} configured successfully.\n") except Exception as err: Log.error("%s\n" % traceback.format_exc()) sys.stderr.write( f"Setup command:{argv[1]} failed for cortx-ha. Error: {err}\n") return errno.EINVAL
def _get_kafka_server_list(conf_url): """ Reads the ConfStore and derives keys related to message bus """ from cortx.utils.conf_store import Conf Conf.load("cluster_config", conf_url) keylist = ["cortx>software>common>message_bus_type", "cortx>software>kafka>servers"] ConfKeysV().validate("exists", "cluster_config", keylist) msg_bus_type = Conf.get("cluster_config", \ "cortx>software>common>message_bus_type") if msg_bus_type != "kafka": raise SetupError(errno.EINVAL, \ "Message Bus do not support type %s" % msg_bus_type) # Read the required keys all_servers = Conf.get("cluster_config", \ "cortx>software>kafka>servers") no_servers = len(all_servers) kafka_server_list = [] port_list = [] for i in range(no_servers): # check if port is mentioned rc = all_servers[i].find(':') if rc == -1: port_list.append("9092") kafka_server_list.append(all_servers[i]) else: port_list.append(all_servers[i][rc + 1:]) kafka_server_list.append(all_servers[i][:rc]) if len(kafka_server_list) == 0: raise SetupError(errno.EINVAL, \ "No valid Kafka server info provided for Config Key \ 'cortx>software>kafka>servers' ") return kafka_server_list, port_list
def setUpClass(cls,\ cluster_conf_path: str = 'yaml:///etc/cortx/cluster.conf'): """Setup test class.""" TestStore.loaded_json = test_current_file('json:///tmp/file.json') TestStore.loaded_toml = test_current_file('toml:///tmp/document.toml') TestStore.loaded_yaml = test_current_file('yaml:///tmp/sample.yaml') TestStore.loaded_ini = test_current_file('ini:///tmp/test.ini') TestStore.loaded_properties = test_current_file( 'properties:///tmp/example.properties') TestStore.loaded_dir = test_current_file('dir:///tmp/conf_dir_test') TestStore.loaded_dict = test_current_file( 'dict:{"k1":"v1","k2":' '{"k3":"v3", "k4":[25,"v4",27]}}') # Get consul endpoints if TestStore._cluster_conf_path: cls.cluster_conf_path = TestStore._cluster_conf_path else: cls.cluster_conf_path = cluster_conf_path with open(url_config_file) as fd: urls = yaml.safe_load(fd)['conf_url_list'] endpoint_key = urls['consul_endpoints'] Conf.load('config', cls.cluster_conf_path, skip_reload=True) endpoint_url = Conf.get('config', endpoint_key) if endpoint_url is not None and 'http' in endpoint_url: url = endpoint_url.replace('http', 'consul') else: raise KvError(errno.EINVAL, "Invalid consul endpoint key %s", endpoint_key) TestStore.loaded_consul = test_current_file(url)
def update_ldap_credentials(self): """Update ldap credentials (rootdn, sgiam) to openldap_config file.""" try: # Load the openldap config file index_id = 'openldap_config_file_write_index' Conf.load(index_id, f'yaml://{self.openldap_config_file}') # get the rootdn credentials from provisoner config file # set the rootdn credentials in to openldap_config file ldap_root_user = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_ROOTDN_USER_KEY')) encrypted_rootdn_pass = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_ROOTDN_PASSWD_KEY')) if encrypted_rootdn_pass is None: Log.error('rootdn password cannot be None.') raise Exception('rootdn password cannot be None.') Conf.set(index_id, f'{self.rootdn_user_key}', f'{ldap_root_user}') Conf.set(index_id, f'{self.rootdn_pass_key}', f'{encrypted_rootdn_pass}') # save openldap config file Conf.save(index_id) except Exception as e: Log.error(f'update rootdn credentials failed, error: {e}') raise Exception(f'update rootdn credentials failed, error: {e}')
def cleanup(pre_factory: bool, config_path: str): """Remove/Delete all the data that was created after post install.""" # delete message_types from cortx.utils.message_bus.error import MessageBusError try: # use gconf to clean and delete all message type in messagebus from cortx.utils.message_bus import MessageBus, MessageBusAdmin Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get(GCONF_INDEX,\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='cleanup') message_types_list = mb.list_message_types() if message_types_list: mb.deregister_message_type(message_types_list) except MessageBusError as e: raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Can not cleanup Message \ Bus. %s", e) return 0
def main(): from cortx.utils.log import Log from cortx.utils.conf_store import Conf Conf.load('cortx_conf', 'json:///etc/cortx/cortx.conf') log_level = Conf.get('cortx_conf', 'utils>log_level', 'INFO') Log.init('support_bundle', '/var/log/cortx/utils/support/', \ level=log_level, backup_count=5, file_size_in_mb=5, \ syslog_server='localhost', syslog_port=514) # Setup Parser parser = argparse.ArgumentParser(description='Support Bundle CLI', \ formatter_class=RawTextHelpFormatter) sub_parser = parser.add_subparsers(title='command', \ help='represents the action from: generate, get_status\n\n', \ dest='command') # Add Command Parsers members = inspect.getmembers(sys.modules[__name__]) for name, cls in members: if name != "Cmd" and name.endswith("Cmd"): cls.add_args(sub_parser) # Parse and Process Arguments try: args = parser.parse_args() out = args.func(args) if out is not None and len(out) > 0: print(out) return 0 except Exception as e: sys.stderr.write("%s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) return errno.EINVAL
def test_confkeys_exist(self): """Check if keys exist.""" try: Conf.load(index, test_backend_url) ConfKeysV().validate('exists', index, ['bridge', 'bridge>name']) except Exception as e: self.fail(f"Unexpected exception: {e}")
def reset(config_path: str): """Remove/Delete all the data/logs that was created by user/testing.""" from cortx.utils.message_bus.error import MessageBusError try: # use gconf to deregister IEM from cortx.utils.message_bus import MessageBusAdmin, MessageBus from cortx.utils.message_bus import MessageProducer Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get('config',\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='reset') message_types_list = mb.list_message_types() if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type,\ message_type=message_type, method='sync') producer.delete() except MessageBusError as e: raise SetupError(e.rc, "Can not reset Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Internal error, can not \ reset Message Bus. %s", e) return 0
def _get_kafka_server_list(conf_url: str): """ Reads the ConfStore and derives keys related to message bus """ Conf.load('cluster_config', conf_url) key_list = ['cortx>software>common>message_bus_type', \ 'cortx>software>kafka>servers'] ConfKeysV().validate('exists', 'cluster_config', key_list) msg_bus_type = Conf.get('cluster_config', key_list[0]) if msg_bus_type != 'kafka': raise SetupError(errno.EINVAL, "Message Bus do not support type \ %s", msg_bus_type) # Read the required keys all_servers = Conf.get('cluster_config', key_list[1]) no_servers = len(all_servers) kafka_server_list = [] port_list = [] for i in range(no_servers): # check if port is mentioned rc = all_servers[i].find(':') if rc == -1: port_list.append('9092') kafka_server_list.append(all_servers[i]) else: port_list.append(all_servers[i][rc + 1:]) kafka_server_list.append(all_servers[i][:rc]) if len(kafka_server_list) == 0: raise SetupError(errno.EINVAL, "No valid Kafka server info \ provided for config key: %s", key_list[1]) return kafka_server_list, port_list
def main(): from cortx.utils.conf_store import Conf argv = sys.argv # Get the log path tmpl_file = argv[3] Conf.load(GCONF_INDEX, tmpl_file) log_dir = Conf.get(GCONF_INDEX, CLUSTER_CONF_LOG_KEY) utils_log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}') # Get the log level log_level = Conf.get(GCONF_INDEX, 'utils>log_level', 'INFO') Log.init('utils_setup', utils_log_path, level=log_level, backup_count=5, \ file_size_in_mb=5) try: desc = "CORTX Utils Setup command" Log.info(f"Starting utils_setup {argv[1]} ") command = Cmd.get_command(desc, argv[1:]) rc = command.process() except SetupError as e: sys.stderr.write("error: %s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) Cmd.usage(argv[0]) rc = e.rc except Exception as e: sys.stderr.write("error: %s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) rc = errno.EINVAL Log.info(f"Command {command} {argv[1]} finished with exit " \ f"code {rc}")
def validate(self, phase: str): """Perform validations for phase.""" try: Conf.load(phase, f'json://{self._preqs_conf_file}') prereqs_block = Conf.get(phase, f'{phase}') if prereqs_block: pip3s = Conf.get(phase, f'{phase}>pip3s') if pip3s: PkgV().validate('pip3s', pip3s) rpms = Conf.get(phase, f'{phase}>rpms') if rpms: PkgV().validate('rpms', rpms) services = Conf.get(phase, f'{phase}>services') if services: for service in services: pid = os.popen('pidof ' + service).read() if pid is None: Log.debug( 'Validation failed for service %s in %s phase' % (service, phase)) raise Exception( 'Validation failed for service %s in %s phase' % (service, phase)) files = Conf.get(phase, f'{phase}>files') if files: PathV().validate('exists', files) Log.debug("%s - pre-requisite validation complete" % phase) except Exception: Log.debug("%s - pre-requisite validation failed" % phase) raise Exception("prereqs validation failed") return 0
def load_config(cls, solution_conf_url, cortx_conf_url): """ Load config """ cls.solution_conf_url = solution_conf_url cls.cortx_conf_url = cortx_conf_url Conf.load(cls._solution_index, cls.solution_conf_url) cls.cortx_conf = MappedConf(cls.cortx_conf_url)
def _create_msg_bus_config(kafka_server_list, port_list): """ Create the config file required for message bus """ from cortx.utils.conf_store import Conf with open(r'/etc/cortx/message_bus.conf.new', 'w+') as file: json.dump({}, file, indent=2) Conf.load("index", "json:///etc/cortx/message_bus.conf.new") Conf.set("index", "message_broker>type", "kafka") for i in range(len(kafka_server_list)): Conf.set("index", f"message_broker>cluster[{i}]", \ {"server": kafka_server_list[i], "port": port_list[i]}) Conf.save("index") # copy this conf file as message_bus.conf cmd = "/bin/mv /etc/cortx/message_bus.conf.new" + \ " /etc/cortx/message_bus.conf" try: cmd_proc = SimpleProcess(cmd) res_op, res_err, res_rc = cmd_proc.run() if res_rc != 0: raise SetupError(errno.EIO, \ "/etc/cortx/message_bus.conf file creation failed, \ rc = %d", res_rc) except Exception as e: raise SetupError(errno.EIO, \ "/etc/cortx/message_bus.conf file creation failed, %s", e) return res_rc
def upgrade(config_path: str, change_set_path: str): """Perform upgrade steps.""" Conf.load(DELTA_INDEX, change_set_path) Conf.load(GCONF_INDEX, config_path, skip_reload=True) delta_keys = Conf.get_keys(DELTA_INDEX) # if message_bus_backend changed, add new and delete old msg bus entries if CHANGED_PREFIX + MSG_BUS_BACKEND_KEY in delta_keys: new_msg_bus_backend = Conf.get( DELTA_INDEX, CHANGED_PREFIX + MSG_BUS_BACKEND_KEY).split('|')[1] Conf.set(GCONF_INDEX, MSG_BUS_BACKEND_KEY, new_msg_bus_backend) for key in delta_keys: if NEW_PREFIX + EXTERNAL_KEY + new_msg_bus_backend in key: new_key = key.split(NEW_PREFIX)[1] new_val = Conf.get(DELTA_INDEX, key).split('|')[1] Conf.set(GCONF_INDEX, new_key, new_val) if DELETED_PREFIX + EXTERNAL_KEY + new_msg_bus_backend in key: delete_key = key.split(DELETED_PREFIX)[1] Conf.delete(GCONF_INDEX, delete_key) # update existing messagebus parameters else: msg_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY) for key in delta_keys: if CHANGED_PREFIX + EXTERNAL_KEY + msg_bus_backend in key: new_val = Conf.get(DELTA_INDEX, key).split('|')[1] change_key = key.split(CHANGED_PREFIX)[1] Conf.set(GCONF_INDEX, change_key, new_val) Conf.save(GCONF_INDEX) Utils.init(config_path) return 0
def read_ldap_credentials(self): """Read ldap credentials (rootdn, sgiam) from the openldap_config file.""" try: # Load the openldap config file index_id = 'openldap_config_file_read_index' Conf.load(index_id, f'yaml://{self.openldap_config_file}') # Read the cluster id from openldap_config file self.cluster_id = Conf.get(index_id, f'{self.cluster_id_key}') cipher_key = Cipher.generate_key( self.cluster_id, self.get_confkey('CONFSTORE_OPENLDAP_CONST_KEY')) # rootdn username/password self.ldap_root_user = Conf.get(index_id, f'{self.rootdn_user_key}') encrypted_rootdn_pass = Conf.get(index_id, f'{self.rootdn_pass_key}') if encrypted_rootdn_pass != None: self.rootdn_passwd = Cipher.decrypt( cipher_key, bytes(str(encrypted_rootdn_pass), 'utf-8')) except Exception as e: Log.error(f'read ldap credentials failed, error: {e}') raise e
def run(self): """Prepare confstore in factory. cortx_setup prepare_confstore """ template_file_path = str(CONFSTORE_CLUSTER_FILE.parent / 'factory_confstore_template') Conf.load('node_config_index', f'json://{CONFSTORE_CLUSTER_FILE}') StateFunExecuter.execute( 'file.managed', fun_kwargs=dict( name=template_file_path, source= 'salt://components/system/files/factory_confstore_template.j2', template='jinja')) template_data = "" with open(template_file_path, 'r') as f: template_data = f.read().splitlines() for data in template_data: if data: key, value = data.split("=") Conf.set('node_config_index', key, value) Conf.save('node_config_index') self.logger.debug("Done")
def setUpClass(cls): """Test Setup class.""" from cortx.utils.log import Log Log.init('support_bundle', '/var/log/cortx/utils/suppoort/', \ level='DEBUG', backup_count=5, file_size_in_mb=5) cls.sb_description = "Test support bundle generation" Conf.load('cluster_conf', 'json:///etc/cortx/cluster.conf') cls.node_name = Conf.get('cluster_conf', 'cluster>srvnode-1')
def _copy_cluster_map(config_path: str): Conf.load('cluster', config_path, skip_reload=True) cluster_data = Conf.get('cluster', 'node') for _, node_data in cluster_data.items(): hostname = node_data.get('hostname') node_name = node_data.get('name') Conf.set('cluster', f'cluster>{node_name}', hostname) Conf.save('cluster')
def _get_cluster_data(config_path): Conf.load('cluster_conf', config_path) message_bus_backend = Conf.get('cluster_conf',\ 'cortx>utils>message_bus_backend') message_server_endpoints = Conf.get('cluster_conf',\ f'cortx>external>{message_bus_backend}>endpoints') cluster_id = Conf.get('cluster_conf', 'cluster>id') return message_server_endpoints, cluster_id
def __init__(self, args: dict): """ Init method. """ self._url = args.config Conf.load(self._index, self._url) self._args = args.args self._execute = SimpleCommand()
def _load_consul_conf(_idx: str): """Load consul conf with given index if not already loaded.""" #TODO: Remove the function when gconf is moved to consul completely. with open(const.CONSUL_CONF_URL, 'r') as f: gconf_consul_url = f.read() try: Conf.load(_idx, gconf_consul_url) except ConfError: return 1
def validate(self): """Validate init command arguments.""" if not self.args.config: raise SetupError(errno.EINVAL, "%s - Argument validation failure. %s", self.name, "Global config is required.") # Validate config inputs Conf.load(PRVSNR_CONFIG_INDEX, self.args.config[0]) self.sspl_init.validate()
def test_002generated_path(self): bundle_obj = SupportBundle.generate(comment=self.sb_description, components=['csm']) time.sleep(10) Conf.load('cluster_conf', 'json:///etc/cortx/cluster.conf') node_name = Conf.get('cluster_conf', 'cluster>srvnode-1') tar_file_name = f"{bundle_obj.bundle_id}_{node_name}.tar.gz" sb_file_path = f'{bundle_obj.bundle_path}/{bundle_obj.bundle_id}/{node_name}/{tar_file_name}' self.assertEqual(os.path.exists(sb_file_path), True)
def config(config_path: str): """Performs configurations.""" # Load required files Conf.load(GCONF_INDEX, config_path) # set cluster nodename:hostname mapping to cluster.conf Utils._copy_cluster_map(config_path) # saving node and host from gconf Utils._configure_rsyslog() return 0