def init(): """ Perform initialization """ # Create message_type for Event Message from cortx.utils.message_bus import MessageBusAdmin try: admin = MessageBusAdmin(admin_id='register') admin.register_message_type(message_types=['IEM'], partitions=1) except MessageBusError as e: raise SetupError(e.rc, "Unable to create message_type. %s", e) # start MessageBus service and check status start_cmd = SimpleProcess("systemctl start cortx_message_bus") _, start_err, start_rc = start_cmd.run() if start_rc != 0: raise SetupError( start_rc, "Unable to start MessageBus Service \ %s", start_err.decode('utf-8')) status_cmd = SimpleProcess("systemctl status cortx_message_bus") _, status_err, status_rc = status_cmd.run() if status_rc != 0: raise SetupError( status_rc, "MessageBus Service is either failed \ inactive. %s", status_err.decode('utf-8')) return 0
def cleanup(pre_factory: bool): """Remove/Delete all the data that was created after post install.""" conf_file = '/etc/cortx/message_bus.conf' if os.path.exists(conf_file): # delete message_types try: from cortx.utils.message_bus import MessageBusAdmin mb = MessageBusAdmin(admin_id='cleanup') message_types_list = mb.list_message_types() if message_types_list: mb.deregister_message_type(message_types_list) except MessageBusError as e: raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Can not cleanup Message \ Bus. %s", e) config_files = ['/etc/cortx/message_bus.conf', \ '/etc/cortx/cluster.conf'] Utils._delete_files(config_files) if pre_factory: # deleting all log files as part of pre-factory cleanup cortx_utils_log_regex = '/var/log/cortx/utils/**/*.log' log_files = glob.glob(cortx_utils_log_regex, recursive=True) Utils._delete_files(log_files) return 0
def reset(config_path: str): """Remove/Delete all the data/logs that was created by user/testing.""" from cortx.utils.message_bus.error import MessageBusError try: # use gconf to deregister IEM from cortx.utils.message_bus import MessageBusAdmin, MessageBus from cortx.utils.message_bus import MessageProducer Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get('config',\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='reset') message_types_list = mb.list_message_types() if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type,\ message_type=message_type, method='sync') producer.delete() except MessageBusError as e: raise SetupError(e.rc, "Can not reset Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Internal error, can not \ reset Message Bus. %s", e) return 0
def cleanup(pre_factory: bool, config_path: str): """Remove/Delete all the data that was created after post install.""" # delete message_types from cortx.utils.message_bus.error import MessageBusError try: # use gconf to clean and delete all message type in messagebus from cortx.utils.message_bus import MessageBus, MessageBusAdmin Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get(GCONF_INDEX,\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) mb = MessageBusAdmin(admin_id='cleanup') message_types_list = mb.list_message_types() if message_types_list: mb.deregister_message_type(message_types_list) except MessageBusError as e: raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Can not cleanup Message \ Bus. %s", e) return 0
def reset(): """Remove/Delete all the data/logs that was created by user/testing.""" try: from cortx.utils.message_bus import MessageBusAdmin from cortx.utils.message_bus.message_bus_client import MessageProducer mb = MessageBusAdmin(admin_id='reset') message_types_list = mb.list_message_types() if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type, \ message_type=message_type, method='sync') producer.delete() except MessageBusError as e: raise SetupError(e.rc, "Can not reset Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Internal error, can not \ reset Message Bus. %s", e) # Clear the logs utils_log_path = '/var/log/cortx/utils/' if os.path.exists(utils_log_path): cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path cmd_proc = SimpleProcess(cmd) _, stderr, rc = cmd_proc.run() if rc != 0: raise SetupError(errors.ERR_OP_FAILED, \ "Can not reset log files. %s", stderr) return 0
def cleanup(): """Remove/Delete all the data that was created after post install.""" conf_file = '/etc/cortx/message_bus.conf' if os.path.exists(conf_file): # delete message_types try: from cortx.utils.message_bus import MessageBusAdmin mb = MessageBusAdmin(admin_id='cleanup') message_types_list = mb.list_message_types() if message_types_list: mb.deregister_message_type(message_types_list) except MessageBusError as e: raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Can not cleanup Message \ Bus. %s", e) config_files = ['/etc/cortx/message_bus.conf', \ '/etc/cortx/cluster.conf'] for each_file in config_files: if os.path.exists(each_file): # delete data/config stored try: os.remove(each_file) except OSError as e: raise SetupError( e.errno, "Error deleting config file %s, \ %s", each_file, e) return 0
def config(config_template: str): """Performs configurations.""" # Copy cluster.conf.sample file to /etc/cortx/cluster.conf Utils._copy_conf_sample_to_conf() # Load required files config_template_index = 'cluster_config' Conf.load('cluster', 'json:///etc/cortx/cluster.conf', skip_reload=True) Conf.load(config_template_index, config_template) try: server_list, port_list, config = \ MessageBrokerFactory.get_server_list(config_template_index) except SetupError: Log.error(f"Could not find server information in {config_template}") raise SetupError(errno.EINVAL, \ "Could not find server information in %s", config_template) Utils._create_msg_bus_config(server_list, port_list, config) # Cluster config server_info = \ Utils._get_server_info(config_template_index, Conf.machine_id) if server_info is None: Log.error(f"Could not find server information in {config_template}") raise SetupError(errno.EINVAL, "Could not find server " +\ "information in %s", config_template) Utils._create_cluster_config(server_info) #set cluster nodename:hostname mapping to cluster.conf Utils._copy_cluster_map(config_template_index) Utils._configure_rsyslog() # get shared storage info from config phase input conf template file shared_storage = Conf.get('cluster_config', 'cortx>support') # set shared storage info to cortx.conf conf file if shared_storage: Utils._set_to_conf_file('support>shared_path', shared_storage) # temporary fix for a common message bus log file # The issue happend when some user other than root:root is trying # to write logs in these log dir/files. This needs to be removed soon! LOG_DIR='/var/log' utils_log_dir = os.path.join(LOG_DIR, 'cortx/utils') #message_bus os.makedirs(os.path.join(utils_log_dir, 'message_bus'), exist_ok=True) os.chmod(os.path.join(utils_log_dir, 'message_bus'), 0o0777) Path(os.path.join(utils_log_dir,'message_bus/message_bus.log')) \ .touch(exist_ok=True) os.chmod(os.path.join(utils_log_dir,'message_bus/message_bus.log'), 0o0666) #iem os.makedirs(os.path.join(utils_log_dir, 'iem'), exist_ok=True) os.chmod(os.path.join(utils_log_dir, 'iem'), 0o0777) Path(os.path.join(utils_log_dir, '/iem/iem.log')).touch(exist_ok=True) os.chmod(os.path.join(utils_log_dir, 'iem/iem.log'), 0o0666) return 0
def reset(): """Remove/Delete all the data/logs that was created by user/testing.""" import time _purge_retry = 20 try: from cortx.utils.message_bus import MessageBusAdmin from cortx.utils.message_bus.message_bus_client import MessageProducer mb = MessageBusAdmin(admin_id='reset') message_types_list = mb.list_message_types() if message_types_list: for message_type in message_types_list: producer = MessageProducer(producer_id=message_type, \ message_type=message_type, method='sync') for retry_count in range(1, (_purge_retry + 2)): if retry_count > _purge_retry: Log.error(f"MessageBusError: {errors.ERR_OP_FAILED} " \ f" Unable to delete messages for message type" \ f" {message_type} after {retry_count} retries") raise MessageBusError(errors.ERR_OP_FAILED,\ "Unable to delete messages for message type" + \ "%s after %d retries", message_type, \ retry_count) rc = producer.delete() if rc == 0: break time.sleep(2 * retry_count) except MessageBusError as e: raise SetupError(e.rc, "Can not reset Message Bus. %s", e) except Exception as e: raise SetupError( errors.ERR_OP_FAILED, "Internal error, can not \ reset Message Bus. %s", e) # Clear the logs utils_log_path = '/var/log/cortx/utils/' if os.path.exists(utils_log_path): cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path cmd_proc = SimpleProcess(cmd) _, stderr, rc = cmd_proc.run() if rc != 0: raise SetupError(errors.ERR_OP_FAILED, \ "Can not reset log files. %s", stderr) return 0
def config(config_template: str): """Performs configurations.""" # Copy cluster.conf.sample file to /etc/cortx/cluster.conf Utils._copy_conf_sample_to_conf() # Load required files config_template_index = 'cluster_config' Conf.load('cluster', 'json:///etc/cortx/cluster.conf') Conf.load(config_template_index, config_template) try: server_list, port_list, config = \ MessageBrokerFactory.get_server_list(config_template_index) except SetupError: Log.error( f"Could not find server information in {config_template}") raise SetupError(errno.EINVAL, \ "Could not find server information in %s", config_template) Utils._create_msg_bus_config(server_list, port_list, config) # Cluster config server_info = \ Utils._get_server_info(config_template_index, Conf.machine_id) if server_info is None: Log.error( f"Could not find server information in {config_template}") raise SetupError(errno.EINVAL, "Could not find server " +\ "information in %s", config_template) Utils._create_cluster_config(server_info) #set cluster nodename:hostname mapping to cluster.conf Utils._copy_cluster_map(config_template_index) Utils._configure_rsyslog() # temporary fix for a common message bus log file # The issue happend when some user other than root:root is trying # to write logs in these log dir/files. This needs to be removed soon! os.makedirs('/var/log/cortx/utils/message_bus', exist_ok=True) os.chmod('/var/log/cortx/utils/message_bus', 0o0777) Path('/var/log/cortx/utils/message_bus/message_bus.log').touch( \ exist_ok=True) os.chmod('/var/log/cortx/utils/message_bus/message_bus.log', 0o0666) return 0
def _get_from_conf_file(key) -> str: """ Fetch and return value for the key from cortx.conf file """ config_file = 'json:///etc/cortx/cortx.conf' Conf.load('config_file', config_file, skip_reload=True) val = Conf.get('config_file', key) if not val: error_msg = f"Value for key: {key}, not found in {config_file}" raise SetupError(errno.EINVAL, error_msg) return val
def _copy_conf_sample_to_conf(): if not os.path.exists("/etc/cortx/cluster.conf.sample"): with open("/etc/cortx/cluster.conf.sample", "w+") as file: json.dump({}, file, indent=2) # copy this sample conf file as cluster.conf try: os.rename('/etc/cortx/cluster.conf.sample', \ '/etc/cortx/cluster.conf') except OSError as e: raise SetupError(e.errno, "Failed to create /etc/cortx/cluster.conf\ %s", e)
def _get_utils_path() -> str: """ Gets install path from cortx.conf and returns utils path """ config_file_path = "/etc/cortx/cortx.conf" Conf.load('config_file', f'yaml:///{config_file_path}') install_path = Conf.get(index='config_file', key='install_path') if not install_path: error_msg = f"install_path not found in {config_file_path}" raise SetupError(errno.EINVAL, error_msg) return install_path + "/cortx/utils"
def _delete_files(files: list): """ Deletes the passed list of files Args: files ([str]): List of files to be deleted Raises: SetupError: If unable to delete file """ for each_file in files: if os.path.exists(each_file): try: os.remove(each_file) except OSError as e: raise SetupError(e.errno, "Error deleting file %s, \ %s", each_file, e) return 0
def init(config_path: str): """Perform initialization.""" # Create message_type for Event Message from cortx.utils.message_bus import MessageBus, MessageBusAdmin from cortx.utils.message_bus.error import MessageBusError try: # Read the config values # use gconf to create IEM topic Conf.load(GCONF_INDEX, config_path, skip_reload=True) message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY) message_server_endpoints = Conf.get(GCONF_INDEX,\ f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints') MessageBus.init(message_server_endpoints) admin = MessageBusAdmin(admin_id='register') admin.register_message_type(message_types=['IEM',\ 'audit_messages'], partitions=1) except MessageBusError as e: if 'TOPIC_ALREADY_EXISTS' not in e.desc: raise SetupError(e.rc, "Unable to create message_type. %s", e) return 0
def _create_msg_bus_config(message_server_list: list, port_list: list, \ config: dict): """ Create the config file required for message bus """ with open(r'/etc/cortx/message_bus.conf.sample', 'w+') as file: json.dump({}, file, indent=2) Conf.load('index', 'json:///etc/cortx/message_bus.conf.sample') Conf.set('index', 'message_broker>type', 'kafka') for i in range(len(message_server_list)): Conf.set('index', f'message_broker>cluster[{i}]>server', \ message_server_list[i]) Conf.set('index', f'message_broker>cluster[{i}]>port', port_list[i]) Conf.set('index', 'message_broker>message_bus', config) Conf.save('index') # copy this conf file as message_bus.conf try: os.rename('/etc/cortx/message_bus.conf.sample', \ '/etc/cortx/message_bus.conf') except OSError as e: raise SetupError(e.errno, "Failed to create \ /etc/cortx/message_bus.conf %s", e)
def test(plan: str): """ Perform configuration testing """ # Runs cortx-py-utils unittests as per test plan try: Log.info("Validating cortx-py-utils-test rpm") PkgV().validate('rpms', ['cortx-py-utils-test']) utils_path = Utils._get_utils_path() import cortx.utils.test as test_dir plan_path = os.path.join(os.path.dirname(test_dir.__file__), \ 'plans/', plan + '.pln') Log.info("Running test plan: %s", plan) cmd = "%s/bin/run_test -t %s" % (utils_path, plan_path) _output, _err, _rc = SimpleProcess(cmd).run(realtime_output=True) if _rc != 0: Log.error("Py-utils Test Failed") raise TestFailed("Py-utils Test Failed. \n Output : %s "\ "\n Error : %s \n Return Code : %s" %(_output, _err, _rc)) except VError as ve: Log.error("Failed at package Validation: %s", ve) raise SetupError(errno.EINVAL, "Failed at package Validation:"\ " %s", ve) return 0