def init():
""" Perform initialization """
# Create message_type for Event Message
from cortx.utils.message_bus import MessageBusAdmin
try:
admin = MessageBusAdmin(admin_id='register')
admin.register_message_type(message_types=['IEM'], partitions=1)
except MessageBusError as e:
raise SetupError(e.rc, "Unable to create message_type. %s", e)
# start MessageBus service and check status
start_cmd = SimpleProcess("systemctl start cortx_message_bus")
_, start_err, start_rc = start_cmd.run()
if start_rc != 0:
raise SetupError(
start_rc, "Unable to start MessageBus Service \
%s", start_err.decode('utf-8'))
status_cmd = SimpleProcess("systemctl status cortx_message_bus")
_, status_err, status_rc = status_cmd.run()
if status_rc != 0:
raise SetupError(
status_rc, "MessageBus Service is either failed \
inactive. %s", status_err.decode('utf-8'))
return 0
def cleanup(pre_factory: bool):
"""Remove/Delete all the data that was created after post install."""
conf_file = '/etc/cortx/message_bus.conf'
if os.path.exists(conf_file):
# delete message_types
try:
from cortx.utils.message_bus import MessageBusAdmin
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
config_files = ['/etc/cortx/message_bus.conf', \
'/etc/cortx/cluster.conf']
Utils._delete_files(config_files)
if pre_factory:
# deleting all log files as part of pre-factory cleanup
cortx_utils_log_regex = '/var/log/cortx/utils/**/*.log'
log_files = glob.glob(cortx_utils_log_regex, recursive=True)
Utils._delete_files(log_files)
return 0
def reset(config_path: str):
"""Remove/Delete all the data/logs that was created by user/testing."""
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to deregister IEM
from cortx.utils.message_bus import MessageBusAdmin, MessageBus
from cortx.utils.message_bus import MessageProducer
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get('config', MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get('config',\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type,\
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
return 0
def cleanup(pre_factory: bool, config_path: str):
"""Remove/Delete all the data that was created after post install."""
# delete message_types
from cortx.utils.message_bus.error import MessageBusError
try:
# use gconf to clean and delete all message type in messagebus
from cortx.utils.message_bus import MessageBus, MessageBusAdmin
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get(GCONF_INDEX,\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
return 0
def reset():
"""Remove/Delete all the data/logs that was created by user/testing."""
try:
from cortx.utils.message_bus import MessageBusAdmin
from cortx.utils.message_bus.message_bus_client import MessageProducer
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = '/var/log/cortx/utils/'
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0
def cleanup():
"""Remove/Delete all the data that was created after post install."""
conf_file = '/etc/cortx/message_bus.conf'
if os.path.exists(conf_file):
# delete message_types
try:
from cortx.utils.message_bus import MessageBusAdmin
mb = MessageBusAdmin(admin_id='cleanup')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not cleanup Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Can not cleanup Message \
Bus. %s", e)
config_files = ['/etc/cortx/message_bus.conf', \
'/etc/cortx/cluster.conf']
for each_file in config_files:
if os.path.exists(each_file):
# delete data/config stored
try:
os.remove(each_file)
except OSError as e:
raise SetupError(
e.errno, "Error deleting config file %s, \
%s", each_file, e)
return 0
def config(config_template: str):
"""Performs configurations."""
# Copy cluster.conf.sample file to /etc/cortx/cluster.conf
Utils._copy_conf_sample_to_conf()
# Load required files
config_template_index = 'cluster_config'
Conf.load('cluster', 'json:///etc/cortx/cluster.conf', skip_reload=True)
Conf.load(config_template_index, config_template)
try:
server_list, port_list, config = \
MessageBrokerFactory.get_server_list(config_template_index)
except SetupError:
Log.error(f"Could not find server information in {config_template}")
raise SetupError(errno.EINVAL, \
"Could not find server information in %s", config_template)
Utils._create_msg_bus_config(server_list, port_list, config)
# Cluster config
server_info = \
Utils._get_server_info(config_template_index, Conf.machine_id)
if server_info is None:
Log.error(f"Could not find server information in {config_template}")
raise SetupError(errno.EINVAL, "Could not find server " +\
"information in %s", config_template)
Utils._create_cluster_config(server_info)
#set cluster nodename:hostname mapping to cluster.conf
Utils._copy_cluster_map(config_template_index)
Utils._configure_rsyslog()
# get shared storage info from config phase input conf template file
shared_storage = Conf.get('cluster_config', 'cortx>support')
# set shared storage info to cortx.conf conf file
if shared_storage:
Utils._set_to_conf_file('support>shared_path', shared_storage)
# temporary fix for a common message bus log file
# The issue happend when some user other than root:root is trying
# to write logs in these log dir/files. This needs to be removed soon!
LOG_DIR='/var/log'
utils_log_dir = os.path.join(LOG_DIR, 'cortx/utils')
#message_bus
os.makedirs(os.path.join(utils_log_dir, 'message_bus'), exist_ok=True)
os.chmod(os.path.join(utils_log_dir, 'message_bus'), 0o0777)
Path(os.path.join(utils_log_dir,'message_bus/message_bus.log')) \
.touch(exist_ok=True)
os.chmod(os.path.join(utils_log_dir,'message_bus/message_bus.log'), 0o0666)
#iem
os.makedirs(os.path.join(utils_log_dir, 'iem'), exist_ok=True)
os.chmod(os.path.join(utils_log_dir, 'iem'), 0o0777)
Path(os.path.join(utils_log_dir, '/iem/iem.log')).touch(exist_ok=True)
os.chmod(os.path.join(utils_log_dir, 'iem/iem.log'), 0o0666)
return 0
def reset():
"""Remove/Delete all the data/logs that was created by user/testing."""
import time
_purge_retry = 20
try:
from cortx.utils.message_bus import MessageBusAdmin
from cortx.utils.message_bus.message_bus_client import MessageProducer
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
for retry_count in range(1, (_purge_retry + 2)):
if retry_count > _purge_retry:
Log.error(f"MessageBusError: {errors.ERR_OP_FAILED} " \
f" Unable to delete messages for message type" \
f" {message_type} after {retry_count} retries")
raise MessageBusError(errors.ERR_OP_FAILED,\
"Unable to delete messages for message type" + \
"%s after %d retries", message_type, \
retry_count)
rc = producer.delete()
if rc == 0:
break
time.sleep(2 * retry_count)
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = '/var/log/cortx/utils/'
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0
def config(config_template: str):
"""Performs configurations."""
# Copy cluster.conf.sample file to /etc/cortx/cluster.conf
Utils._copy_conf_sample_to_conf()
# Load required files
config_template_index = 'cluster_config'
Conf.load('cluster', 'json:///etc/cortx/cluster.conf')
Conf.load(config_template_index, config_template)
try:
server_list, port_list, config = \
MessageBrokerFactory.get_server_list(config_template_index)
except SetupError:
Log.error(
f"Could not find server information in {config_template}")
raise SetupError(errno.EINVAL, \
"Could not find server information in %s", config_template)
Utils._create_msg_bus_config(server_list, port_list, config)
# Cluster config
server_info = \
Utils._get_server_info(config_template_index, Conf.machine_id)
if server_info is None:
Log.error(
f"Could not find server information in {config_template}")
raise SetupError(errno.EINVAL, "Could not find server " +\
"information in %s", config_template)
Utils._create_cluster_config(server_info)
#set cluster nodename:hostname mapping to cluster.conf
Utils._copy_cluster_map(config_template_index)
Utils._configure_rsyslog()
# temporary fix for a common message bus log file
# The issue happend when some user other than root:root is trying
# to write logs in these log dir/files. This needs to be removed soon!
os.makedirs('/var/log/cortx/utils/message_bus', exist_ok=True)
os.chmod('/var/log/cortx/utils/message_bus', 0o0777)
Path('/var/log/cortx/utils/message_bus/message_bus.log').touch( \
exist_ok=True)
os.chmod('/var/log/cortx/utils/message_bus/message_bus.log', 0o0666)
return 0
def _get_from_conf_file(key) -> str:
""" Fetch and return value for the key from cortx.conf file """
config_file = 'json:///etc/cortx/cortx.conf'
Conf.load('config_file', config_file, skip_reload=True)
val = Conf.get('config_file', key)
if not val:
error_msg = f"Value for key: {key}, not found in {config_file}"
raise SetupError(errno.EINVAL, error_msg)
return val
def _copy_conf_sample_to_conf():
if not os.path.exists("/etc/cortx/cluster.conf.sample"):
with open("/etc/cortx/cluster.conf.sample", "w+") as file:
json.dump({}, file, indent=2)
# copy this sample conf file as cluster.conf
try:
os.rename('/etc/cortx/cluster.conf.sample', \
'/etc/cortx/cluster.conf')
except OSError as e:
raise SetupError(e.errno, "Failed to create /etc/cortx/cluster.conf\
%s", e)
def _get_utils_path() -> str:
""" Gets install path from cortx.conf and returns utils path """
config_file_path = "/etc/cortx/cortx.conf"
Conf.load('config_file', f'yaml:///{config_file_path}')
install_path = Conf.get(index='config_file', key='install_path')
if not install_path:
error_msg = f"install_path not found in {config_file_path}"
raise SetupError(errno.EINVAL, error_msg)
return install_path + "/cortx/utils"
def _delete_files(files: list):
"""
Deletes the passed list of files
Args:
files ([str]): List of files to be deleted
Raises:
SetupError: If unable to delete file
"""
for each_file in files:
if os.path.exists(each_file):
try:
os.remove(each_file)
except OSError as e:
raise SetupError(e.errno, "Error deleting file %s, \
%s", each_file, e)
return 0
def init(config_path: str):
"""Perform initialization."""
# Create message_type for Event Message
from cortx.utils.message_bus import MessageBus, MessageBusAdmin
from cortx.utils.message_bus.error import MessageBusError
try:
# Read the config values
# use gconf to create IEM topic
Conf.load(GCONF_INDEX, config_path, skip_reload=True)
message_bus_backend = Conf.get(GCONF_INDEX, MSG_BUS_BACKEND_KEY)
message_server_endpoints = Conf.get(GCONF_INDEX,\
f'{EXTERNAL_KEY}>{message_bus_backend}>endpoints')
MessageBus.init(message_server_endpoints)
admin = MessageBusAdmin(admin_id='register')
admin.register_message_type(message_types=['IEM',\
'audit_messages'], partitions=1)
except MessageBusError as e:
if 'TOPIC_ALREADY_EXISTS' not in e.desc:
raise SetupError(e.rc, "Unable to create message_type. %s", e)
return 0
def _create_msg_bus_config(message_server_list: list, port_list: list, \
config: dict):
""" Create the config file required for message bus """
with open(r'/etc/cortx/message_bus.conf.sample', 'w+') as file:
json.dump({}, file, indent=2)
Conf.load('index', 'json:///etc/cortx/message_bus.conf.sample')
Conf.set('index', 'message_broker>type', 'kafka')
for i in range(len(message_server_list)):
Conf.set('index', f'message_broker>cluster[{i}]>server', \
message_server_list[i])
Conf.set('index', f'message_broker>cluster[{i}]>port', port_list[i])
Conf.set('index', 'message_broker>message_bus', config)
Conf.save('index')
# copy this conf file as message_bus.conf
try:
os.rename('/etc/cortx/message_bus.conf.sample', \
'/etc/cortx/message_bus.conf')
except OSError as e:
raise SetupError(e.errno, "Failed to create \
/etc/cortx/message_bus.conf %s", e)
def test(plan: str):
""" Perform configuration testing """
# Runs cortx-py-utils unittests as per test plan
try:
Log.info("Validating cortx-py-utils-test rpm")
PkgV().validate('rpms', ['cortx-py-utils-test'])
utils_path = Utils._get_utils_path()
import cortx.utils.test as test_dir
plan_path = os.path.join(os.path.dirname(test_dir.__file__), \
'plans/', plan + '.pln')
Log.info("Running test plan: %s", plan)
cmd = "%s/bin/run_test -t %s" % (utils_path, plan_path)
_output, _err, _rc = SimpleProcess(cmd).run(realtime_output=True)
if _rc != 0:
Log.error("Py-utils Test Failed")
raise TestFailed("Py-utils Test Failed. \n Output : %s "\
"\n Error : %s \n Return Code : %s" %(_output, _err, _rc))
except VError as ve:
Log.error("Failed at package Validation: %s", ve)
raise SetupError(errno.EINVAL, "Failed at package Validation:"\
" %s", ve)
return 0
