def install_files(self, PRODUCT):
"""Configure required log files."""
dbus_service = DbusServiceHandler()
# Copy rsyslog configuration
if not os.path.exists(self.RSYSLOG_CONF):
shutil.copyfile(
"%s/low-level/files/%s" % (SSPL_BASE_DIR, self.RSYSLOG_CONF),
self.RSYSLOG_CONF)
if not os.path.exists(self.RSYSLOG_SSPL_CONF):
shutil.copyfile(
"%s/low-level/files/%s" %
(SSPL_BASE_DIR, self.RSYSLOG_SSPL_CONF),
self.RSYSLOG_SSPL_CONF)
# Create soft link for SINGLE product name service to existing LDR_R1, LR2 service
# Instead of keeping separate service file for SINGLE product with same content.
currentProduct = "%s/conf/sspl-ll.service.%s" % (SSPL_BASE_DIR,
PRODUCT)
if (PRODUCT == "SINGLE" and not os.path.exists(currentProduct)) or \
(PRODUCT == "DUAL" and not os.path.exists(currentProduct)):
os.symlink("%s/conf/sspl-ll.service.%s" % (SSPL_BASE_DIR, PRODUCT),
currentProduct)
if PRODUCT == "CLUSTER" and not os.path.exists(currentProduct):
os.symlink("%s/conf/sspl-ll.service.LR2" % (SSPL_BASE_DIR),
currentProduct)
# Copy sspl-ll.service file and enable service
shutil.copyfile(currentProduct, "/etc/systemd/system/sspl-ll.service")
dbus_service.enable('sspl-ll.service')
daemon_reload_cmd = "systemctl daemon-reload"
output, error, returncode = SimpleProcess(daemon_reload_cmd).run()
if returncode != 0:
raise SetupError(returncode, error, daemon_reload_cmd)
# Copy IEC mapping files
os.makedirs("%s/iem/iec_mapping" % (PRODUCT_BASE_DIR), exist_ok=True)
distutils.dir_util.copy_tree(
"%s/low-level/files/iec_mapping/" % (SSPL_BASE_DIR),
"%s/iem/iec_mapping" % (PRODUCT_BASE_DIR))
# Skip this step if sspl is being configured for node replacement scenario as sspl configurations are
# already available in consul on the healthy node
# Running script for fetching the config using salt and feeding values to consul
# Onward LR2, consul will be abstracted out and it won't exit as hard dependeny of SSPL
if PRODUCT == "LDR_R1":
if not os.path.exists(REPLACEMENT_NODE_ENV_VAR_FILE):
config_from_salt = "%s/sspl_fetch_config_from_salt.py %s %s" % (
self._script_dir, self.ENVIRONMENT, PRODUCT)
output, error, returncode = SimpleProcess(
config_from_salt).run()
if returncode != 0:
raise SetupError(returncode, error, config_from_salt)
class SSPLPostInstall:
"""Prepare environment for SSPL service."""
name = "sspl_post_install"
def __init__(self):
"""Initialize varibales for post install."""
consts.SSPL_LOG_PATH = "/var/log/%s/sspl/" % consts.PRODUCT_FAMILY
consts.SSPL_BUNDLE_PATH = "/var/%s/sspl/bundle/" % consts.PRODUCT_FAMILY
self.state_file = "%s/state.txt" % consts.DATA_PATH
self.dbus_service = DbusServiceHandler()
def validate(self):
"""Check below requirements are met in setup.
1. Check if given product is supported by SSPL
2. Check if given setup is supported by SSPL
3. Check if required pre-requisites softwares are installed.
4. Validate BMC connectivity
5. Validate storage controller connectivity
"""
machine_id = Utility.get_machine_id()
# Validate input/provisioner configs
self.product = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"cortx>release>product")
self.setup = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"cortx>release>setup")
node_type = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>type" % machine_id)
if node_type.lower() not in ["vm", "virtual"]:
bmc_ip = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>bmc>ip" % machine_id)
enclosure_id = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>storage>enclosure_id" % machine_id)
primary_ip = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"storage_enclosure>%s>controller>primary>ip" % enclosure_id)
secondary_ip = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"storage_enclosure>%s>controller>secondary>ip" % enclosure_id)
# Validate product support
if self.product not in consts.enabled_products:
raise SetupError(errno.EINVAL,
"Product '%s' is not in sspl supported product list: %s",
self.product, consts.enabled_products)
# Validate setup support
if self.setup not in consts.setups:
raise SetupError(errno.EINVAL,
"Setup '%s' is not in sspl supported setup list: %s",
self.setup, consts.setups)
# Validate required pip3s and rpms are installed
self.validate_dependencies(self.setup)
# Validate BMC & Storage controller IP reachability
if node_type.lower() not in ["vm", "virtual"]:
# cluster_id required for decrypting the secret is only available from
# the prepare stage. However accessibility validation will be done in
# prepare stage. So at this time, validating ip reachability is fine.
NetworkV().validate("connectivity", [bmc_ip, primary_ip, secondary_ip])
@staticmethod
def validate_dependencies(setup):
"""Validate pre-requisites software packages."""
pip3_3ps_packages_main = {
"cryptography": "2.8",
"jsonschema": "3.2.0",
"pika": "1.1.0",
"pyinotify": "0.9.6",
"python-daemon": "2.2.4",
"requests": "2.25.1",
"zope.component": "4.6.2",
"zope.event": "4.5.0",
"zope.interface": "5.2.0"
}
rpm_3ps_packages = {
"hdparm": "9.43",
"ipmitool": "1.8.18",
"lshw": "B.02.18",
"python3": "3.6.8",
"python36-dbus": "1.2.4",
"python36-gobject": "3.22.0",
"python36-paramiko": "2.1.1",
"python36-psutil": "5.6.7",
"shadow-utils": "4.6",
"smartmontools": "7.0",
"systemd-python36": "1.0.0",
"udisks2": "2.8.4"
}
ssu_dependency_rpms = [
"sg3_utils",
"gemhpi",
"pull_sea_logs",
"python-hpi",
"zabbix-agent-lib",
"zabbix-api-gescheit",
"zabbix-xrtx-lib",
"python-openhpi-baselib",
"zabbix-collector"
]
ssu_required_process = [
"openhpid",
"dcs-collectord"
]
vm_dependency_rpms = []
pkg_validator = PkgV()
pkg_validator.validate_pip3_pkgs(host=socket.getfqdn(),
pkgs=pip3_3ps_packages_main, skip_version_check=False)
pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(),
pkgs=rpm_3ps_packages, skip_version_check=False)
# Check for sspl required processes and misc dependencies if
# setup/role is other than cortx
if setup == "ssu":
pkg_validator.validate("rpms", ssu_dependency_rpms)
ServiceV().validate("isrunning", ssu_required_process)
elif setup == "vm" or setup == "gw" or setup == "cmu":
# No dependency currently. Keeping this section as it
# may be needed in future.
pkg_validator.validate("rpms", vm_dependency_rpms)
# No processes to check in VM environment
def process(self):
"""Create SSPL user and required config files."""
self.create_sspl_conf()
# Load sspl config
Conf.load(consts.SSPL_CONFIG_INDEX, consts.sspl_config_path)
self.create_user()
self.create_directories_and_ownership()
self.configure_sspl_syslog()
self.install_sspl_service_files()
self.enable_sspl_service()
def create_sspl_conf(self):
"""Install product specific sspl config."""
# Copy and load product specific sspl config
if not os.path.exists(consts.file_store_config_path):
shutil.copyfile("%s/conf/sspl.conf.%s.yaml" % (consts.SSPL_BASE_DIR,
self.product), consts.file_store_config_path)
def create_user(self):
"""Add sspl-ll user and validate user creation."""
os.system("/usr/sbin/useradd -r %s -s /sbin/nologin \
-c 'User account to run the %s service'" % (consts.USER, consts.USER))
usernames = [x[0] for x in pwd.getpwall()]
if consts.USER not in usernames:
raise SetupError(errno.EINVAL,
"User %s doesn't exit. Please add user." % consts.USER)
# Add sspl-ll user to required groups and sudoers file etc.
sspl_reinit = "%s/low-level/framework/sspl_reinit" % consts.SSPL_BASE_DIR
_ , error, rc = SimpleProcess(sspl_reinit).run()
if rc:
raise SetupError(rc, "%s failed for with error : %e", sspl_reinit, error)
def create_directories_and_ownership(self):
"""Create ras persistent cache directory and state file.
Assign ownership recursively on the configured directory.
The created state file will be used later by SSPL resourse agent(HA).
"""
# Extract the data path
sspldp = Utility.get_config_value(consts.SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>data_path")
if not sspldp:
raise SetupError(errno.EINVAL, "Data path not set in sspl.conf")
sspl_uid = Utility.get_uid(consts.USER)
sspl_gid = Utility.get_gid(consts.USER)
if sspl_uid == -1 or sspl_gid == -1:
raise SetupError(errno.EINVAL,
"No user found with name : %s", consts.USER)
# Create sspl data directory if not exists
os.makedirs(sspldp, exist_ok=True)
# Create state file under sspl data directory
if not os.path.exists(self.state_file):
file = open(self.state_file, "w")
file.close()
Utility.set_ownership_recursively(consts.SSPL_CONFIGURED_DIR,sspl_uid,sspl_gid)
# Create SSPL log and bundle directories
os.makedirs(consts.SSPL_LOG_PATH, exist_ok=True)
os.makedirs(consts.SSPL_BUNDLE_PATH, exist_ok=True)
# Create /tmp/dcs/hpi if required. Not required for '<product>' role
if self.setup != "cortx":
os.makedirs(consts.HPI_PATH, mode=0o777, exist_ok=True)
zabbix_uid = Utility.get_uid("zabbix")
if zabbix_uid != -1:
os.chown(consts.HPI_PATH, zabbix_uid, -1)
# Create mdadm.conf to set ACL on it.
with open(consts.MDADM_PATH, 'a'):
os.utime(consts.MDADM_PATH)
os.chmod(consts.MDADM_PATH, mode=0o666)
os.chown(consts.MDADM_PATH, sspl_uid, -1)
def configure_sspl_syslog(self):
"""Configure log file path in rsyslog and update logrotate config file."""
system_files_root = "%s/low-level/files" % consts.SSPL_BASE_DIR
sspl_log_file_path = Utility.get_config_value(consts.SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>sspl_log_file_path")
sspl_sb_log_file_path = sspl_log_file_path.replace("/sspl.log","/sspl_support_bundle.log")
iem_log_file_path = Utility.get_config_value(consts.SSPL_CONFIG_INDEX,
"IEMSENSOR>log_file_path")
manifest_log_file_path = sspl_log_file_path.replace("/sspl.log","/manifest.log")
# IEM configuration
os.makedirs("%s/iem/iec_mapping" % consts.PRODUCT_BASE_DIR, exist_ok=True)
distutils.dir_util.copy_tree("%s/iec_mapping/" % system_files_root,
"%s/iem/iec_mapping" % consts.PRODUCT_BASE_DIR)
if not os.path.exists(consts.RSYSLOG_IEM_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_IEM_CONF),
consts.RSYSLOG_IEM_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_IEM_CONF,
'File.*[=,"]', 'File="%s"' % iem_log_file_path)
# SSPL rsys log configuration
if not os.path.exists(consts.RSYSLOG_SSPL_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_SSPL_CONF),
consts.RSYSLOG_SSPL_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_SSPL_CONF, 'File.*[=,"]',
'File="%s"' % sspl_log_file_path)
# Manifest Bundle log configuration
if not os.path.exists(consts.RSYSLOG_MSB_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_MSB_CONF),
consts.RSYSLOG_MSB_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_MSB_CONF, 'File.*[=,"]',
'File="%s"' % manifest_log_file_path)
# Support Bundle log configuration
if not os.path.exists(consts.RSYSLOG_SB_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_SB_CONF),
consts.RSYSLOG_SB_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_SB_CONF, 'File.*[=,"]',
'File="%s"' % sspl_sb_log_file_path)
# Configure logrotate
# Create logrotate dir in case it's not present
os.makedirs(consts.LOGROTATE_DIR, exist_ok=True)
Utility.replace_expr("%s/etc/logrotate.d/iem_messages" % system_files_root,
0, iem_log_file_path)
Utility.replace_expr("%s/etc/logrotate.d/sspl_logs" % system_files_root,
0, sspl_log_file_path)
Utility.replace_expr("%s/etc/logrotate.d/sspl_sb_logs" % system_files_root,
0, sspl_sb_log_file_path)
shutil.copy2("%s/etc/logrotate.d/iem_messages" % system_files_root,
consts.IEM_LOGROTATE_CONF)
shutil.copy2("%s/etc/logrotate.d/sspl_logs" % system_files_root,
consts.SSPL_LOGROTATE_CONF)
shutil.copy2("%s/etc/logrotate.d/manifest_logs" % system_files_root,
consts.MSB_LOGROTATE_CONF)
shutil.copy2("%s/etc/logrotate.d/sspl_sb_logs" % system_files_root,
consts.SB_LOGROTATE_CONF)
# This rsyslog restart will happen after successful updation of rsyslog
# conf file and before sspl starts. If at all this will be removed from
# here, there will be a chance that SSPL intial logs will not be present in
# "/var/log/<product>/sspl/sspl.log" file. So, initial logs needs to be collected from
# "/var/log/messages"
service = DbusServiceHandler()
service.restart('rsyslog.service')
def install_sspl_service_files(self):
"""Copy service file to systemd location based on product."""
# Create soft link for SINGLE product name service to existing LDR_R1, LR2 service
# Instead of keeping separate service file for SINGLE product with same content.
currentProduct = "%s/conf/sspl-ll.service.%s" % (consts.SSPL_BASE_DIR,
self.product)
if (self.product == "SINGLE" and not os.path.exists(currentProduct)) or \
(self.product == "DUAL" and not os.path.exists(currentProduct)):
os.symlink("%s/conf/sspl-ll.service.%s" % (consts.SSPL_BASE_DIR, self.product),
currentProduct)
if self.product == "CLUSTER" and not os.path.exists(currentProduct):
os.symlink("%s/conf/sspl-ll.service.LR2" % (consts.SSPL_BASE_DIR),
currentProduct)
shutil.copyfile(currentProduct, "/etc/systemd/system/sspl-ll.service")
def enable_sspl_service(self):
"""Enable sspl-ll service."""
self.dbus_service.enable("sspl-ll.service")
daemon_reload_cmd = "systemctl daemon-reload"
output, error, rc = SimpleProcess(daemon_reload_cmd).run()
if rc != 0:
raise SetupError(rc, error, daemon_reload_cmd)
class SSPLTestCmd:
"""Starts test based on plan (sanity|alerts|self_primary|self_secondary)."""
def __init__(self, args: list):
self.args = args
self.name = "sspl_test"
self.plan = "self"
self.avoid_rmq = False
self.dbus_service = DbusServiceHandler()
if args.config and args.config[0]:
self.sspl_test_gc_url = args.config[0]
else:
self.sspl_test_gc_url = global_config_path
self.sspl_test_gc_copy_file = "/etc/sspl_test_gc_url.yaml"
def validate(self):
"""Check for required packages are installed."""
# RPM dependency
rpm_deps = {"cortx-sspl-test": None}
# python 3rd party package dependency
pip3_3ps_packages_test = {"Flask": "1.1.1"}
pkg_validator = PkgV()
pkg_validator.validate_pip3_pkgs(host=socket.getfqdn(),
pkgs=pip3_3ps_packages_test,
skip_version_check=False)
pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(),
pkgs=rpm_deps,
skip_version_check=True)
# Load global, sspl and test configs
Conf.load(SSPL_CONFIG_INDEX, sspl_config_path)
Conf.load(SSPL_TEST_CONFIG_INDEX, sspl_test_config_path)
# Take copy of supplied config passed to sspl_test and load it
with open(self.sspl_test_gc_copy_file, "w") as f:
f.write("")
self.sspl_test_gc_copy_url = "yaml://%s" % self.sspl_test_gc_copy_file
Conf.load(SSPL_TEST_GLOBAL_CONFIG, self.sspl_test_gc_copy_url)
Conf.load("global_config", self.sspl_test_gc_url)
Conf.copy("global_config", SSPL_TEST_GLOBAL_CONFIG)
# Validate input configs
machine_id = Utility.get_machine_id()
self.node_type = Conf.get(SSPL_TEST_GLOBAL_CONFIG,
"server_node>%s>type" % machine_id)
enclosure_id = Conf.get(
SSPL_TEST_GLOBAL_CONFIG,
"server_node>%s>storage>enclosure_id" % machine_id)
self.enclosure_type = Conf.get(
SSPL_TEST_GLOBAL_CONFIG,
"storage_enclosure>%s>type" % enclosure_id)
def process(self):
"""Run test using user requested test plan."""
self.plan = self.args.plan[0]
self.avoid_rmq = self.args.avoid_rmq
# if self.plan is other than "self"
# then only config change and service restart is required.
if self.plan not in ["self", "self_primary", "self_secondary"]:
# Take back up of sspl test config
sspl_test_backup = '/etc/sspl_tests.conf.back'
shutil.copyfile(sspl_test_file_path, sspl_test_backup)
# Add global config in sspl_test config and revert the changes once test completes.
# Global config path in sspl_tests.conf will be referred by sspl_tests later
sspl_global_config_url = Conf.get(
SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url")
Conf.set(SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
self.sspl_test_gc_copy_url)
Conf.save(SSPL_CONFIG_INDEX)
# Enable & disable sensors based on environment
update_sensor_info(SSPL_TEST_CONFIG_INDEX, self.node_type,
self.enclosure_type)
# TODO: Move lines 99-131 & 152-159 to RunQATest class
# Create dummy service and add service name in /etc/sspl.conf
service_name = "dummy_service.service"
service_file_path_src = f"{TEST_DIR}/alerts/os/dummy_service_files/dummy_service.service"
service_executable_code_src = f"{TEST_DIR}/alerts/os/dummy_service_files/dummy_service.py"
service_file_path_des = "/etc/systemd/system"
service_executable_code_des = "/var/cortx/sspl/test"
os.makedirs(service_executable_code_des, 0o777, exist_ok=True)
shutil.copy(service_executable_code_src,
f'{service_executable_code_des}/dummy_service.py')
# Make service file executable.
cmd = f"chmod +x {service_executable_code_des}/dummy_service.py"
_, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
print("%s error occurred while executing cmd: %s" %
(error, cmd))
print("failed to assign execute permission for dummy_service.py."\
" dummy_service will fail.")
# Copy service file to /etc/systemd/system/ path.
shutil.copyfile(service_file_path_src,
f'{service_file_path_des}/dummy_service.service')
cmd = "systemctl daemon-reload"
_, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
print(f"failed to execute '{cmd}', systemctl will be unable"\
f" to manage the dummy_service.service \nError: {error}")
self.dbus_service.enable(service_name)
self.dbus_service.start(service_name)
service_list = Conf.get(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>monitored_services")
service_list.append(service_name)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>monitored_services",
service_list)
threshold_inactive_time_original = Conf.get(
SSPL_CONFIG_INDEX, "SERVICEMONITOR>threshold_inactive_time")
threshold_inactive_time_new = 30
Conf.set(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>threshold_inactive_time",
threshold_inactive_time_new)
Conf.save(SSPL_CONFIG_INDEX)
# TODO: Convert shell script to python
# from cortx.sspl.sspl_test.run_qa_test import RunQATest
# RunQATest(self.plan, self.avoid_rmq).run()
CMD = "%s/run_qa_test.sh %s %s" % (TEST_DIR, self.plan,
self.avoid_rmq)
try:
output, error, rc = SimpleProcess(CMD).run(
realtime_output=True)
except KeyboardInterrupt:
rc = 1
error = "KeyboardInterrupt occurred while executing sspl test."
# Restore the original path/file & service, then throw exception
# if execution is failed.
service_list.remove(service_name)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>monitored_services",
service_list)
Conf.set(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>threshold_inactive_time",
threshold_inactive_time_original)
Conf.set(SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
sspl_global_config_url)
Conf.save(SSPL_CONFIG_INDEX)
shutil.copyfile(sspl_test_backup, sspl_test_file_path)
if rc != 0:
raise TestException("%s - ERROR: %s - CMD %s" %
(self.name, error, CMD))
else:
# TODO: Convert shell script to python
# from cortx.sspl.sspl_test.run_qa_test import RunQATest
# RunQATest(self.plan, self.avoid_rmq).run()
try:
CMD = "%s/run_qa_test.sh %s %s" % (TEST_DIR, self.plan,
self.avoid_rmq)
output, error, returncode = SimpleProcess(CMD).run(
realtime_output=True)
except KeyboardInterrupt:
raise TestException(
"KeyboardInterrupt occurred while executing sspl test.")
except Exception as error:
raise TestException(
"Error occurred while executing self test: %s" % error)
class SSPLTestCmd:
"""Starts test based on plan (sanity|alerts|self_primary|self_secondary)."""
def __init__(self, args: list):
self.args = args
self.name = "sspl_test"
self.plan = "self_primary"
self.avoid_rmq = False
self.dbus_service = DbusServiceHandler()
# Load global, sspl and test configs
Conf.load(SSPL_CONFIG_INDEX, sspl_config_path)
global_config_url = Conf.get(
SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url")
Conf.load(GLOBAL_CONFIG_INDEX, global_config_url)
Conf.load(SSPL_TEST_CONFIG_INDEX, sspl_test_config_path)
@staticmethod
def validate():
"""Check for required packages are installed."""
# python 3rd party package dependency
pip3_3ps_packages_test = {"Flask": "1.1.1"}
pkg_validator = PkgV()
pkg_validator.validate_pip3_pkgs(host=None,
pkgs=pip3_3ps_packages_test,
skip_version_check=False)
def process(self):
self.plan = self.args.plan[0]
self.avoid_rmq = self.args.avoid_rmq
# Take back up of sspl test config
sspl_test_backup = '/etc/sspl_tests.conf.back'
shutil.copyfile(sspl_test_file_path, sspl_test_backup)
# Add global config in sspl_test config and revert the changes once test completes.
# Global config path in sspl_tests.conf will be referred by sspl_tests later
global_config_copy_url = Conf.get(
SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url")
Conf.copy(GLOBAL_CONFIG_INDEX, SSPL_TEST_CONFIG_INDEX)
Conf.set(SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
sspl_test_config_path)
Conf.save(SSPL_CONFIG_INDEX)
# Enable & disable sensors based on environment
update_sensor_info(SSPL_TEST_CONFIG_INDEX)
# Get rabbitmq values from sspl.conf and update sspl_tests.conf
rmq_passwd = Conf.get(SSPL_CONFIG_INDEX,
"RABBITMQEGRESSPROCESSOR>password")
Conf.set(SSPL_TEST_CONFIG_INDEX, "RABBITMQEGRESSPROCESSOR>password",
rmq_passwd)
Conf.save(SSPL_TEST_CONFIG_INDEX)
# TODO: Move lines 90-116 & 125-127 to RunQATest class
# Create dummy service and add service name in /etc/sspl.conf
service_name = "dummy_service.service"
service_file_path_src = f"{TEST_DIR}/alerts/os/dummy_service_files/dummy_service.service"
service_executable_code_src = f"{TEST_DIR}/alerts/os/dummy_service_files/dummy_service.py"
service_file_path_des = "/etc/systemd/system"
service_executable_code_des = "/var/cortx/sspl/test"
os.makedirs(service_executable_code_des, 0o777, exist_ok=True)
shutil.copy(service_executable_code_src,
f'{service_executable_code_des}/dummy_service.py')
# Make service file executable.
cmd = f"chmod +x {service_executable_code_des}/dummy_service.py"
_, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
print("%s error occurred while executing cmd: %s" % (error, cmd))
print("failed to assign execute permission for dummy_service.py."\
" dummy_service will fail.")
# Copy service file to /etc/systemd/system/ path.
shutil.copyfile(service_file_path_src,
f'{service_file_path_des}/dummy_service.service')
cmd = "systemctl daemon-reload"
_, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
print(f"failed to execute '{cmd}', systemctl will be unable"\
f" to manage the dummy_service.service \nError: {error}")
self.dbus_service.enable(service_name)
self.dbus_service.start(service_name)
service_list = Conf.get(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>monitored_services")
service_list.append(service_name)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>monitored_services",
service_list)
threshold_inactive_time_original = Conf.get(
SSPL_CONFIG_INDEX, "SERVICEMONITOR>threshold_inactive_time")
threshold_inactive_time_new = 30
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>threshold_inactive_time",
threshold_inactive_time_new)
Conf.save(SSPL_CONFIG_INDEX)
# TODO: Convert shell script to python
# from cortx.sspl.sspl_test.run_qa_test import RunQATest
# RunQATest(self.plan, self.avoid_rmq).run()
CMD = "%s/run_qa_test.sh %s %s" % (TEST_DIR, self.plan, self.avoid_rmq)
output, error, returncode = SimpleProcess(CMD).run(
realtime_output=True)
# Restore the original path/file & service, then throw exception
# if execution is failed.
service_list.remove(service_name)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>monitored_services",
service_list)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>threshold_inactive_time",
threshold_inactive_time_original)
Conf.set(SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
global_config_copy_url)
Conf.save(SSPL_CONFIG_INDEX)
shutil.copyfile(sspl_test_backup, sspl_test_file_path)
self.dbus_service.restart('sspl-ll.service')
if returncode != 0:
raise SetupError(returncode, "%s - ERROR: %s - CMD %s", self.name,
error, CMD)
class SSPLTestCmd:
"""Starts test based on plan (sanity|alerts|dev_sanity|full|performance|scalability|regression)."""
def __init__(self, args: list):
self.args = args
self.name = "sspl_test"
self.plan = "sanity"
self.coverage_enabled = self.args.coverage
self.dbus_service = DbusServiceHandler()
if args.config and args.config[0]:
self.sspl_test_gc_url = args.config[0]
else:
self.sspl_test_gc_url = global_config_path
self.sspl_test_gc_copy_file = "/etc/sspl_test_gc_url.yaml"
def validate(self):
"""Check for required packages are installed."""
# RPM dependency
rpm_deps = {"cortx-sspl-test": None}
# python 3rd party package dependency
pip3_packages_dep = {"Flask": "1.1.1", "coverage": "5.5"}
if not self.coverage_enabled:
pip3_packages_dep.pop("coverage")
# Validate pip3 python pkg with required version.
for pkg, version in pip3_packages_dep.items():
installed_pkg = None
uninstalled_pkg = False
try:
pkg_req = Requirement.parse(f"{pkg}=={version}")
installed_pkg = working_set.find(pkg_req)
except VersionConflict:
cmd = f'pip3 uninstall -y {pkg}'
_, err, ret = SimpleProcess(cmd).run()
if ret:
raise TestException(
"Failed to uninstall the pip3 pkg: %s(v%s), "
"due to an Error: %s" % (pkg, version, err))
uninstalled_pkg = True
except Exception as err:
raise TestException("Failed at verification of pip3 pkg: %s, "
"due to an Error: %s" % (pkg, err))
if not installed_pkg or uninstalled_pkg:
cmd = f'pip3 install {pkg}=={version}'
_, err, ret = SimpleProcess(cmd).run()
if ret:
raise TestException(
"Failed to install the pip3 pkg: %s(v%s), "
"due to an Error: %s" % (pkg, version, err))
logger.info(f"Ensured Package Dependency: {pkg}(v{version}).")
# Validate rpm dependencies
pkg_validator = PkgV()
pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(),
pkgs=rpm_deps,
skip_version_check=True)
# Load global, sspl and test configs
Conf.load(SSPL_CONFIG_INDEX, sspl_config_path)
Conf.load(SSPL_TEST_CONFIG_INDEX, sspl_test_config_path)
# Take copy of supplied config passed to sspl_test and load it
with open(self.sspl_test_gc_copy_file, "w") as f:
f.write("")
self.sspl_test_gc_copy_url = "yaml://%s" % self.sspl_test_gc_copy_file
Conf.load(SSPL_TEST_GLOBAL_CONFIG, self.sspl_test_gc_copy_url)
Conf.load("global_config", self.sspl_test_gc_url)
Conf.copy("global_config", SSPL_TEST_GLOBAL_CONFIG)
# Validate input configs
machine_id = Utility.get_machine_id()
self.node_type = Conf.get(SSPL_TEST_GLOBAL_CONFIG,
"server_node>%s>type" % machine_id)
enclosure_id = Conf.get(
SSPL_TEST_GLOBAL_CONFIG,
"server_node>%s>storage>enclosure_id" % machine_id)
self.enclosure_type = Conf.get(
SSPL_TEST_GLOBAL_CONFIG,
"storage_enclosure>%s>type" % enclosure_id)
def process(self):
"""Run test using user requested test plan."""
self.plan = self.args.plan[0]
# if self.plan is other than "self"
# then only config change and service restart is required.
if self.plan not in IVT_TEST_PLANS:
# Take back up of sspl test config
sspl_test_backup = '/etc/sspl_tests.conf.back'
shutil.copyfile(sspl_test_file_path, sspl_test_backup)
# Add global config in sspl_test config and revert the changes once
# test completes. Global config path in sspl_tests.conf will be
# referred by sspl_tests later
sspl_global_config_url = Conf.get(
SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url")
Conf.set(SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
self.sspl_test_gc_copy_url)
Conf.save(SSPL_CONFIG_INDEX)
# Enable & disable sensors based on environment
update_sensor_info(SSPL_TEST_CONFIG_INDEX, self.node_type,
self.enclosure_type)
# TODO: Move lines 99-131 & 152-159 to RunQATest class
# Create dummy service and add service name in /etc/sspl.conf
service_name = "dummy_service.service"
service_file_path_src = \
f"{TEST_DIR}/alerts/os/dummy_service_files/dummy_service.service"
service_executable_code_src = \
f"{TEST_DIR}/alerts/os/dummy_service_files/dummy_service.py"
service_file_path_des = "/etc/systemd/system"
service_executable_code_des = "/var/cortx/sspl/test"
os.makedirs(service_executable_code_des, 0o777, exist_ok=True)
shutil.copy(service_executable_code_src,
f'{service_executable_code_des}/dummy_service.py')
# Make service file executable.
cmd = f"chmod +x {service_executable_code_des}/dummy_service.py"
_, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
logger.error("%s error occurred while executing cmd: %s" %
(error, cmd))
logger.error("failed to assign execute permission for"
"dummy_service.py. dummy_service will fail.")
# Copy service file to /etc/systemd/system/ path.
shutil.copyfile(service_file_path_src,
f'{service_file_path_des}/dummy_service.service')
cmd = "systemctl daemon-reload"
_, error, returncode = SimpleProcess(cmd).run()
if returncode != 0:
logger.error(f"failed to execute '{cmd}',"
"systemctl will be unable to manage the"
f"dummy_service.service \n Error: {error}")
self.dbus_service.enable(service_name)
self.dbus_service.start(service_name)
service_list = Conf.get(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>monitored_services")
service_list.append(service_name)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>monitored_services",
service_list)
threshold_inactive_time_original = Conf.get(
SSPL_CONFIG_INDEX, "SERVICEMONITOR>threshold_inactive_time")
threshold_inactive_time_new = 30
Conf.set(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>threshold_inactive_time",
threshold_inactive_time_new)
Conf.save(SSPL_CONFIG_INDEX)
cpu_usage_alert_wait = Conf.get(
SSPL_CONFIG_INDEX,
"NODEDATAMSGHANDLER>high_cpu_usage_wait_threshold")
memory_usage_alert_wait = Conf.get(
SSPL_CONFIG_INDEX,
"NODEDATAMSGHANDLER>high_memory_usage_wait_threshold")
cpu_usage_alert_wait_new = 10
memory_usage_alert_wait_new = 20
Conf.set(SSPL_CONFIG_INDEX,
"NODEDATAMSGHANDLER>high_cpu_usage_wait_threshold",
cpu_usage_alert_wait_new)
Conf.set(SSPL_CONFIG_INDEX,
"NODEDATAMSGHANDLER>high_memory_usage_wait_threshold",
memory_usage_alert_wait_new)
Conf.save(SSPL_CONFIG_INDEX)
# TODO: Convert shell script to python
# from cortx.sspl.sspl_test.run_qa_test import RunQATest
# RunQATest(self.plan, self.coverage_enabled).run()
CMD = "%s/run_qa_test.sh --plan %s --coverage %s"\
%(TEST_DIR, self.plan, self.coverage_enabled)
try:
_, error, rc = SimpleProcess(CMD).run(realtime_output=True)
except KeyboardInterrupt:
rc = 1
error = "KeyboardInterrupt occurred while executing sspl test."
logger.error("%s - ERROR: %s - CMD %s" %
(self.name, error, CMD))
# Restore the original path/file & service, then throw exception
# if execution is failed.
service_list.remove(service_name)
Conf.set(SSPL_CONFIG_INDEX, "SERVICEMONITOR>monitored_services",
service_list)
Conf.set(SSPL_CONFIG_INDEX,
"SERVICEMONITOR>threshold_inactive_time",
threshold_inactive_time_original)
Conf.set(SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
sspl_global_config_url)
Conf.set(SSPL_CONFIG_INDEX,
"NODEDATAMSGHANDLER>high_cpu_usage_wait_threshold",
cpu_usage_alert_wait)
Conf.set(SSPL_CONFIG_INDEX,
"NODEDATAMSGHANDLER>high_memory_usage_wait_threshold",
memory_usage_alert_wait)
Conf.save(SSPL_CONFIG_INDEX)
shutil.copyfile(sspl_test_backup, sspl_test_file_path)
if rc != 0:
raise TestException("%s - ERROR: %s - CMD %s" %
(self.name, error, CMD))
print('Restarting the SSPL service..')
CMD = "systemctl restart sspl-ll"
try:
SimpleProcess(CMD).run(realtime_output=True)
except Exception as error:
raise TestException(
"Error occurred while executing sspl tests: %s" % error)
elif self.plan in NOT_IMPLEMENTED_TEST_PLANS:
print("Tests skipped, test plan %s not applicable for SSPL." %
(self.plan))
return 0
else:
# TODO: Convert shell script to python
# from cortx.sspl.sspl_test.run_qa_test import RunQATest
# RunQATest(self.plan).run()
try:
CMD = "%s/run_qa_test.sh --plan %s" % (TEST_DIR, self.plan)
_, error, returncode = SimpleProcess(CMD).run(
realtime_output=True)
except KeyboardInterrupt:
msg = "KeyboardInterrupt occurred while executing sspl test."
logger.error(msg)
raise TestException(msg)
except Exception as error:
msg = "Error occurred while executing self test: %s" % error
logger.error(msg)
raise TestException(msg)
class SSPLPostInstall:
"""Prepare environment for SSPL service."""
name = "sspl_post_install"
def __init__(self):
"""Initialize varibales for post install."""
consts.SSPL_LOG_PATH = "/var/log/%s/sspl/" % consts.PRODUCT_FAMILY
consts.SSPL_BUNDLE_PATH = "/var/%s/sspl/bundle/" % consts.PRODUCT_FAMILY
self.state_file = "%s/state.txt" % consts.DATA_PATH
def validate(self):
"""Check below requirements are met in setup.
1. Check if given product is supported by SSPL
2. Check if given setup is supported by SSPL
3. Check if required pre-requisites softwares are installed.
4. Validate BMC connectivity
5. Validate storage controller connectivity
"""
machine_id = Utility.get_machine_id()
# Validate input/provisioner configs
self.product = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"cortx>release>product")
self.setup = Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"cortx>release>setup")
node_type = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX, "server_node>%s>type" % machine_id)
if node_type.lower() not in ["vm", "virtual"]:
bmc_ip = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>bmc>ip" % machine_id)
enclosure_id = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX,
"server_node>%s>storage>enclosure_id" % machine_id)
Utility.get_config_value(consts.PRVSNR_CONFIG_INDEX,
"storage_enclosure>%s>type" % enclosure_id)
primary_ip = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX,
"storage_enclosure>%s>controller>primary>ip" % enclosure_id)
secondary_ip = Utility.get_config_value(
consts.PRVSNR_CONFIG_INDEX,
"storage_enclosure>%s>controller>secondary>ip" % enclosure_id)
# Validate product support
if self.product not in consts.enabled_products:
msg = "Product '%s' is not in sspl supported product list: %s" % (
self.product, consts.enabled_products)
logger.error(msg)
raise SetupError(errno.EINVAL, msg)
# Validate setup support
if self.setup not in consts.setups:
msg = "Setup '%s' is not in sspl supported setup list: %s" % (
self.setup, consts.setups)
logger.error(msg)
raise SetupError(errno.EINVAL, msg)
# Validate required pip3s and rpms are installed
self.validate_dependencies(self.setup)
# Validate BMC & Storage controller IP reachability
if node_type.lower() not in ["vm", "virtual"]:
# cluster_id required for decrypting the secret is only available from
# the prepare stage. However accessibility validation will be done in
# prepare stage. So at this time, validating ip reachability is fine.
max_retry = 3
for i in range(max_retry):
try:
NetworkV().validate("connectivity",
[bmc_ip, primary_ip, secondary_ip])
break
except VError:
if i == (max_retry - 1):
raise
time.sleep(1)
@staticmethod
def validate_dependencies(setup):
"""Validate pre-requisites software packages."""
pip3_3ps_packages_main = {
"cryptography": "2.8",
"jsonschema": "3.2.0",
"pika": "1.1.0",
"pyinotify": "0.9.6",
"python-daemon": "2.2.4",
"requests": "2.25.1",
"zope.component": "4.6.2",
"zope.event": "4.5.0",
"zope.interface": "5.2.0"
}
rpm_3ps_packages = {
"hdparm": "9.43",
"ipmitool": "1.8.18",
"lshw": "B.02.18",
"python3": "3.6.8",
"python36-dbus": "1.2.4",
"python36-gobject": "3.22.0",
"python36-paramiko": "2.1.1",
"python36-psutil": "5.6.7",
"shadow-utils": "4.6",
"smartmontools": "7.0",
"systemd-python36": "1.0.0",
"udisks2": "2.8.4"
}
ssu_dependency_rpms = [
"sg3_utils", "gemhpi", "pull_sea_logs", "python-hpi",
"zabbix-agent-lib", "zabbix-api-gescheit", "zabbix-xrtx-lib",
"python-openhpi-baselib", "zabbix-collector"
]
ssu_required_process = ["openhpid", "dcs-collectord"]
vm_dependency_rpms = []
pkg_validator = PkgV()
pkg_validator.validate_pip3_pkgs(host=socket.getfqdn(),
pkgs=pip3_3ps_packages_main,
skip_version_check=False)
pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(),
pkgs=rpm_3ps_packages,
skip_version_check=False)
# Check for sspl required processes and misc dependencies if
# setup/role is other than cortx
if setup == "ssu":
pkg_validator.validate("rpms", ssu_dependency_rpms)
ServiceV().validate("isrunning", ssu_required_process)
elif setup == "vm" or setup == "gw" or setup == "cmu":
# No dependency currently. Keeping this section as it
# may be needed in future.
pkg_validator.validate("rpms", vm_dependency_rpms)
# No processes to check in VM environment
def process(self):
"""Create SSPL user and required config files."""
# dbus module import is implicit in cortx utils. Keeping this
# after dependency validation will enrich the use of
# validate_dependencies() method.
from cortx.utils.service import DbusServiceHandler
self.dbus_service = DbusServiceHandler()
# Create and load sspl config
self.create_sspl_conf()
Conf.load(consts.SSPL_CONFIG_INDEX, consts.sspl_config_path)
# Update sspl.conf with provisioner supplied input config copy
Conf.set(consts.SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
consts.global_config_path)
Conf.save(consts.SSPL_CONFIG_INDEX)
self.create_user()
self.create_directories_and_ownership()
self.install_sspl_service_files()
self.enable_sspl_service()
def create_sspl_conf(self):
"""Install product specific sspl config."""
# Copy and load product specific sspl config
if not os.path.exists(consts.file_store_config_path):
shutil.copyfile(
"%s/conf/sspl.conf.%s.yaml" %
(consts.SSPL_BASE_DIR, self.product),
consts.file_store_config_path)
def validate_user_existence(self):
max_retry = 3
for i in range(max_retry):
sspl_uid = Utility.get_uid(consts.USER)
if sspl_uid != -1:
break
else:
if i == (max_retry - 1):
msg = "No user found with name : %s" % (consts.USER)
logger.error(msg)
raise SetupError(errno.EINVAL, msg)
time.sleep(1)
def validate_group_existence(self):
max_retry = 3
for i in range(max_retry):
sspl_gid = Utility.get_gid(consts.USER)
if sspl_gid != -1:
break
else:
if i == (max_retry - 1):
msg = "No group found with name : %s" % (consts.USER)
logger.error(msg)
raise SetupError(errno.EINVAL, msg)
time.sleep(1)
def create_user(self):
"""Add sspl-ll user and validate user creation."""
command = f"/usr/sbin/useradd -r {consts.USER} -s /sbin/nologin \
-c 'User account to run the {consts.USER} service'"
_, error, rc = SimpleProcess(command).run()
# rc 9 means user already exists
if rc != 0 and rc != 9:
logger.error(f"Failed to create sspl user. ERROR: {error}")
raise SetupError(rc, error, command)
self.validate_user_existence()
self.validate_group_existence()
# Add sspl-ll user to required groups and sudoers file etc.
sspl_reinit = "%s/low-level/framework/sspl_reinit" % consts.SSPL_BASE_DIR
_, error, rc = SimpleProcess(sspl_reinit).run()
if rc:
msg = "%s failed for with error : %e" % (sspl_reinit, error)
logger.error(msg)
raise SetupError(rc, msg)
def create_directories_and_ownership(self):
"""Create ras persistent cache directory and state file.
Assign ownership recursively on the configured directory.
The created state file will be used later by SSPL resourse agent(HA).
"""
# Extract the data path
sspldp = Utility.get_config_value(consts.SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>data_path")
if not sspldp:
raise SetupError(errno.EINVAL, "Data path not set in sspl.conf")
sspl_uid = Utility.get_uid(consts.USER)
sspl_gid = Utility.get_gid(consts.USER)
# Create sspl data directory if not exists
os.makedirs(sspldp, exist_ok=True)
# Create state file under sspl data directory
if not os.path.exists(self.state_file):
file = open(self.state_file, "w")
file.close()
# Create SSPL log and bundle directories
os.makedirs(consts.SSPL_LOG_PATH, exist_ok=True)
os.makedirs(consts.SSPL_BUNDLE_PATH, exist_ok=True)
# set ownership for SSPL dirs
list_root_dir = [consts.SSPL_CONFIGURED_DIR, consts.SSPL_LOG_PATH]
Utility.set_ownership_recursively(list_root_dir, sspl_uid, sspl_gid)
# Create /tmp/dcs/hpi if required. Not required for '<product>' role
if self.setup != "cortx":
os.makedirs(consts.HPI_PATH, mode=0o777, exist_ok=True)
zabbix_uid = Utility.get_uid("zabbix")
if zabbix_uid != -1:
os.chown(consts.HPI_PATH, zabbix_uid, -1)
# Create mdadm.conf to set ACL on it.
with open(consts.MDADM_PATH, 'a'):
os.utime(consts.MDADM_PATH)
os.chmod(consts.MDADM_PATH, mode=0o666)
os.chown(consts.MDADM_PATH, sspl_uid, -1)
def install_sspl_service_files(self):
"""Copy service file to systemd location based on product."""
# Create soft link for SINGLE product name service to existing LDR_R1, LR2 service
# Instead of keeping separate service file for SINGLE product with same content.
currentProduct = "%s/conf/sspl-ll.service.%s" % (consts.SSPL_BASE_DIR,
self.product)
if (self.product == "SINGLE" and not os.path.exists(currentProduct)) or \
(self.product == "DUAL" and not os.path.exists(currentProduct)):
os.symlink(
"%s/conf/sspl-ll.service.%s" %
(consts.SSPL_BASE_DIR, self.product), currentProduct)
if self.product == "CLUSTER" and not os.path.exists(currentProduct):
os.symlink("%s/conf/sspl-ll.service.LR2" % (consts.SSPL_BASE_DIR),
currentProduct)
shutil.copyfile(currentProduct, "/etc/systemd/system/sspl-ll.service")
def enable_sspl_service(self):
"""Enable sspl-ll service."""
self.dbus_service.enable("sspl-ll.service")
daemon_reload_cmd = "systemctl daemon-reload"
output, error, rc = SimpleProcess(daemon_reload_cmd).run()
if rc != 0:
logger.error(f"Failed in enable sspl service. ERROR: {error}")
raise SetupError(rc, error, daemon_reload_cmd)