def test_service_inactive_alert(args):
check_sspl_ll_is_running()
check_service_is_running(service_name)
# Simulate Fault alert by stopping the service.
DbusServiceHandler().stop(service_name)
time.sleep(WAIT_TIME)
sensor_response = read_ingress_queue()
assert_on_mismatch(sensor_response, "fault")
# Simulate Fault resolved alert.
DbusServiceHandler().start(service_name)
time.sleep(5)
sensor_response = read_ingress_queue()
assert_on_mismatch(sensor_response, "fault_resolved")
def __init__(self):
super(ServiceMsgHandler, self).__init__(self.MODULE_NAME,
self.PRIORITY)
self._service_actuator = None
self._query_utility = None
self._dbus_service = DbusServiceHandler()
# Flag to indicate suspension of module
self._suspended = False
def test_service_restart_case(args):
check_sspl_ll_is_running()
check_service_is_running(service_name)
DbusServiceHandler().restart(service_name)
time.sleep(WAIT_TIME)
sensor_response = read_ingress_queue()
check_service_is_running(service_name)
if sensor_response:
print(sensor_response)
assert (sensor_response['info']['alert_type'] != "fault")
def install_files(self, PRODUCT):
"""Configure required log files."""
dbus_service = DbusServiceHandler()
# Copy rsyslog configuration
if not os.path.exists(self.RSYSLOG_CONF):
shutil.copyfile(
"%s/low-level/files/%s" % (SSPL_BASE_DIR, self.RSYSLOG_CONF),
self.RSYSLOG_CONF)
if not os.path.exists(self.RSYSLOG_SSPL_CONF):
shutil.copyfile(
"%s/low-level/files/%s" %
(SSPL_BASE_DIR, self.RSYSLOG_SSPL_CONF),
self.RSYSLOG_SSPL_CONF)
# Create soft link for SINGLE product name service to existing LDR_R1, LR2 service
# Instead of keeping separate service file for SINGLE product with same content.
currentProduct = "%s/conf/sspl-ll.service.%s" % (SSPL_BASE_DIR,
PRODUCT)
if (PRODUCT == "SINGLE" and not os.path.exists(currentProduct)) or \
(PRODUCT == "DUAL" and not os.path.exists(currentProduct)):
os.symlink("%s/conf/sspl-ll.service.%s" % (SSPL_BASE_DIR, PRODUCT),
currentProduct)
if PRODUCT == "CLUSTER" and not os.path.exists(currentProduct):
os.symlink("%s/conf/sspl-ll.service.LR2" % (SSPL_BASE_DIR),
currentProduct)
# Copy sspl-ll.service file and enable service
shutil.copyfile(currentProduct, "/etc/systemd/system/sspl-ll.service")
dbus_service.enable('sspl-ll.service')
daemon_reload_cmd = "systemctl daemon-reload"
output, error, returncode = SimpleProcess(daemon_reload_cmd).run()
if returncode != 0:
raise SetupError(returncode, error, daemon_reload_cmd)
# Copy IEC mapping files
os.makedirs("%s/iem/iec_mapping" % (PRODUCT_BASE_DIR), exist_ok=True)
distutils.dir_util.copy_tree(
"%s/low-level/files/iec_mapping/" % (SSPL_BASE_DIR),
"%s/iem/iec_mapping" % (PRODUCT_BASE_DIR))
# Skip this step if sspl is being configured for node replacement scenario as sspl configurations are
# already available in consul on the healthy node
# Running script for fetching the config using salt and feeding values to consul
# Onward LR2, consul will be abstracted out and it won't exit as hard dependeny of SSPL
if PRODUCT == "LDR_R1":
if not os.path.exists(REPLACEMENT_NODE_ENV_VAR_FILE):
config_from_salt = "%s/sspl_fetch_config_from_salt.py %s %s" % (
self._script_dir, self.ENVIRONMENT, PRODUCT)
output, error, returncode = SimpleProcess(
config_from_salt).run()
if returncode != 0:
raise SetupError(returncode, error, config_from_salt)
def __init__(self, args: list):
self.args = args
self.name = "sspl_test"
self.plan = "self"
self.avoid_rmq = False
self.dbus_service = DbusServiceHandler()
if args.config and args.config[0]:
self.sspl_test_gc_url = args.config[0]
else:
self.sspl_test_gc_url = global_config_path
self.sspl_test_gc_copy_file = "/etc/sspl_test_gc_url.yaml"
def __init__(self, args: list):
self.args = args
self.name = "sspl_test"
self.plan = "self_primary"
self.avoid_rmq = False
self.dbus_service = DbusServiceHandler()
# Load global, sspl and test configs
Conf.load(SSPL_CONFIG_INDEX, sspl_config_path)
global_config_url = Conf.get(
SSPL_CONFIG_INDEX, "SYSTEM_INFORMATION>global_config_copy_url")
Conf.load(GLOBAL_CONFIG_INDEX, global_config_url)
Conf.load(SSPL_TEST_CONFIG_INDEX, sspl_test_config_path)
def __init__(self, args: list):
self.args = args
self.name = "sspl_test"
self.plan = "sanity"
self.coverage_enabled = self.args.coverage
self.test_dir = f"/opt/seagate/{PRODUCT_FAMILY}/sspl/sspl_test"
if self.args.v2:
self.test_dir = f"/opt/seagate/{PRODUCT_FAMILY}/sspl/sspl_test/functional_tests/v2"
sspl_constants.sspl_test_file_path = sspl_testv2_file_path
sspl_constants.sspl_test_config_path = sspl_testv2_config_path
self.dbus_service = DbusServiceHandler()
if args.config and args.config[0]:
self.sspl_test_gc_url = args.config[0]
else:
self.sspl_test_gc_url = global_config_path
self.sspl_test_gc_copy_file = "/etc/sspl_test_gc_url.yaml"
def process(self):
# stop sspl service
dbus_service = DbusServiceHandler()
dbus_service.stop('sspl-ll.service')
# Remove sspl_conf
self.del_file(file_store_config_path)
# Remove sspl-configured file
self.del_file(SSPL_CONFIGURED)
# Remove sspl data
shutil.rmtree(DATA_PATH, ignore_errors=True)
# Remove sspl-ll user if preset
CMD = "id -u sspl-ll"
output, error, returncode = SimpleProcess(CMD).run()
if returncode != 0:
raise SetupError(returncode, "ERROR: %s - CMD %s", error, CMD)
else:
self.user_present = True
if self.user_present:
CMD = "/usr/sbin/userdel sspl-ll"
output, error, returncode = SimpleProcess(CMD).run()
if returncode != 0:
raise SetupError(returncode, "ERROR: %s - CMD %s", error, CMD)
# Remove log directories
shutil.rmtree(f"/var/log/{PRODUCT_FAMILY}/sspl", ignore_errors=True)
shutil.rmtree(f"/var/log/{PRODUCT_FAMILY}/iem", ignore_errors=True)
# Remove rsyslog config files
self.del_file("/etc/rsyslog.d/0-iemfwd.conf")
self.del_file("/etc/rsyslog.d/1-ssplfwd.conf")
# Remove logrotate config files
self.del_file("/etc/logrotate.d/iem_messages")
self.del_file("/etc/logrotate.d/sspl_logs")
# Remove SSPL configuration files
shutil.rmtree("/etc/sspl-ll", ignore_errors=True)
self.del_file("/etc/sspl.conf.bak")
def __init__(self):
super(SystemdService, self).__init__()
# Use d-bus to communicate with systemd
# Described at: http://www.freedesktop.org/wiki/Software/systemd/dbus/
# Obtain an instance of d-bus to communicate with systemd
self._bus = SystemBus()
# Obtain a manager interface to d-bus for communications with systemd
systemd = self._bus.get_object('org.freedesktop.systemd1',
'/org/freedesktop/systemd1')
self._manager = Interface(
systemd, dbus_interface='org.freedesktop.systemd1.Manager')
# Subscribe to signal changes
self._manager.Subscribe()
# create service cls obj.
self._service = DbusServiceHandler()
def process(self):
dbus_service = DbusServiceHandler()
# Stop SSPL service if state is active
service_state = dbus_service.get_state('sspl-ll.service')
if service_state._state == 'active':
logger.warning ("SSPL service should have been stopped,"
f"before {self.name} interface is invoked")
logger.warning("Stopping SSPL service now...")
dbus_service.stop('sspl-ll.service')
# Remove sspl data
shutil.rmtree(DATA_PATH, ignore_errors=True)
# Clear log data from log files and delete 'log.gz' files
FileStore().truncate(f"/var/log/{PRODUCT_FAMILY}/sspl/", '.log')
FileStore().truncate(f"/var/log/{PRODUCT_FAMILY}/iem/", '.log')
FileStore().delete(
f"/var/log/{PRODUCT_FAMILY}/sspl/", '.log.gz')
FileStore().delete(
f"/var/log/{PRODUCT_FAMILY}/iem/", '.log.gz')
def process(self):
dbus_service = DbusServiceHandler()
# Stop SSPL service if state is active
service_state = dbus_service.get_state('sspl-ll.service')
if service_state._state == 'active':
logger.warning ("SSPL service should have been stopped,"
f"before {self.name} interface is invoked")
logger.warning("Stopping SSPL service now...")
dbus_service.stop('sspl-ll.service')
# Remove sspl data
shutil.rmtree(DATA_PATH, ignore_errors=True)
# Remove log files
Reset.del_files_from_dir('.log', f"/var/log/{PRODUCT_FAMILY}/sspl/")
Reset.del_files_from_dir('.log.gz',
f"/var/log/{PRODUCT_FAMILY}/sspl/")
Reset.del_files_from_dir('.log', f"/var/log/{PRODUCT_FAMILY}/iem/")
Reset.del_files_from_dir('.log.gz',
f"/var/log/{PRODUCT_FAMILY}/iem/")
def process(self):
"""Create SSPL user and required config files."""
# dbus module import is implicit in cortx utils. Keeping this
# after dependency validation will enrich the use of
# validate_dependencies() method.
from cortx.utils.service import DbusServiceHandler
self.dbus_service = DbusServiceHandler()
# Create and load sspl config
self.create_sspl_conf()
Conf.load(consts.SSPL_CONFIG_INDEX, consts.sspl_config_path)
# Update sspl.conf with provisioner supplied input config copy
Conf.set(consts.SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>global_config_copy_url",
consts.global_config_path)
Conf.save(consts.SSPL_CONFIG_INDEX)
self.create_user()
self.create_directories_and_ownership()
self.install_sspl_service_files()
self.enable_sspl_service()
def check_service_is_running(service_name):
state = DbusServiceHandler().get_state(service_name).state
assert (state == "active")
return state
def configure_sspl_syslog(self):
"""Configure log file path in rsyslog and update logrotate config file."""
system_files_root = "%s/low-level/files" % consts.SSPL_BASE_DIR
sspl_log_file_path = Utility.get_config_value(consts.SSPL_CONFIG_INDEX,
"SYSTEM_INFORMATION>sspl_log_file_path")
sspl_sb_log_file_path = sspl_log_file_path.replace("/sspl.log","/sspl_support_bundle.log")
iem_log_file_path = Utility.get_config_value(consts.SSPL_CONFIG_INDEX,
"IEMSENSOR>log_file_path")
manifest_log_file_path = sspl_log_file_path.replace("/sspl.log","/manifest.log")
# IEM configuration
os.makedirs("%s/iem/iec_mapping" % consts.PRODUCT_BASE_DIR, exist_ok=True)
distutils.dir_util.copy_tree("%s/iec_mapping/" % system_files_root,
"%s/iem/iec_mapping" % consts.PRODUCT_BASE_DIR)
if not os.path.exists(consts.RSYSLOG_IEM_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_IEM_CONF),
consts.RSYSLOG_IEM_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_IEM_CONF,
'File.*[=,"]', 'File="%s"' % iem_log_file_path)
# SSPL rsys log configuration
if not os.path.exists(consts.RSYSLOG_SSPL_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_SSPL_CONF),
consts.RSYSLOG_SSPL_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_SSPL_CONF, 'File.*[=,"]',
'File="%s"' % sspl_log_file_path)
# Manifest Bundle log configuration
if not os.path.exists(consts.RSYSLOG_MSB_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_MSB_CONF),
consts.RSYSLOG_MSB_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_MSB_CONF, 'File.*[=,"]',
'File="%s"' % manifest_log_file_path)
# Support Bundle log configuration
if not os.path.exists(consts.RSYSLOG_SB_CONF):
shutil.copyfile("%s/%s" % (system_files_root, consts.RSYSLOG_SB_CONF),
consts.RSYSLOG_SB_CONF)
# Update log location as per sspl.conf
Utility.replace_expr(consts.RSYSLOG_SB_CONF, 'File.*[=,"]',
'File="%s"' % sspl_sb_log_file_path)
# Configure logrotate
# Create logrotate dir in case it's not present
os.makedirs(consts.LOGROTATE_DIR, exist_ok=True)
Utility.replace_expr("%s/etc/logrotate.d/iem_messages" % system_files_root,
0, iem_log_file_path)
Utility.replace_expr("%s/etc/logrotate.d/sspl_logs" % system_files_root,
0, sspl_log_file_path)
Utility.replace_expr("%s/etc/logrotate.d/sspl_sb_logs" % system_files_root,
0, sspl_sb_log_file_path)
shutil.copy2("%s/etc/logrotate.d/iem_messages" % system_files_root,
consts.IEM_LOGROTATE_CONF)
shutil.copy2("%s/etc/logrotate.d/sspl_logs" % system_files_root,
consts.SSPL_LOGROTATE_CONF)
shutil.copy2("%s/etc/logrotate.d/manifest_logs" % system_files_root,
consts.MSB_LOGROTATE_CONF)
shutil.copy2("%s/etc/logrotate.d/sspl_sb_logs" % system_files_root,
consts.SB_LOGROTATE_CONF)
# This rsyslog restart will happen after successful updation of rsyslog
# conf file and before sspl starts. If at all this will be removed from
# here, there will be a chance that SSPL intial logs will not be present in
# "/var/log/<product>/sspl/sspl.log" file. So, initial logs needs to be collected from
# "/var/log/messages"
service = DbusServiceHandler()
service.restart('rsyslog.service')
# along with this program. If not, see <https://www.gnu.org/licenses/>. # For any questions about this software or licensing, please email # [email protected] or [email protected]. import os import shutil import time from cortx.utils.process import SimpleProcess from cortx.utils.service import DbusServiceHandler path = os.path.dirname(os.path.abspath(__file__)) service_name = "dummy_service.service" service_file_path = f"{path}/{service_name}" service_executable_code_src = f"{path}/dummy_service.py" service_executable_code_des = "/var/cortx/sspl/test" dbus_service = DbusServiceHandler() def simulate_fault_alert(): """Simulate fault for dummy service by deleting executable service file.""" os.remove(f"{service_executable_code_des}/dummy_service.py") dbus_service.restart(service_name) time.sleep(5) def restore_service_file(): """Simulate fault resolved for dummy service by creating executable service file.""" shutil.copy(service_executable_code_src, service_executable_code_des) # Make service file executable. cmd = f"chmod +x {service_executable_code_des}/dummy_service.py"
def __init__(self):
"""Initialize varibales for post install."""
consts.SSPL_LOG_PATH = "/var/log/%s/sspl/" % consts.PRODUCT_FAMILY
consts.SSPL_BUNDLE_PATH = "/var/%s/sspl/bundle/" % consts.PRODUCT_FAMILY
self.state_file = "%s/state.txt" % consts.DATA_PATH
self.dbus_service = DbusServiceHandler()
def config_sspl(self):
if (os.geteuid() != 0):
raise SetupError(errno.EINVAL,
"Run this command with root privileges!!")
if not os.path.isfile(consts.file_store_config_path):
raise SetupError(errno.EINVAL,
"Missing configuration!! Create and rerun.",
consts.file_store_config_path)
# Put minion id, consul_host and consul_port in conf file
# Onward LDR_R2, salt will be abstracted out and it won't
# exist as a hard dependeny of SSPL
if consts.PRODUCT_NAME == "LDR_R1":
from framework.utils.salt_util import SaltInterface
salt_util = SaltInterface()
salt_util.update_config_file(consts.file_store_config_path)
if os.path.isfile(consts.SSPL_CONFIGURED):
os.remove(consts.SSPL_CONFIGURED)
# Add sspl-ll user to required groups and sudoers file etc.
sspl_reinit = [
f"{consts.SSPL_BASE_DIR}/low-level/framework/sspl_reinit",
self.product
]
_, error, returncode = SimpleProcess(sspl_reinit).run()
if returncode:
raise SetupError(
returncode, "%s/low-level/framework/sspl_reinit failed for"
"product %s with error : %e", consts.SSPL_BASE_DIR,
self.product, error)
os.makedirs(consts.SSPL_CONFIGURED_DIR, exist_ok=True)
with open(consts.SSPL_CONFIGURED, 'a'):
os.utime(consts.SSPL_CONFIGURED)
# SSPL Log file configuration
# SSPL_LOG_FILE_PATH = self.getval_from_ssplconf('sspl_log_file_path')
SSPL_LOG_FILE_PATH = Conf.get(consts.SSPL_CONFIG_INDEX,
'SYSTEM_INFORMATION>sspl_log_file_path')
IEM_LOG_FILE_PATH = Conf.get(consts.SSPL_CONFIG_INDEX,
'IEMSENSOR>log_file_path')
if SSPL_LOG_FILE_PATH:
self.replace_expr(consts.RSYSLOG_SSPL_CONF, 'File.*[=,"]',
'File="%s"' % SSPL_LOG_FILE_PATH)
self.replace_expr(
f"{consts.SSPL_BASE_DIR}/low-level/files/etc/logrotate.d/sspl_logs",
0, SSPL_LOG_FILE_PATH)
# IEM configuration
# Configure log file path in Rsyslog and logrotate configuration file
IEM_LOG_FILE_PATH = Conf.get(consts.SSPL_CONFIG_INDEX,
'IEMSENSOR>log_file_path')
if IEM_LOG_FILE_PATH:
self.replace_expr(consts.RSYSLOG_IEM_CONF, 'File.*[=,"]',
'File="%s"' % IEM_LOG_FILE_PATH)
self.replace_expr(
f'{consts.SSPL_BASE_DIR}/low-level/files/etc/logrotate.d/iem_messages',
0, IEM_LOG_FILE_PATH)
else:
self.replace_expr(
consts.RSYSLOG_IEM_CONF, 'File.*[=,"]',
'File="/var/log/%s/iem/iem_messages"' % consts.PRODUCT_FAMILY)
# Create logrotate dir in case it's not present for dev environment
if not os.path.exists(consts.LOGROTATE_DIR):
os.makedirs(consts.LOGROTATE_DIR)
shutil.copy2(
'%s/low-level/files/etc/logrotate.d/iem_messages' %
consts.SSPL_BASE_DIR, consts.IEM_LOGROTATE_CONF)
shutil.copy2(
'%s/low-level/files/etc/logrotate.d/sspl_logs' %
consts.SSPL_BASE_DIR, consts.SSPL_LOGROTATE_CONF)
# This rsyslog restart will happen after successful updation of rsyslog
# conf file and before sspl starts. If at all this will be removed from
# here, there will be a chance that SSPL intial logs will not be present in
# "/var/log/<product>/sspl/sspl.log" file. So, initial logs needs to be collected from
# "/var/log/messages"
service = DbusServiceHandler()
service.restart('rsyslog.service')
# For node replacement scenario consul will not be running on the new node. But,
# there will be two instance of consul running on healthy node. When new node is configured
# consul will be brought back on it. We are using VIP to connect to consul. So, if consul
# is not running on new node, we dont need to error out.
# If consul is not running, exit
# Onward LDR_R2, consul will be abstracted out and it won't
# exit as hard dependeny of SSPL
if consts.PRODUCT_NAME == 'LDR_R1':
if not os.path.exists(consts.REPLACEMENT_NODE_ENV_VAR_FILE):
ServiceV().validate('isrunning', ['consul'], is_process=True)
# Get the types of server and storage we are currently running on and
# enable/disable sensor groups in the conf file accordingly.
update_sensor_info(consts.SSPL_CONFIG_INDEX)
self.create_message_types()