def validate_passwordless_ssh(self, user, nodes):
"""Check passwordless ssh."""
for node in nodes:
cmd = ("ssh -o PreferredAuthentications=publickey "
f"-o StrictHostKeyChecking=no {user}@{node} /bin/true")
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Passwordless ssh not configured for {node}."
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.ECONNREFUSED, res)
def __search_pkg(self, cmd):
# print(f"Running {cmd}")
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
if retcode != 0:
raise VError(errno.EINVAL,
"cmd: %s failed with error code: %d" % (cmd, retcode))
if stderr:
raise VError(errno.EINVAL,
"cmd: %s failed with stderr: %s" % (cmd, stderr))
# To calm down codacy.
return stdout.decode("utf-8")
def validate_network_drivers(self, driver, nodes):
"""Check if drivers (eg: Mellanox OFED) are proper."""
for node in nodes:
cmd = f"ssh {node} rpm -qa | grep {driver}"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Given Driver '{driver}' Is Not Installed on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
def test_011_verify_SB_generate_after_rsyslog_service_stop(self):
"""Validate SB generate while rsyslog service is down."""
cmd = "systemctl stop rsyslog"
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
self.assertEqual(rc, 0)
bundle_obj = SupportBundle.generate(
comment=TestSupportBundle.sb_description, \
components=['utils'])
time.sleep(60)
tar_file_name = f"{bundle_obj.bundle_id}_"\
f"{TestSupportBundle.node_name}.tar.gz"
sb_file_path = f"{bundle_obj.bundle_path}/{bundle_obj.bundle_id}/"\
f"{TestSupportBundle.node_name}/{tar_file_name}"
self.assertEqual(os.path.exists(sb_file_path), True)
cmd = "systemctl start rsyslog"
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
self.assertEqual(rc, 0)
time.sleep(5)
ServiceV().validate('isrunning', ['rsyslog'])
def validate_hba(self, provider, nodes):
"""Check HBA presence and ports."""
for node in nodes:
if provider.lower() == "lsi":
cmd = f"ssh {node} lspci -nn | grep 'SCSI'"
# TO DO: Use paramiko for ssh.
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (
"Host Bus Adapters (HBA) for "
f"SAS channels not detected on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
ports_cmd = f"ssh {node} ls /sys/class/scsi_host/ | wc -l"
ports_cmd_proc = SimpleProcess(ports_cmd)
ports_result = ports_cmd_proc.run()
if ports_result[1] or ports_result[2]:
res = (
"Host Bus Adapters (HBA) for "
f"SAS channels not detected on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {ports_result[0]}. {ports_result[1]}"
)
raise VError(errno.EINVAL, res)
res = ports_result[0].decode('utf-8').strip()
if int(res) == 0:
res = (
"Host Bus Adapters (HBA) for "
f"SAS channels not detected on {node}. "
"For storage connectivity over SAS channels "
"to JBOD/RBOD there is expectation for a PCIe HBA card "
"to be present. Please check HW, if this system "
"expects a connection to either JBOD or RBOD.")
raise VError(errno.EINVAL, res)
def test_009_cli_verify_SB_generate_after_rsyslog_service_stop(self):
"""Validate SB generate while rsyslog service is down."""
cmd = "systemctl stop rsyslog"
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
self.assertEqual(rc, 0)
stdout, stderr, rc = SB_generate_CLI(configs['SB_single_comp'])
self.assertIsInstance(stdout, bytes)
self.assertEqual(stderr, b'')
self.assertEqual(rc, 0)
time.sleep(15)
bundle_id = stdout.decode('utf-8').split('|')[1]
bundle_path = stdout.decode('utf-8').split('->')[1]
tar_file_name = f"{bundle_id.strip()}_{TestSupportBundleCli.node_name}.tar.gz"
sb_file_path = f"{(bundle_path.split('.')[0]).strip()}/"\
f"{bundle_id.strip()}/{TestSupportBundleCli.node_name}/{tar_file_name}"
self.assertEqual(os.path.exists(sb_file_path), True)
cmd = "systemctl start rsyslog"
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
self.assertEqual(rc, 0)
ServiceV().validate('isrunning', ['rsyslog'])
def test_011_cli_verify_SB_generate_after_cluster_stop(self):
"""Validate SB generate while cluster is down."""
cmd = "pcs cluster stop --all"
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
self.assertEqual(rc, 0)
stdout, stderr, rc = SB_generate_CLI(configs['SB_single_comp'])
self.assertIsInstance(stdout, bytes)
self.assertEqual(stderr, b'')
self.assertEqual(rc, 0)
time.sleep(10)
bundle_id = stdout.decode('utf-8').split('|')[1]
bundle_path = stdout.decode('utf-8').split('->')[1]
tar_file_name = f"{bundle_id.strip()}_{TestSupportBundleCli.node_name}.tar.gz"
sb_file_path = f"{(bundle_path.split('.')[0]).strip()}/"\
f"{bundle_id.strip()}/{TestSupportBundleCli.node_name}/{tar_file_name}"
self.assertEqual(os.path.exists(sb_file_path), True)
cmd = "pcs cluster start --all"
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
self.assertEqual(rc, 0)
time.sleep(5)
def test_conf_cli_properties_wrong_format_kv(self):
"""
Test by reading invalid k#V format key value and validate
the result error message
"""
with open(r'/tmp/example_invalid.properties', 'w+') as file:
file.write("key1#val1")
cmd = "conf properties:///tmp/example_invalid.properties get name"
cmd_proc = SimpleProcess(cmd)
try:
result_data = cmd_proc.run()
except Exception:
self.assertEqual(result_data[2], '22')
def start_services(self, s3services_list):
"""Start services specified as parameter."""
for service_name in s3services_list:
cmd = ['/bin/systemctl', 'is-active', f'{service_name}']
handler = SimpleProcess(cmd)
self.logger.info(f"Check {service_name} service is active or not ")
res_op, res_err, res_rc = handler.run()
if res_rc != 0:
self.logger.info(f"Service {service_name} is not active")
try:
# if service name not found in the ha_service_map then use systemctl
service_name = self.ha_service_map[service_name]
cmd = ['cortx', 'start', f'{service_name}']
except KeyError:
cmd = ['/bin/systemctl', 'start', f'{service_name}']
self.logger.info(f"Command: {cmd}")
handler = SimpleProcess(cmd)
res_op, res_err, res_rc = handler.run()
if res_rc != 0:
raise Exception(f"{cmd} failed with err: {res_err}, out: {res_op}, ret: {res_rc}")
else:
self.logger.info(f"Service {service_name} is already active")
def _provision_components(cortx_conf: MappedConf, interfaces: Enum,
apply_phase: str):
"""Invoke Mini Provisioners of cluster components."""
node_id, node_name = CortxProvisioner._get_node_info(cortx_conf)
num_components = int(cortx_conf.get(f'node>{node_id}>num_components'))
for interface in interfaces:
for comp_idx in range(0, num_components):
key_prefix = f'node>{node_id}>components[{comp_idx}]'
component_name = cortx_conf.get(f'{key_prefix}>name')
# Get services.
service_idx = 0
services = []
while (cortx_conf.get(f'{key_prefix}>services[{service_idx}]')
is not None):
services.append(
cortx_conf.get(
f'{key_prefix}>services[{service_idx}]'))
service_idx = service_idx + 1
service = 'all' if service_idx == 0 else ','.join(services)
if apply_phase == ProvisionerStages.UPGRADE.value:
version = cortx_conf.get(f'{key_prefix}>version')
# Skip update for component if it is already updated.
is_updated = CortxProvisioner._is_component_updated(
component_name, version)
if is_updated is True:
Log.info(
f'{component_name} is already updated with {version} version.'
)
continue
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase,
ProvisionerStatus.PROGRESS.value)
cmd = (
f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}"
f" --config {cortx_conf._conf_url} --services {service}")
Log.info(f"{cmd}")
cmd_proc = SimpleProcess(cmd)
_, err, rc = cmd_proc.run()
if rc != 0:
CortxProvisioner._update_provisioning_status(
cortx_conf, node_id, apply_phase,
ProvisionerStatus.ERROR.value)
raise CortxProvisionerError(rc,
"%s phase of %s, failed. %s",
interface.value,
component_name, err)
# Update version for each component if Provisioning successful.
component_version = CortxProvisioner.cortx_release.get_component_version(
component_name)
cortx_conf.set(f'{key_prefix}>version', component_version)
def validate_hca(self, provider, nodes):
"""Check if HCA presence and ports"""
for node in nodes:
if provider.lower() in self.hca_checks:
cmd = f"ssh {node} lspci -nn | grep '{provider.capitalize()}'"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (
f"{provider.capitalize()} Host Channel Adapters card "
f"(possibly with Infiniband capability) is not detected on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD '{cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
ports_cmd = f"{cmd} | wc -l"
ports_cmd_proc = SimpleProcess(ports_cmd)
ports_run_result = ports_cmd_proc.run()
if ports_run_result[1] or ports_run_result[2]:
res = (
f"{provider.capitalize()} Host Channel Adapters ports were not "
f"detected by command {cmd}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {ports_run_result[0]}. {ports_run_result[1]}"
)
raise VError(errno.EINVAL, res)
res = (ports_run_result[0].decode('utf-8').strip())
if int(res) == 0:
res = (
f"{provider.capitalize()} Host Channel Adapters ports were not "
f"detected by command {cmd}. "
"For high-speed data network, it is expected to "
"have at least one port present over some PCIe slot. ")
raise VError(errno.EINVAL, res)
def _add_more_log(self):
"""
Collect Extra Log
"""
cmd_list = [const.PCS_STATUS, const.PCS_FAILCOUNT_STATUS]
with open(const.HA_CMDS_OUTPUT, "w") as file:
for cmd in cmd_list:
_err = ""
_proc = SimpleProcess(cmd)
_output, _err, _rc = _proc.run(universal_newlines=True)
file.write("\n\n\n********************************\n")
file.write(f"Command: {cmd}\nReturn Code: {_rc}\nOutput:\n")
file.write(f"{_output}")
file.write(f"Error if any: {_err}\n\n\n")
def reload_services(self, s3services_list):
"""Reload services specified as parameter."""
for service_name in s3services_list:
try:
# if service name not found in the ha_service_map then use systemctl
service_name = self.ha_service_map[service_name]
cmd = ['cortx', 'reload', f'{service_name}']
except KeyError:
cmd = ['/bin/systemctl', 'reload', f'{service_name}']
handler = SimpleProcess(cmd)
self.logger.info(f"reloading {service_name}")
res_op, res_err, res_rc = handler.run()
if res_rc != 0:
raise Exception(f"{cmd} failed with err: {res_err}, out: {res_op}, ret: {res_rc}")
def configure_openldap(self):
"""Install and Configure Openldap over Non-SSL."""
# 1. Install and Configure Openldap over Non-SSL.
# 2. Enable slapd logging in rsyslog config
# 3. Set openldap-replication
# 4. Check number of nodes in the cluster
self.logger.info('Open ldap configuration started')
cmd = [
'/opt/seagate/cortx/s3/install/ldap/setup_ldap.sh',
'--ldapadminpasswd', f'{self.ldap_passwd}', '--rootdnpasswd',
f'{self.rootdn_passwd}', '--forceclean', '--skipssl'
]
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
self.logger.info(f'output of setup_ldap.sh: {stdout}')
if retcode != 0:
self.logger.error(f'error of setup_ldap.sh: {stderr}')
raise S3PROVError(
f"{cmd} failed with err: {stderr}, out: {stdout}, ret: {retcode}"
)
else:
self.logger.warning(f'warning of setup_ldap.sh: {stderr}')
if os.path.isfile(
"/opt/seagate/cortx/s3/install/ldap/rsyslog.d/slapdlog.conf"):
try:
os.makedirs("/etc/rsyslog.d")
except OSError as e:
if e.errno != errno.EEXIST:
raise S3PROVError(
f"mkdir /etc/rsyslog.d failed with errno: {e.errno}, exception: {e}"
)
shutil.copy(
'/opt/seagate/cortx/s3/install/ldap/rsyslog.d/slapdlog.conf',
'/etc/rsyslog.d/slapdlog.conf')
# restart rsyslog service
try:
self.logger.info("Restarting rsyslog service...")
service_list = ["rsyslog"]
self.restart_services(service_list)
except Exception as e:
self.logger.error(f'Failed to restart rsyslog service, error: {e}')
raise e
self.logger.info("Restarted rsyslog service...")
# set openldap-replication
self.configure_openldap_replication()
self.logger.info('Open ldap configuration completed')
def post_install():
""" Performs post install operations """
# Check python packages and install if something is missing
cmd = "pip3 freeze"
cmd_proc = SimpleProcess(cmd)
stdout, stderr, retcode = cmd_proc.run()
result = stdout.decode("utf-8") if retcode == 0 else stderr.decode("utf-8")
with open('/opt/seagate/cortx/utils/conf/requirements.txt') as f:
pkgs = f.readlines()
# pkgs will have \n in every string. Need to remove that
for package in enumerate(pkgs):
if result.find(package[1][:-1]) == -1:
raise SetupError(errno.EINVAL, "Required python package %s is missing" % package[1][:-1])
return 0
def __init__(self):
from cortx.utils.conf_store import Conf
Conf.load("index", "json:///etc/cortx/message_bus.conf")
self._server = Conf.get("index", 'message_broker>cluster[0]>server')
# Create a test topic
cmd = "/opt/kafka/bin/kafka-topics.sh --create" + \
" --topic mytest --bootstrap-server " + self._server + ":9092"
try:
cmd_proc = SimpleProcess(cmd)
res_op, res_err, res_rc = cmd_proc.run()
if res_rc != 0:
raise SetupError(errno.EINVAL, "Unable to test the config")
except Exception as e:
raise SetupError(errno.EINVAL, \
"Unable to test the config, %s", e)
def validate_bmc_stonith_config(self, node, bmc_ip, bmc_user, bmc_passwd):
""" Validations for BMC STONITH Configuration
"""
cmd = f"ssh {node} fence_ipmilan -P -a {bmc_ip} -o status -l {bmc_user} -p {bmc_passwd}"
cmd_proc = SimpleProcess(cmd)
result = list(cmd_proc.run())
if result[1] or result[2]:
msg = f"Failed to check BMC STONITH Config. Command: '{cmd}', Return Code: '{result[2]}'."
for i in range(2):
if result[i]:
if not isinstance(result[i], str):
result[i] = result[i].decode("utf-8")
res = result[i].replace('\r', '').replace('\n', '')
msg += f' {res}.'
raise VError(errno.EINVAL, msg)
def shutdown_services(self, s3services_list):
"""Stop services."""
for service_name in s3services_list:
try:
# if service name not found in the ha_service_map then use systemctl
service_name = self.ha_service_map[service_name]
cmd = ['cortx', 'stop', f'{service_name}']
except KeyError:
cmd = ['/bin/systemctl', 'stop', f'{service_name}']
handler = SimpleProcess(cmd)
self.logger.info(f"shutting down {service_name}")
res_op, res_err, res_rc = handler.run()
if res_rc != 0:
raise Exception(
f"{cmd} failed with err: {res_err}, out: {res_op}, ret: {res_rc}"
)
def validate_salt_minion_connectivity(self, nodes):
"""Check salt minion connectivity."""
if os.environ.get('USER') != 'root':
res = ("validate_salt_minion: "
"This interface requires root privileges.")
raise VError(errno.EACCES, res)
for node in nodes:
cmd = f"salt -t 5 {node} test.ping"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Salt minion {node} unreachable."
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.ECONNREFUSED, res)
def get_unread_count(self, consumer_group: str):
"""
Gets the count of unread messages from the Kafka message server
Parameters:
consumer_group A String that represents Consumer Group ID.
"""
table = []
try:
cmd = "/opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server "\
+ self._servers + " --describe --group " + consumer_group
cmd_proc = SimpleProcess(cmd)
res_op, res_err, res_rc = cmd_proc.run()
if res_rc != 0:
raise MessageBusError(
errno.EINVAL, "Unable to get the message \
count. %s", res_err)
decoded_string = res_op.decode('utf-8')
if decoded_string == '':
raise MessageBusError(
errno.EINVAL, "No active consumers in \
the consumer group, %s", consumer_group)
elif 'Error' in decoded_string:
raise MessageBusError(
errno.EINVAL, "Unable to get the message \
count. %s", decoded_string)
else:
split_rows = decoded_string.split('\n')
rows = [row.split(' ') for row in split_rows if row != '']
for each_row in rows:
new_row = [item for item in each_row if item != '']
table.append(new_row)
index = table[0].index('LAG')
unread_count = [int(lag[index]) for lag in table if lag[index] \
!= 'LAG' and lag[index] != '-']
if len(unread_count) == 0:
raise MessageBusError(
errno.EINVAL, "No active consumers \
in the consumer group, %s", consumer_group)
return sum(unread_count)
except Exception as e:
raise MessageBusError(
errno.EINVAL, "Unable to get the message \
count. %s", e)
def __execute_cmd(self, cmd):
"""
Execute command using SSHChannel or SimpleProcees and returns result.
Uses SimpleProcess to execute the command on passwordless ssh configured
host. Otherwise, uses SSHChannel to execute command.
"""
if self.ssh:
retcode, result = self.ssh.execute(cmd)
else:
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
result = stdout.decode("utf-8") if retcode == 0 else stderr.decode(
"utf-8")
if retcode != 0:
raise VError(errno.EINVAL,
"Command failure. cmd: %s stderr: %s" % (cmd, result))
return result
def validate_services(self, host, services):
"""Check if services are running."""
for service in services:
if host != None:
cmd = f"ssh {host} systemctl status {service}"
else:
cmd = f"systemctl status {service}"
handler = SimpleProcess(cmd)
_, stderr, retcode = handler.run()
if retcode != 0:
raise VError(errno.EINVAL,
"cmd: %s failed with error code: %d"
%(cmd, retcode))
if stderr:
raise VError(errno.EINVAL,
"cmd: %s failed with stderr: %s"
%(cmd, stderr))
def post_install():
""" Performs post install operations """
# Do the python packages installation
cmd = "/bin/pip3 install -r " \
+ "/opt/seagate/cortx/utils/conf/requirements.txt"
try:
cmd_proc = SimpleProcess(cmd)
res_op, res_err, res_rc = cmd_proc.run()
if res_rc != 0:
raise SetupError(errno.ENOENT, \
"Python Package installation failed, rc=%d", \
res_rc)
except Exception as e:
raise SetupError(errno.ENOENT, \
"Python Package installation failed, %s", e)
return res_rc
def __ping_bmc(self, node):
""" Ping BMC IP
"""
ip = self.__get_bmc_ip(node)
cmd = f"ssh {node} ping -c 1 -W 1 {ip}"
cmd_proc = SimpleProcess(cmd)
result = list(cmd_proc.run())
if result[1] or result[2]:
msg = f"Ping failed for IP '{ip}'. Command: '{cmd}', Return Code: '{result[2]}'."
for i in range(2):
if result[i]:
if not isinstance(result[i], str):
result[i] = result[i].decode("utf-8")
res = result[i].replace('\r', '').replace('\n', '')
msg += f' {res}.'
raise VError(errno.ECONNREFUSED, msg)
def process(self):
"""Main processing function."""
sys.stdout.write(f"Processing {self.name} {self.url}\n")
try:
cmd = [
'/opt/seagate/cortx/s3/scripts/s3-sanity-test.sh', '-p',
f'{self.ldap_passwd}'
]
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
if retcode != 0:
raise Exception(
f"{cmd} failed with err: {stderr}, out: {stdout}, ret: {retcode}"
)
sys.stdout.write(f"{cmd}:{stdout}:{stderr}:{retcode}\n")
except Exception as e:
raise Exception(f"{self}: {cmd} exception: {e}")
def create_auth_jks_password(self):
"""Create random password for auth jks keystore."""
cmd = [
'sh', '/opt/seagate/cortx/auth/scripts/create_auth_jks_password.sh'
]
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
self.logger.info(f'output of create_auth_jks_password.sh: {stdout}')
if retcode != 0:
self.logger.error(
f'error of create_auth_jks_password.sh: {stderr}')
raise S3PROVError(
f"{cmd} failed with err: {stderr}, out: {stdout}, ret: {retcode}"
)
else:
self.logger.warning(
f'warning of create_auth_jks_password.sh: {stderr}')
self.logger.info(' Successfully set auth JKS keystore password.')
def validate_ip_connectivity(self, ips):
"""Check if IPs are reachable."""
unreachable_ips = []
for ip in ips:
if ip.count(".") == 3 and all(self._is_valid_ipv4_part(ip_part)
for ip_part in ip.split(".")):
cmd = f"ping -c 1 -W 1 {ip}"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[2]:
unreachable_ips.append(ip)
else:
raise VError(errno.EINVAL, f"Invalid ip {ip}.")
if len(unreachable_ips) != 0:
raise VError(
errno.ECONNREFUSED, "Ping failed for IP(s). %s" % unreachable_ips)
def get_log_size(self, message_type: str):
""" Gets size of log across all the partitions """
total_size = 0
cmd = "/opt/kafka/bin/kafka-log-dirs.sh --describe --bootstrap-server "\
+ self._servers + " --topic-list " + message_type
try:
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
decoded_string = run_result[0].decode('utf-8')
output_json = json.loads(re.search(r'({.+})', decoded_string).\
group(0))
for brokers in output_json['brokers']:
partition = brokers['logDirs'][0]['partitions']
for each_partition in partition:
total_size += each_partition['size']
return total_size
except Exception as e:
raise MessageBusError(errno.EINVAL, "Unable to fetch log size for \
message type %s. %s" , message_type, e)
def reset():
"""Remove/Delete all the data/logs that was created by user/testing."""
import time
_purge_retry = 20
try:
from cortx.utils.message_bus import MessageBusAdmin
from cortx.utils.message_bus.message_bus_client import MessageProducer
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
for retry_count in range(1, (_purge_retry + 2)):
if retry_count > _purge_retry:
Log.error(f"MessageBusError: {errors.ERR_OP_FAILED} " \
f" Unable to delete messages for message type" \
f" {message_type} after {retry_count} retries")
raise MessageBusError(errors.ERR_OP_FAILED,\
"Unable to delete messages for message type" + \
"%s after %d retries", message_type, \
retry_count)
rc = producer.delete()
if rc == 0:
break
time.sleep(2 * retry_count)
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = '/var/log/cortx/utils/'
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0
def validate_controller_accessibility(self, ip, username, password):
"""Check contoller console is accessible to node."""
# Check if ssh connection is successful
try:
session = SSHChannel(host=ip, username=username, password=password)
session.disconnect()
except:
err = traceback.format_exc()
raise VError(
errno.EINVAL,
"Failed to create ssh connection to %s, Error: %s" % (ip, err))
# ping controller IP
cmd = "ping -c 1 -W 1 %s" % ip
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
msg = "Ping failed for IP '%s'. Command: '%s', Return Code: '%s'." % (
ip, cmd, rc)
msg += stderr.decode("utf-8").replace('\r', '').replace('\n', '')
raise VError(errno.EINVAL, msg)
