def test_conf_cli_by_get_from_properties_store(self):
""" Test by retrieving a value from properties store using get api """
cmd = "conf properties:///tmp/example.properties get name"
cmd_proc = SimpleProcess(cmd)
result_data = cmd_proc.run()
self.assertTrue(True if result_data[2] == 0 and result_data[0] ==
b'["Homebridge"]\n' else False, result_data[1])
def test_conf_cli_by_get(self):
""" Test by retrieving a value using get api """
cmd = "conf json:///tmp/file1.json get bridge>name"
cmd_proc = SimpleProcess(cmd)
result_data = cmd_proc.run()
self.assertTrue(True if result_data[2] == 0 and
result_data[0] == b'["Homebridge"]\n' else False, result_data[1])
def test_003generate_all_component(self):
cmd = "support_bundle generate 'sample comment'"
cmd_proc = SimpleProcess(cmd)
stdout, stderr, rc = cmd_proc.run()
self.assertIsInstance(stdout, bytes)
self.assertEqual(stderr, b'')
self.assertEqual(rc, 0)
def test_iem_receive_with_valid_cmdline_arguments(self):
# send data to receive
SimpleProcess(self.send_with_valid_cmdline_args).run()
cmd = self.valid_recv
output, _, rc = SimpleProcess(cmd).run()
self.assertTrue('critical' in output.decode('utf-8'))
self.assertEqual(rc, 0)
async def _exc_components_cmd(commands: List, bundle_id: str, path: str, \
component: str, node_name: str, comment: str, config_url:str,
services:str, binlogs:bool, coredumps:bool, stacktrace:bool,
duration:str, size_limit:str):
"""
Executes the Command for Bundle Generation of Every Component.
commands: Command of the component :type:str
bundle_id: Unique Bundle ID of the generation process. :type:str
path: Path to create the tar by components :type:str
component: Name of Component to be executed :type: str
node_name: Name of Node where the Command is being Executed :type:str
comment: User Comment: type:str
"""
for command in commands:
# SB Framework will not parse additional filters until all the components
# accept filters in their respective support bundle scripts.
cli_cmd = f"{command} -b {bundle_id} -t {path} -c {config_url}"\
f" -s {services} --duration {duration} --size_limit {size_limit}"\
f" --binlogs {binlogs} --coredumps {coredumps} --stacktrace {stacktrace}"
Log.info(f"Executing command -> {cli_cmd}")
cmd_proc = SimpleProcess(cli_cmd)
output, err, return_code = cmd_proc.run()
Log.debug(f"Command Output -> {output} {err}, {return_code}")
if return_code != 0:
Log.error(f"Command Output -> {output} {err}, {return_code}")
else:
Log.debug(f"Command Output -> {output} {err}, {return_code}")
return component, return_code
def get_server_details():
"""Returns a dictionary of server information using ipmitool.
Grep 'FRU device description on ID 0' information from the output
of 'ipmitool fru print'. Server details includes Hostname, Board and
Product information.
"""
fru_info = {
"Host": socket.getfqdn(),
"Board Mfg": None,
"Board Product": None,
"Board Part Number": None,
"Product Name": None,
"Product Part Number": None
}
cmd = "ipmitool fru print"
prefix = "FRU Device Description : Builtin FRU Device (ID 0)"
search_res = ""
res_op, _, res_rc = SimpleProcess(cmd).run()
if isinstance(res_op, bytes):
res_op = res_op.decode("utf-8")
if res_rc == 0:
# Get only 'FRU Device Description : Builtin FRU Device (ID 0)' information
search_res = re.search(
r"((.*%s[\S\n\s]+ID 1\)).*)|(.*[\S\n\s]+)" % prefix, res_op)
if search_res:
search_res = search_res.group()
for key in fru_info.keys():
if key in search_res:
device_desc = re.search(r"%s[\s]+:[\s]+([\w-]+)(.*)" % key, res_op)
if device_desc:
value = device_desc.groups()[0]
fru_info.update({key: value})
return fru_info
def init(self, *args, **kwargs):
"""Perform initialization. Raises exception on error."""
Kafka._validate_kafka_installation()
kafka_servers = args[0]
curr_host_fqdn = gethostname()
curr_host_ip = gethostbyname(curr_host_fqdn)
hosts = [server.split(':')[0] if ':' in server else server \
for server in kafka_servers]
if not (curr_host_fqdn in hosts or curr_host_ip in hosts) :
return 0 # return if current host is not a kafka server
_, err, rc = SimpleProcess("systemctl restart kafka-zookeeper").run()
if rc != 0:
raise KafkaSetupError(rc, "Unable to start Zookeeper", err)
attempt = 0
while attempt <= 100 :
_, _, rc = SimpleProcess(
"echo ruok|nc `hostname` 2181|grep imok").run()
if rc != 0 :
attempt += 1
sleep(1)
continue
break
if attempt > 100 :
raise KafkaSetupError(ERR_OP_FAILED, "Unable to start Zookeeper")
_, err, rc = SimpleProcess("systemctl restart kafka").run()
if rc != 0:
raise KafkaSetupError(rc, "Unable to start Kakfa", err)
return 0
def test_conf_cli_multiple_kv_delim(self):
"""Test by trying to set a value into given key position with multiple kv_delim argument."""
set_cmd = "conf json:///tmp/file1.json set -d : : cluster>id:093d"
set_cmd_proc = SimpleProcess(set_cmd)
result_data = set_cmd_proc.run()
self.assertTrue(
True if result_data[2] == 0 and result_data[0] == b'' else False)
def set_lshw_input_data(self):
"""
KvStoreFactory can not accept a dictionary as direct input and output
It will support only JSON, YAML, TOML, INI, PROPERTIES files. So here
we are fetching the lshw data and adding that to a file for further
execution.
"""
input_file = None
output_file = None
response, err, returncode = SimpleProcess("lshw -json").run()
if returncode:
msg = f"Failed to capture Node support data. Error:{str(err)}"
logger.error(self.log.svc_log(msg))
raise ResourceMapError(errno.EINVAL, msg)
try:
with open(LSHW_FILE, 'w+') as fp:
json.dump(json.loads(response.decode("utf-8")), fp, indent=4)
with open(MANIFEST_OUTPUT_FILE, 'w+') as fp:
json.dump({}, fp, indent=4)
input_file = KvStoreFactory.get_instance(
f'json://{LSHW_FILE}').load()
output_file = KvStoreFactory.get_instance(
f'json://{MANIFEST_OUTPUT_FILE}')
except Exception as e:
msg = "Error in getting {0} file: {1}".format(LSHW_FILE, e)
logger.error(self.log.svc_log(msg))
raise ResourceMapError(errno.EINVAL, msg)
return input_file, output_file
def reset():
"""Remove/Delete all the data/logs that was created by user/testing."""
try:
from cortx.utils.message_bus import MessageBusAdmin
from cortx.utils.message_bus.message_bus_client import MessageProducer
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
for message_type in message_types_list:
producer = MessageProducer(producer_id=message_type, \
message_type=message_type, method='sync')
producer.delete()
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(
errors.ERR_OP_FAILED, "Internal error, can not \
reset Message Bus. %s", e)
# Clear the logs
utils_log_path = '/var/log/cortx/utils/'
if os.path.exists(utils_log_path):
cmd = "find %s -type f -name '*.log' -exec truncate -s 0 {} +" % utils_log_path
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
raise SetupError(errors.ERR_OP_FAILED, \
"Can not reset log files. %s", stderr)
return 0
def run_cmd(self, cmd, check_error=True, secret=None):
"""
Run command and throw error if cmd failed
Args:
cmd ([string]): Command to execute on system.
Raises:
Exception: raise command failed exception.
HACommandTerminated: Command termineted exception
Returns:
string: Command output.
"""
try:
cmd_help = cmd.replace(secret,
"****") if secret is not None else cmd
_err = ""
_proc = SimpleProcess(cmd)
_output, _err, _rc = _proc.run(universal_newlines=True)
Log.debug(
f"cmd: {cmd_help}, output: {_output}, err: {_err}, rc: {_rc}")
if _rc != 0 and check_error:
Log.error(
f"cmd: {cmd_help}, output: {_output}, err: {_err}, rc: {_rc}"
)
raise Exception(f"Failed to execute {cmd_help}")
return _output, _err, _rc
except Exception as e:
Log.error(f"Failed to execute {cmd_help} Error: {e}.")
raise HACommandTerminated(
f"Failed to execute {cmd_help} Error: {e}.")
def get(self, key):
"""Get pillar data for key."""
cmd = f"salt-call pillar.get {key} --out=json"
cmd_proc = SimpleProcess(cmd)
out, err, rc = cmd_proc.run()
if rc != 0:
msg = ("pillar.get: Failed to get data for %s."
" 'salt' command not found. " % key) if rc == 127 \
else "pillar.get: Failed to get data for %s. %s" % (key, err)
raise KvError(rc, msg)
res = None
try:
res = json.loads(out)
res = res['local']
except Exception as ex:
raise KvError(
errno.ENOENT,
"pillar.get: Failed to get data for %s. %s" % (key, ex))
if not res:
raise KvError(errno.ENOENT, f"get: No pillar data for key: {key}")
return res
def test_bmc_is_accessible(bmc_ip, bmc_user, bmc_passwd):
"""Check if BMC is accessible through KCS or LAN."""
channel_interface = Conf.get(
consts.SSPL_CONFIG_INDEX, "BMC_INTERFACE>default", "system")
if channel_interface == "system":
# Check BMC is accessible through KCS
cmd = "sudo ipmitool channel info"
expected_channel = "KCS"
channel_found = None
channel_protocol = "Channel Protocol Type"
res_op, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc == 0:
res_op = res_op.decode()
search_res = re.search(
r"%s[\s]+:[\s]+(\w+)(.*)" % channel_protocol, res_op)
if search_res:
channel_found = search_res.groups()[0]
if expected_channel != channel_found:
logger.critical(
"UNEXPECTED BMC CHANNEL TYPE FOUND. Expected: '%s' Found: '%s'" %(
expected_channel, channel_found))
else:
res_err = res_err.decode()
kcs_errors = ("could not find inband device", "driver timeout")
if not any(err for err in kcs_errors if err in res_err):
raise Exception("BMC is NOT accessible through KCS - ERROR: %s" % res_err)
elif channel_interface == "lan":
# Check BMC is accessible through LAN
subcommand = "channel info"
cmd = "sudo ipmitool -H %s -U %s -P %s -I lan %s" %(
bmc_ip, bmc_user, bmc_passwd, subcommand)
_, res_err, res_rc = SimpleProcess(cmd).run()
if res_rc != 0:
raise Exception("BMC is NOT accessible over lan - ERROR: %s" % res_err.decode())
def restore_config():
if os.path.exists("/tmp/lan_disable"):
SimpleProcess("rm -rf /tmp/lan_disable").run()
sleep(0.1)
elif os.path.exists("/tmp/kcs_disable"):
SimpleProcess("rm -rf /tmp/kcs_disable").run()
sleep(0.1)
def reset():
""" Remove/Delete all the data that was created after post install """
conf_file = '/etc/cortx/message_bus.conf'
if os.path.exists(conf_file):
# delete message_types
from cortx.utils.message_bus import MessageBusAdmin
try:
mb = MessageBusAdmin(admin_id='reset')
message_types_list = mb.list_message_types()
if message_types_list:
mb.deregister_message_type(message_types_list)
except MessageBusError as e:
raise SetupError(e.rc, "Can not reset Message Bus. %s", e)
except Exception as e:
raise SetupError(errors.ERR_OP_FAILED, "Can not reset Message Bus. \
%s", e)
# Stop MessageBus Service
cmd = SimpleProcess("systemctl stop cortx_message_bus")
_, stderr, res_rc = cmd.run()
if res_rc != 0:
raise SetupError(res_rc, "Unable to stop MessageBus Service. \
%s", stderr.decode('utf-8'))
return 0
def test_conf_cli_by_get_list(self):
""" Test by retrieving list of values for given keys seperated by ;"""
cmd = "conf json:///tmp/file1.json get bridge>name;bridge>lte_type[0]>name"
cmd_proc = SimpleProcess(cmd)
result_data = cmd_proc.run()
self.assertTrue(True if result_data[2]==0 and
result_data[0]==b'["Homebridge", "3g"]\n' else False, result_data[1])
def configure_openldap_replication(self):
"""Configure openldap replication within a storage set."""
self.logger.info('Cleaning up old Openldap replication configuration')
# Delete ldap replication cofiguration
self.delete_replication_config()
self.logger.info('Open ldap replication configuration started')
storage_set_count = self.get_confvalue(
self.get_confkey('CONFIG>CONFSTORE_STORAGE_SET_COUNT_KEY').replace(
"cluster-id", self.cluster_id))
index = 0
while index < int(storage_set_count):
server_nodes_list = self.get_confkey(
'CONFIG>CONFSTORE_STORAGE_SET_SERVER_NODES_KEY').replace(
"cluster-id",
self.cluster_id).replace("storage-set-count", str(index))
server_nodes_list = self.get_confvalue(server_nodes_list)
if type(server_nodes_list) is str:
# list is stored as string in the confstore file
server_nodes_list = literal_eval(server_nodes_list)
if len(server_nodes_list) > 1:
self.logger.info(
f'Setting ldap-replication for storage_set:{index}')
Path(self.s3_tmp_dir).mkdir(parents=True, exist_ok=True)
ldap_hosts_list_file = os.path.join(
self.s3_tmp_dir, "ldap_hosts_list_file.txt")
with open(ldap_hosts_list_file, "w") as f:
for node_machine_id in server_nodes_list:
private_fqdn = self.get_confvalue(
f'server_node>{node_machine_id}>network>data>private_fqdn'
)
f.write(f'{private_fqdn}\n')
self.logger.info(
f'output of ldap_hosts_list_file.txt: {private_fqdn}'
)
cmd = [
'/opt/seagate/cortx/s3/install/ldap/replication/setupReplicationScript.sh',
'-h', ldap_hosts_list_file, '-p', f'{self.rootdn_passwd}'
]
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
self.logger.info(
f'output of setupReplicationScript.sh: {stdout}')
os.remove(ldap_hosts_list_file)
if retcode != 0:
self.logger.error(
f'error of setupReplicationScript.sh: {stderr}')
raise S3PROVError(
f"{cmd} failed with err: {stderr}, out: {stdout}, ret: {retcode}"
)
else:
self.logger.warning(
f'warning of setupReplicationScript.sh: {stderr}')
index += 1
# TODO: set replication across storage-sets
self.logger.info('Open ldap replication configuration completed')
def test_conf_cli_kv_delim_no_value(self):
"""Test by trying to set given key with no value provided."""
set_cmd = "conf json:///tmp/file1.json set -d : cluster>id093d"
set_cmd_proc = SimpleProcess(set_cmd)
result_data = set_cmd_proc.run()
self.assertTrue(
False if result_data[2] > 0 and result_data[0] != b'' else True)
def test_conf_cli_by_get_diff(self):
""" Test by retrieving a value using get api """
cmd = "conf json:///tmp/file1.json diff json:///tmp/file2.json -k version;branch"
cmd_proc = SimpleProcess(cmd)
result_data = cmd_proc.run()
self.assertTrue(True if result_data[2] == 0 and
result_data[0] == b'1c1\n< [2.0.0, main]\n---\n> [2.0.1, stable]\n\n' else False, result_data[1])
def run_cmd(self, cmd, check_error=True):
"""
Run command and throw error if cmd failed
Args:
cmd ([string]): Command to execute on system.
Raises:
Exception: raise command failed exception.
HACommandTerminated: Command termineted exception
Returns:
string: Command output.
"""
try:
_err = ""
_proc = SimpleProcess(cmd)
_output, _err, _rc = _proc.run(universal_newlines=True)
Log.debug(f"cmd: {cmd}, output: {_output}, err: {_err}, rc: {_rc}")
if _rc != 0 and check_error:
Log.error(
f"cmd: {cmd}, output: {_output}, err: {_err}, rc: {_rc}")
raise Exception(f"Failed to execute {cmd}")
return _output, _err, _rc
except Exception as e:
Log.error("Failed to execute %s Error: %s %s" % (cmd, e, _err))
raise HACommandTerminated("Failed to execute %s Error: %s %s" %
(cmd, e, _err))
def get_rabbitmq_cluster_nodes(self):
cluster_nodes = None
if self.rabbitmq_major_release == '3' and \
self.rabbitmq_maintenance_release == '8':
rmq_cluster_status_cmd = '/usr/sbin/rabbitmqctl cluster_status' + \
' --formatter json'
output, error, returncode = SimpleProcess(rmq_cluster_status_cmd).run()
try:
rabbitmq_cluster_status = json.loads(output)
except Exception:
raise SetupError(
errno.EINVAL,
"RabbitMQ cluster status is not okay! \n, status : %s",
output)
if returncode:
raise SetupError(returncode, error)
running_nodes = rabbitmq_cluster_status['running_nodes']
for i, node in enumerate(running_nodes):
running_nodes[i] = node.replace('rabbit@', '')
cluster_nodes = " ".join(running_nodes)
elif self.rabbitmq_version == '3.3.5':
rmq_cluster_status_cmd = "rabbitmqctl cluster_status"
output, error, returncode = SimpleProcess(rmq_cluster_status_cmd).run()
cluster_nodes = re.search(
r"running_nodes,\['rabbit@(?P<cluster_nodes>.+?)'\]",
output.decode()).groupdict()["cluster_nodes"]
else:
raise SetupError(
errno.EINVAL,
"This RabbitMQ version : %s is not supported",
self.rabbitmq_version)
return cluster_nodes
def _create_msg_bus_config(kafka_server_list, port_list):
""" Create the config file required for message bus """
from cortx.utils.conf_store import Conf
with open(r'/etc/cortx/message_bus.conf.new', 'w+') as file:
json.dump({}, file, indent=2)
Conf.load("index", "json:///etc/cortx/message_bus.conf.new")
Conf.set("index", "message_broker>type", "kafka")
for i in range(len(kafka_server_list)):
Conf.set("index", f"message_broker>cluster[{i}]", \
{"server": kafka_server_list[i], "port": port_list[i]})
Conf.save("index")
# copy this conf file as message_bus.conf
cmd = "/bin/mv /etc/cortx/message_bus.conf.new" + \
" /etc/cortx/message_bus.conf"
try:
cmd_proc = SimpleProcess(cmd)
res_op, res_err, res_rc = cmd_proc.run()
if res_rc != 0:
raise SetupError(errno.EIO, \
"/etc/cortx/message_bus.conf file creation failed, \
rc = %d", res_rc)
except Exception as e:
raise SetupError(errno.EIO, \
"/etc/cortx/message_bus.conf file creation failed, %s", e)
return res_rc
def get_devices(self):
output, err, returncode = SimpleProcess(
f"mdadm --detail --test {self.raid}").run()
output = (output + err).decode()
devices = []
for state in re.findall(r"^\s*\d+\s*\d+\s*\d+\s*\d+\s*(.*)", output,
re.MULTILINE):
device = {}
device["state"] = state
device["identity"] = {}
path = re.findall(r"\/dev\/(.*)", state)
if path:
device["identity"]["path"] = f"/dev/{path[0]}"
output, _, returncode = SimpleProcess(
f'smartctl -i {device["identity"]["path"]} --json').run()
if returncode == 0:
output = json.loads(output)
try:
device["identity"]["serialNumber"] = output[
"serial_number"]
except KeyError:
device["identity"]["serialNumber"] = "None"
else:
device["identity"]["serialNumber"] = "None"
devices.append(device)
return devices
def process(self):
"""Main processing function."""
#TODO: remove the return in next sprint
self.logger.info(f"Processing {self.name} {self.url}")
self.logger.info("validations started")
self.phase_prereqs_validate(self.name)
self.phase_keys_validate(self.url, self.name)
self.logger.info("validations completed")
self.logger.info("test started")
try:
self.read_endpoint_value()
self.logger.info(f"Endpoint fqdn {self.endpoint}")
cmd = [
SANITY_SCRIPT_PATH, LDAP_SWITCH, f'{self.ldap_passwd}',
ENDPOINT_SWITCH, f'{self.endpoint}'
]
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
self.logger.info(f'output of s3-sanity-test.sh: {stdout}')
if retcode != 0:
self.logger.error(f'error of s3-sanity-test.sh: {stderr}')
raise Exception(
f"{cmd} failed with err: {stderr}, out: {stdout}, ret: {retcode}"
)
else:
self.logger.warning(f'warning of s3-sanity-test.sh: {stderr}')
except Exception as e:
raise Exception(f"{self}: {cmd} exception: {e}")
self.logger.info("test completed")
def _exc_components_cmd(commands: List, bundle_id: str, path: str,
component: str, node_name: str, comment: str):
"""
Executes the Command for Bundle Generation of Every Component.
:param commands: Command of the component :type:str
:param bundle_id: Unique Bundle ID of the generation process. :type:str
:param path: Path to create the tar by components :type:str
:param component: Name of Component to be executed :type: str
:param node_name:Name of Node where the Command is being Executed
:type:str
:param comment: :User Comment: type:str
:return:
"""
for command in commands:
Log.info(f"Executing command -> {command} {bundle_id} {path}")
cmd_proc = SimpleProcess(f"{command} {bundle_id} {path}")
output, err, return_code = cmd_proc.run()
Log.debug(f"Command Output -> {output} {err}, {return_code}")
if return_code != 0:
Log.error(f"Command Output -> {output} {err}, {return_code}")
ComponentsBundle._publish_log(
f"Bundle generation failed for '{component}'", ERROR,
bundle_id, node_name, comment)
else:
ComponentsBundle._publish_log(
f"Bundle generation started for '{component}'", INFO,
bundle_id, node_name, comment)
def test_conf_cli_by_get_diff(self):
""" Test by retrieving a value using get api """
cmd = "conf json:///tmp/file1.json diff json:///tmp/file2.json -k version;branch"
cmd_proc = SimpleProcess(cmd)
result_data = cmd_proc.run()
self.assertTrue(
True if result_data[2] == 0 and
(result_data[0] is not None) else False, result_data[1])
def test_conf_cli_compare(self):
cmd = "conf json:///tmp/file1.json compare json:///tmp/file2.json"
cmd_proc = SimpleProcess(cmd)
result_data = cmd_proc.run()
print(result_data)
print(result_data[0])
print(result_data[1])
self.assertEqual(result_data[2], 0)
def tearDownClass(cls):
"""Test teardown class."""
cmds = ["systemctl start rsyslog", "systemctl start elasticsearch", \
"pcs cluster start --all"]
for cmd in cmds:
cmd_proc = SimpleProcess(cmd)
_, _, rc = cmd_proc.run()
assert (rc == 0)
def disable_services(self, s3services_list):
"""Disable services specified as parameter."""
for service_name in s3services_list:
cmd = ['/bin/systemctl', 'disable', f'{service_name}']
handler = SimpleProcess(cmd)
self.logger.info(f"Disabling {service_name}")
res_op, res_err, res_rc = handler.run()
if res_rc != 0:
raise Exception(f"{cmd} failed with err: {res_err}, out: {res_op}, ret: {res_rc}")
def test_conf_cli_invalid_kv_delim(self):
"""
Test by trying to set a value into given key position with
invalid kv_delim mentioned
"""
set_cmd = "conf json:///tmp/file1.json set -d { cluster>id}093d"
set_cmd_proc = SimpleProcess(set_cmd)
result_data = set_cmd_proc.run()
self.assertTrue(True if result_data[2] == 22 else False)