def validate_hca(self, provider, nodes):
"""Check if HCA presence and ports."""
for node in nodes:
if provider.lower() in self.hca_checks:
cmd = f"ssh {node} lspci -nn | grep '{provider.capitalize()}'"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"{provider.capitalize()} Host Channel Adapters card "
f"(possibly with Infiniband capability) is not detected on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD '{cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
ports_cmd = f"{cmd} | wc -l"
ports_cmd_proc = SimpleProcess(ports_cmd)
ports_run_result = ports_cmd_proc.run()
if ports_run_result[1] or ports_run_result[2]:
res = (f"{provider.capitalize()} Host Channel Adapters ports were not "
f"detected by command {cmd}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {ports_run_result[0]}. {ports_run_result[1]}")
raise VError(errno.EINVAL, res)
res = (ports_run_result[0].decode('utf-8').strip())
if int(res) == 0:
res = (f"{provider.capitalize()} Host Channel Adapters ports were not "
f"detected by command {cmd}. "
"For high-speed data network, it is expected to "
"have at least one port present over some PCIe slot. ")
raise VError(errno.EINVAL, res)
def validate_service_status(self):
"""Validate service status."""
url = f"http://{self.host}:{self.port}{self.service_url}"
try:
status_code = requests.get(url).status_code
if status_code != 200:
raise VError(errno.ECOMM, (
f"Error {status_code} obtained while "
f"connecting to {self.service_name} service on {self.host}:{self.port}."
))
except requests.exceptions.InvalidURL:
raise VError(errno.EINVAL,
(f"Either or all inputs host '{self.host}' and port "
f"'{self.port}' are invalid."))
except requests.exceptions.ConnectionError:
raise VError(
errno.ECONNREFUSED,
f"No {self.service_name} service running on {self.host}:{self.port}"
)
except requests.exceptions.RequestException as req_ex:
raise VError(
errno.ECOMM,
(f"Error connecting to {self.service_name} service on "
f"{self.host}:{self.port}. {req_ex}"))
def _validate(cortx_solution_config: dict):
"""
Validates a given node to have required properties
Raises exception if there is any entry missing
"""
required_keys_for_cortx_conf = ['external', 'common']
for k in required_keys_for_cortx_conf:
if cortx_solution_config.get(k) is None:
raise VError(
errno.EINVAL,
f"'{k}' property is unspecified in cortx_config.")
if k == 'external':
required_external_keys = const.REQUIRED_EXTERNAL_SW
for e_key in required_external_keys:
try:
if cortx_solution_config[k][e_key]['endpoints'] is None:
raise VError(
errno.EINVAL,
f"'Endpoint for {e_key}' is unspecified in cortx_config."
)
except KeyError as e:
raise CortxProvisionerError(
errno.EINVAL,
f'{str(e)} is unspecified for external in cortx_config.'
)
def validate_paths(self, host, paths):
"""Check if path are found and matching."""
for path in paths:
if ":" not in path:
raise VError(
errno.EINVAL,
"Input should contain colon ':' separated values, given: %s"
% path)
_type = path.split(":")[0]
_path = path.split(":")[1]
if _type not in ["file", "dir", "link", "device"]:
raise VError(errno.EINVAL, "invalid type: %s" % _type)
if _type == "file":
__type = "regular file"
elif _type == "dir":
__type = "directory"
elif _type == "link":
__type = "symbolic link"
elif _type == "device":
__type = "block special file"
else:
raise VError(errno.EINVAL, "Invalid type: %s" % _type)
if _path == "":
raise VError(
errno.EINVAL,
"Absolute path must be provided, given: %s!" % path)
if host != None:
result = self.__run_cmd(f"ssh {host} stat {_path} --printf=%F")
else:
result = self.__run_cmd(f"stat {_path} --printf=%F")
if result.find(f"{__type}") == -1:
raise VError(errno.EINVAL, "object: %s not found" % path)
def validate_lvm(self, nodes):
"""Validate lvms are present and size."""
for node in nodes:
cmd = f"ssh {node} vgdisplay | grep vg_metadata_{node}"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Failed to get vg_metadata_{node} on {node}."
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
cmd = f"ssh {node} vgdisplay | grep vg_metadata | wc -l"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Failed to get lvms on {node}."
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
res = (run_result[0].decode('utf-8').strip())
if not res or (int(res) != len(nodes)):
raise VError(errno.EINVAL,
f"No. of Lvms {res} is not correct for {node}.")
def validate(self, v_type, args):
"""
Process BMC validations.
Usage (arguments to be provided):
1. bmc accessible <node1> <node2> <...>
2. bmc stonith <node> <bmc_ip> <bmc_user> <bmc_passwd>
"""
if not isinstance(args, list):
raise VError(errno.EINVAL, f"Invalid parameters '{args}'")
if v_type == "accessible":
if len(args) < 1:
raise VError(errno.EINVAL, "No parameters specified")
self.validate_bmc_accessibility(args)
elif v_type == "stonith":
if len(args) < 4:
raise VError(
errno.EINVAL,
f"Insufficient parameters '{args}' for 'bmc stonith'. Refer usage."
)
elif len(args) > 4:
raise VError(
errno.EINVAL,
f"Too many parameters '{args}' for 'bmc stonith'. Refer usage."
)
self.validate_bmc_stonith_config(args[0], args[1], args[2],
args[3])
else:
raise VError(errno.EINVAL,
f"Action parameter '{v_type}' not supported")
def __search_pkg(self, cmd):
# print(f"Running {cmd}")
handler = SimpleProcess(cmd)
stdout, stderr, retcode = handler.run()
if retcode != 0:
raise VError(errno.EINVAL,
"cmd: %s failed with error code: %d" % (cmd, retcode))
if stderr:
raise VError(errno.EINVAL,
"cmd: %s failed with stderr: %s" % (cmd, stderr))
# To calm down codacy.
return stdout.decode("utf-8")
def test_reconnect_ok(self):
""" Check ssh re-connect after disconnection caller """
# Close ssh connection
self.session.disconnect()
if self.session.is_connection_alive():
raise VError(errno.EINVAL,
f"SSH connection is not closed with client '{self.session.host}'")
# Establish ssh to same client without invoking instance
# and validate ssh connection is active
self.session.connect()
if not self.session.is_connection_alive():
raise VError(errno.EINVAL,
f"Connection is lost with client '{self.session.host}'")
def validate_salt_minion_connectivity(self, nodes):
"""Check salt minion connectivity."""
if os.environ.get('USER') != 'root':
res = ("validate_salt_minion: "
"This interface requires root privileges.")
raise VError(errno.EACCES, res)
for node in nodes:
cmd = f"salt -t 5 {node} test.ping"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Salt minion {node} unreachable."
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.ECONNREFUSED, res)
def _is_ip(self, ip):
try:
return ip.count(".") == 3 and all([
self._is_valid_ipv4_part(ip_part) for ip_part in ip.split(".")
])
except Exception as e:
raise VError(errno.EINVAL, f"Invalid IP {ip}.") from e
def validate_services(self, host, services):
"""Check if services are running."""
for service in services:
if host != None:
cmd = f"ssh {host} systemctl status {service}"
else:
cmd = f"systemctl status {service}"
handler = SimpleProcess(cmd)
_, stderr, retcode = handler.run()
if retcode != 0:
raise VError(errno.EINVAL,
"cmd: %s failed with error code: %d"
%(cmd, retcode))
if stderr:
raise VError(errno.EINVAL,
"cmd: %s failed with stderr: %s"
%(cmd, stderr))
def validate(self, v_type, args):
"""
Process consul validations.
Usage (arguments to be provided):
1. consul service localhost 8500
"""
if not isinstance(args, list):
raise VError(errno.EINVAL, "Invalid parameters %s" % args)
if len(args) < 2:
raise VError(errno.EINVAL, "Insufficient parameters. %s" % args)
if v_type == "service":
self.validate_service_status(args[0], args[1])
else:
raise VError(errno.EINVAL,
"Action parameter %s not supported" % v_type)
def test_command_execution_ok(self):
""" Check if command is executed successfully """
cmd = "whoami"
rc, output = self.session.execute(cmd)
if rc != 0 or self.__user not in output:
raise VError(
errno.EINVAL, f"Command execution failed on {self.hostname}.\
Command: '{cmd}'\
Return Code: {rc}")
def validate(self, v_type, args):
"""
Process salt validations.
Usage (arguments to be provided):
1. salt minions <nodes>
"""
if not isinstance(args, list):
raise VError(errno.EINVAL, "Invalid parameters %s" % args)
if len(args) < 1:
raise VError(errno.EINVAL, "Insufficient parameters. %s" % args)
if v_type == "minions":
self.validate_salt_minion_connectivity(args)
else:
raise VError(errno.EINVAL,
"Action parameter %s not supported" % v_type)
def validate(self, v_type, args):
"""
Process network validations.
Usage (arguments to be provided):
1. network connectivity <ip1> <ip2> <ip3>
"""
if not isinstance(args, list):
raise VError(errno.EINVAL, "Invalid parameters %s" % args)
if len(args) < 1:
raise VError(errno.EINVAL, "Insufficient parameters. %s" % args)
if v_type == "connectivity":
self.validate_ip_connectivity(args)
else:
raise VError(errno.EINVAL,
"Action parameter %s not supported" % v_type)
def validate_ip_connectivity(self, ips):
"""Check if IPs are reachable."""
unreachable_ips = []
for ip in ips:
if ip.count(".") == 3 and all(self._is_valid_ipv4_part(ip_part)
for ip_part in ip.split(".")):
cmd = f"ping -c 1 -W 1 {ip}"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[2]:
unreachable_ips.append(ip)
else:
raise VError(errno.EINVAL, f"Invalid ip {ip}.")
if len(unreachable_ips) != 0:
raise VError(
errno.ECONNREFUSED, "Ping failed for IP(s). %s" % unreachable_ips)
def validate_bmc_stonith_config(self, node, bmc_ip, bmc_user, bmc_passwd):
"""Validations for BMC STONITH Configuration."""
cmd = f"fence_ipmilan -P -a {bmc_ip} -o status -l {bmc_user} -p {bmc_passwd}"
rc, output = self.session.execute(cmd)
if rc != 0:
msg = f"Failed to check BMC STONITH Config. Command: '{cmd}',\
Return Code: '{rc}'."
msg += output
raise VError(errno.EINVAL, msg)
def validate_pip3s(self, host, pkgs):
"""Check if pip3 pkg is installed."""
for pkg in pkgs:
if host != None:
result = self.__search_pkg(f"ssh {host} pip3 list")
else:
result = self.__search_pkg("pip3 list")
if result.find(f"{pkg}") == -1:
raise VError(errno.EINVAL, "pip3 pkg: %s not found" % pkg)
def validate(self, v_type, args):
"""
Process elasticsearch validations.
Usage (arguments to be provided):
1. elasticsearch service localhost 9200
"""
if not isinstance(args, list):
raise VError(errno.EINVAL, f"Invalid parameters {args}")
if len(args) < 2:
raise VError(errno.EINVAL, f"Insufficient parameters. {args}")
if v_type == "service":
HttpService("elasticsearch", args[0],
args[1]).validate_service_status()
else:
raise VError(errno.EINVAL,
f"Action parameter {v_type} not supported")
def validate_inband_bmc_channel(self, node):
"""Get BMC channel information (inband)."""
cmd = f"ipmitool {self.channel_cmd}"
rc, output = self.session.execute(cmd)
if rc != 0:
msg = f"Failed to get BMC channel info of '{node}''. Command: '{cmd}',\
Return Code: '{rc}'."
msg += output
raise VError(errno.EINVAL, msg)
return True
def __ping_bmc(self, node):
"""Ping BMC IP."""
ip = self.__get_bmc_ip(node)
cmd = f"ping -c 1 -W 1 {ip}"
rc, output = self.session.execute(cmd)
if rc != 0:
msg = f"Ping failed for IP '{ip}'. Command: '{cmd}',\
Return Code: '{rc}'."
msg += output
raise VError(errno.ECONNREFUSED, msg)
def validate(self, v_type, args):
"""
Process BMC validations.
Usage (arguments to be provided):
1. bmc accessible <node> <bmc_ip> <bmc_user> <bmc_passwd>
2. bmc stonith <node> <bmc_ip> <bmc_user> <bmc_passwd>
"""
if not isinstance(args, list):
raise VError(errno.EINVAL, f"Invalid parameters '{args}'")
if len(args) < 1:
raise VError(errno.EINVAL, "No parameters specified")
if len(args) < 4:
raise VError(
errno.EINVAL,
f"Insufficient parameters '{args}' for 'bmc validate'. Refer usage."
)
elif len(args) > 4:
raise VError(
errno.EINVAL,
f"Too many parameters '{args}' for 'bmc validate'. Refer usage."
)
# Root user to execute ipmitool command
user = "******"
node = args[0]
self.session = SSHChannel(node,
user,
pkey_filename='/root/.ssh/id_rsa_prvsnr')
if v_type == "accessible":
self.validate_bmc_accessibility(node, args[1], args[2], args[3])
elif v_type == "stonith":
self.validate_bmc_stonith_config(node, args[1], args[2], args[3])
else:
raise VError(errno.EINVAL,
f"Action parameter '{v_type}' not supported")
self.session.disconnect()
def __get_bmc_ip(self, node):
"""Get BMC IP along with status of command."""
cmd = "ipmitool lan print 1 | grep 'IP Address'"
rc, output = self.session.execute(cmd)
if rc != 0:
msg = f"Failed to get BMC IP of {node}. Command: '{cmd}',\
Return Code: '{rc}'."
msg += output
raise VError(errno.EINVAL, msg)
bmc_ip = output.split()[-1]
return bmc_ip
def validate_network_drivers(self, driver, nodes):
"""Check if drivers (eg: Mellanox OFED) are proper."""
for node in nodes:
cmd = f"ssh {node} rpm -qa | grep {driver}"
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Given Driver '{driver}' Is Not Installed on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
def validate_bmc_channel_over_lan(self, bmc_ip, bmc_user, bmc_passwd):
"""Get BMC channel information over lan (out-of-band)."""
# check BMC ip is accessible over lan (out of band)
cmd = f"ipmitool -H {bmc_ip} -U {bmc_user} -P {bmc_passwd} -I lan {self.channel_cmd}"
rc, output = self.session.execute(cmd)
if rc != 0:
msg = f"Failed to get BMC channel info of '{bmc_ip}' over lan. Command: '{cmd}',\
Return Code: '{rc}'."
msg += output
raise VError(errno.EINVAL, msg)
return True
def validate_controller_accessibility(self, ip, username, password):
"""Check contoller console is accessible to node."""
# Check if ssh connection is successful
try:
session = SSHChannel(host=ip, username=username, password=password)
session.disconnect()
except:
err = traceback.format_exc()
raise VError(
errno.EINVAL,
"Failed to create ssh connection to %s, Error: %s" % (ip, err))
# ping controller IP
cmd = "ping -c 1 -W 1 %s" % ip
cmd_proc = SimpleProcess(cmd)
_, stderr, rc = cmd_proc.run()
if rc != 0:
msg = "Ping failed for IP '%s'. Command: '%s', Return Code: '%s'." % (
ip, cmd, rc)
msg += stderr.decode("utf-8").replace('\r', '').replace('\n', '')
raise VError(errno.EINVAL, msg)
def validate(self, v_type: str, index: str, keylist: list):
"""
Process confstore key validations.
Usage (arguments to be provided):
1. exists keylist <i.e. ['k1', 'k2>k3', ...]>
"""
if v_type == "exists":
return self.validate_keys(index, keylist)
else:
raise VError(errno.EINVAL,
"Action parameter %s not supported" % v_type)
def __get_bmc_power_status(self, node):
"""Get BMC power status."""
cmd = "ipmitool chassis status | grep 'System Power'"
rc, output = self.session.execute(cmd)
if rc != 0:
msg = f"Failed to get BMC power status of {node}. Command: '{cmd}',\
Return Code: '{rc}'."
msg += output
raise VError(errno.EINVAL, msg)
pw_status = output.split()[-1]
return pw_status
def validate_passwordless_ssh(self, user, nodes):
"""Check passwordless ssh."""
for node in nodes:
cmd = ("ssh -o PreferredAuthentications=publickey "
f"-o StrictHostKeyChecking=no {user}@{node} /bin/true")
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (f"Passwordless ssh not configured for {node}."
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.ECONNREFUSED, res)
def validate_hba(self, provider, nodes):
"""Check HBA presence and ports."""
for node in nodes:
if provider.lower() == "lsi":
cmd = f"ssh {node} lspci -nn | grep 'SCSI'"
# TO DO: Use paramiko for ssh.
cmd_proc = SimpleProcess(cmd)
run_result = cmd_proc.run()
if run_result[1] or run_result[2]:
res = (
"Host Bus Adapters (HBA) for "
f"SAS channels not detected on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}")
raise VError(errno.EINVAL, res)
ports_cmd = f"ssh {node} ls /sys/class/scsi_host/ | wc -l"
ports_cmd_proc = SimpleProcess(ports_cmd)
ports_result = ports_cmd_proc.run()
if ports_result[1] or ports_result[2]:
res = (
"Host Bus Adapters (HBA) for "
f"SAS channels not detected on {node}. "
f"Also, check if '{node}' is valid. "
f"CMD {cmd} failed. {ports_result[0]}. {ports_result[1]}"
)
raise VError(errno.EINVAL, res)
res = ports_result[0].decode('utf-8').strip()
if int(res) == 0:
res = (
"Host Bus Adapters (HBA) for "
f"SAS channels not detected on {node}. "
"For storage connectivity over SAS channels "
"to JBOD/RBOD there is expectation for a PCIe HBA card "
"to be present. Please check HW, if this system "
"expects a connection to either JBOD or RBOD.")
raise VError(errno.EINVAL, res)