def validate_hca(self, provider, nodes): """Check if HCA presence and ports.""" for node in nodes: if provider.lower() in self.hca_checks: cmd = f"ssh {node} lspci -nn | grep '{provider.capitalize()}'" cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = (f"{provider.capitalize()} Host Channel Adapters card " f"(possibly with Infiniband capability) is not detected on {node}. " f"Also, check if '{node}' is valid. " f"CMD '{cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.EINVAL, res) ports_cmd = f"{cmd} | wc -l" ports_cmd_proc = SimpleProcess(ports_cmd) ports_run_result = ports_cmd_proc.run() if ports_run_result[1] or ports_run_result[2]: res = (f"{provider.capitalize()} Host Channel Adapters ports were not " f"detected by command {cmd}. " f"Also, check if '{node}' is valid. " f"CMD {cmd} failed. {ports_run_result[0]}. {ports_run_result[1]}") raise VError(errno.EINVAL, res) res = (ports_run_result[0].decode('utf-8').strip()) if int(res) == 0: res = (f"{provider.capitalize()} Host Channel Adapters ports were not " f"detected by command {cmd}. " "For high-speed data network, it is expected to " "have at least one port present over some PCIe slot. ") raise VError(errno.EINVAL, res)
def validate_service_status(self): """Validate service status.""" url = f"http://{self.host}:{self.port}{self.service_url}" try: status_code = requests.get(url).status_code if status_code != 200: raise VError(errno.ECOMM, ( f"Error {status_code} obtained while " f"connecting to {self.service_name} service on {self.host}:{self.port}." )) except requests.exceptions.InvalidURL: raise VError(errno.EINVAL, (f"Either or all inputs host '{self.host}' and port " f"'{self.port}' are invalid.")) except requests.exceptions.ConnectionError: raise VError( errno.ECONNREFUSED, f"No {self.service_name} service running on {self.host}:{self.port}" ) except requests.exceptions.RequestException as req_ex: raise VError( errno.ECOMM, (f"Error connecting to {self.service_name} service on " f"{self.host}:{self.port}. {req_ex}"))
def _validate(cortx_solution_config: dict): """ Validates a given node to have required properties Raises exception if there is any entry missing """ required_keys_for_cortx_conf = ['external', 'common'] for k in required_keys_for_cortx_conf: if cortx_solution_config.get(k) is None: raise VError( errno.EINVAL, f"'{k}' property is unspecified in cortx_config.") if k == 'external': required_external_keys = const.REQUIRED_EXTERNAL_SW for e_key in required_external_keys: try: if cortx_solution_config[k][e_key]['endpoints'] is None: raise VError( errno.EINVAL, f"'Endpoint for {e_key}' is unspecified in cortx_config." ) except KeyError as e: raise CortxProvisionerError( errno.EINVAL, f'{str(e)} is unspecified for external in cortx_config.' )
def validate_paths(self, host, paths): """Check if path are found and matching.""" for path in paths: if ":" not in path: raise VError( errno.EINVAL, "Input should contain colon ':' separated values, given: %s" % path) _type = path.split(":")[0] _path = path.split(":")[1] if _type not in ["file", "dir", "link", "device"]: raise VError(errno.EINVAL, "invalid type: %s" % _type) if _type == "file": __type = "regular file" elif _type == "dir": __type = "directory" elif _type == "link": __type = "symbolic link" elif _type == "device": __type = "block special file" else: raise VError(errno.EINVAL, "Invalid type: %s" % _type) if _path == "": raise VError( errno.EINVAL, "Absolute path must be provided, given: %s!" % path) if host != None: result = self.__run_cmd(f"ssh {host} stat {_path} --printf=%F") else: result = self.__run_cmd(f"stat {_path} --printf=%F") if result.find(f"{__type}") == -1: raise VError(errno.EINVAL, "object: %s not found" % path)
def validate_lvm(self, nodes): """Validate lvms are present and size.""" for node in nodes: cmd = f"ssh {node} vgdisplay | grep vg_metadata_{node}" cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = (f"Failed to get vg_metadata_{node} on {node}." f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.EINVAL, res) cmd = f"ssh {node} vgdisplay | grep vg_metadata | wc -l" cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = (f"Failed to get lvms on {node}." f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.EINVAL, res) res = (run_result[0].decode('utf-8').strip()) if not res or (int(res) != len(nodes)): raise VError(errno.EINVAL, f"No. of Lvms {res} is not correct for {node}.")
def validate(self, v_type, args): """ Process BMC validations. Usage (arguments to be provided): 1. bmc accessible <node1> <node2> <...> 2. bmc stonith <node> <bmc_ip> <bmc_user> <bmc_passwd> """ if not isinstance(args, list): raise VError(errno.EINVAL, f"Invalid parameters '{args}'") if v_type == "accessible": if len(args) < 1: raise VError(errno.EINVAL, "No parameters specified") self.validate_bmc_accessibility(args) elif v_type == "stonith": if len(args) < 4: raise VError( errno.EINVAL, f"Insufficient parameters '{args}' for 'bmc stonith'. Refer usage." ) elif len(args) > 4: raise VError( errno.EINVAL, f"Too many parameters '{args}' for 'bmc stonith'. Refer usage." ) self.validate_bmc_stonith_config(args[0], args[1], args[2], args[3]) else: raise VError(errno.EINVAL, f"Action parameter '{v_type}' not supported")
def __search_pkg(self, cmd): # print(f"Running {cmd}") handler = SimpleProcess(cmd) stdout, stderr, retcode = handler.run() if retcode != 0: raise VError(errno.EINVAL, "cmd: %s failed with error code: %d" % (cmd, retcode)) if stderr: raise VError(errno.EINVAL, "cmd: %s failed with stderr: %s" % (cmd, stderr)) # To calm down codacy. return stdout.decode("utf-8")
def test_reconnect_ok(self): """ Check ssh re-connect after disconnection caller """ # Close ssh connection self.session.disconnect() if self.session.is_connection_alive(): raise VError(errno.EINVAL, f"SSH connection is not closed with client '{self.session.host}'") # Establish ssh to same client without invoking instance # and validate ssh connection is active self.session.connect() if not self.session.is_connection_alive(): raise VError(errno.EINVAL, f"Connection is lost with client '{self.session.host}'")
def validate_salt_minion_connectivity(self, nodes): """Check salt minion connectivity.""" if os.environ.get('USER') != 'root': res = ("validate_salt_minion: " "This interface requires root privileges.") raise VError(errno.EACCES, res) for node in nodes: cmd = f"salt -t 5 {node} test.ping" cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = (f"Salt minion {node} unreachable." f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.ECONNREFUSED, res)
def _is_ip(self, ip): try: return ip.count(".") == 3 and all([ self._is_valid_ipv4_part(ip_part) for ip_part in ip.split(".") ]) except Exception as e: raise VError(errno.EINVAL, f"Invalid IP {ip}.") from e
def validate_services(self, host, services): """Check if services are running.""" for service in services: if host != None: cmd = f"ssh {host} systemctl status {service}" else: cmd = f"systemctl status {service}" handler = SimpleProcess(cmd) _, stderr, retcode = handler.run() if retcode != 0: raise VError(errno.EINVAL, "cmd: %s failed with error code: %d" %(cmd, retcode)) if stderr: raise VError(errno.EINVAL, "cmd: %s failed with stderr: %s" %(cmd, stderr))
def validate(self, v_type, args): """ Process consul validations. Usage (arguments to be provided): 1. consul service localhost 8500 """ if not isinstance(args, list): raise VError(errno.EINVAL, "Invalid parameters %s" % args) if len(args) < 2: raise VError(errno.EINVAL, "Insufficient parameters. %s" % args) if v_type == "service": self.validate_service_status(args[0], args[1]) else: raise VError(errno.EINVAL, "Action parameter %s not supported" % v_type)
def test_command_execution_ok(self): """ Check if command is executed successfully """ cmd = "whoami" rc, output = self.session.execute(cmd) if rc != 0 or self.__user not in output: raise VError( errno.EINVAL, f"Command execution failed on {self.hostname}.\ Command: '{cmd}'\ Return Code: {rc}")
def validate(self, v_type, args): """ Process salt validations. Usage (arguments to be provided): 1. salt minions <nodes> """ if not isinstance(args, list): raise VError(errno.EINVAL, "Invalid parameters %s" % args) if len(args) < 1: raise VError(errno.EINVAL, "Insufficient parameters. %s" % args) if v_type == "minions": self.validate_salt_minion_connectivity(args) else: raise VError(errno.EINVAL, "Action parameter %s not supported" % v_type)
def validate(self, v_type, args): """ Process network validations. Usage (arguments to be provided): 1. network connectivity <ip1> <ip2> <ip3> """ if not isinstance(args, list): raise VError(errno.EINVAL, "Invalid parameters %s" % args) if len(args) < 1: raise VError(errno.EINVAL, "Insufficient parameters. %s" % args) if v_type == "connectivity": self.validate_ip_connectivity(args) else: raise VError(errno.EINVAL, "Action parameter %s not supported" % v_type)
def validate_ip_connectivity(self, ips): """Check if IPs are reachable.""" unreachable_ips = [] for ip in ips: if ip.count(".") == 3 and all(self._is_valid_ipv4_part(ip_part) for ip_part in ip.split(".")): cmd = f"ping -c 1 -W 1 {ip}" cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[2]: unreachable_ips.append(ip) else: raise VError(errno.EINVAL, f"Invalid ip {ip}.") if len(unreachable_ips) != 0: raise VError( errno.ECONNREFUSED, "Ping failed for IP(s). %s" % unreachable_ips)
def validate_bmc_stonith_config(self, node, bmc_ip, bmc_user, bmc_passwd): """Validations for BMC STONITH Configuration.""" cmd = f"fence_ipmilan -P -a {bmc_ip} -o status -l {bmc_user} -p {bmc_passwd}" rc, output = self.session.execute(cmd) if rc != 0: msg = f"Failed to check BMC STONITH Config. Command: '{cmd}',\ Return Code: '{rc}'." msg += output raise VError(errno.EINVAL, msg)
def validate_pip3s(self, host, pkgs): """Check if pip3 pkg is installed.""" for pkg in pkgs: if host != None: result = self.__search_pkg(f"ssh {host} pip3 list") else: result = self.__search_pkg("pip3 list") if result.find(f"{pkg}") == -1: raise VError(errno.EINVAL, "pip3 pkg: %s not found" % pkg)
def validate(self, v_type, args): """ Process elasticsearch validations. Usage (arguments to be provided): 1. elasticsearch service localhost 9200 """ if not isinstance(args, list): raise VError(errno.EINVAL, f"Invalid parameters {args}") if len(args) < 2: raise VError(errno.EINVAL, f"Insufficient parameters. {args}") if v_type == "service": HttpService("elasticsearch", args[0], args[1]).validate_service_status() else: raise VError(errno.EINVAL, f"Action parameter {v_type} not supported")
def validate_inband_bmc_channel(self, node): """Get BMC channel information (inband).""" cmd = f"ipmitool {self.channel_cmd}" rc, output = self.session.execute(cmd) if rc != 0: msg = f"Failed to get BMC channel info of '{node}''. Command: '{cmd}',\ Return Code: '{rc}'." msg += output raise VError(errno.EINVAL, msg) return True
def __ping_bmc(self, node): """Ping BMC IP.""" ip = self.__get_bmc_ip(node) cmd = f"ping -c 1 -W 1 {ip}" rc, output = self.session.execute(cmd) if rc != 0: msg = f"Ping failed for IP '{ip}'. Command: '{cmd}',\ Return Code: '{rc}'." msg += output raise VError(errno.ECONNREFUSED, msg)
def validate(self, v_type, args): """ Process BMC validations. Usage (arguments to be provided): 1. bmc accessible <node> <bmc_ip> <bmc_user> <bmc_passwd> 2. bmc stonith <node> <bmc_ip> <bmc_user> <bmc_passwd> """ if not isinstance(args, list): raise VError(errno.EINVAL, f"Invalid parameters '{args}'") if len(args) < 1: raise VError(errno.EINVAL, "No parameters specified") if len(args) < 4: raise VError( errno.EINVAL, f"Insufficient parameters '{args}' for 'bmc validate'. Refer usage." ) elif len(args) > 4: raise VError( errno.EINVAL, f"Too many parameters '{args}' for 'bmc validate'. Refer usage." ) # Root user to execute ipmitool command user = "******" node = args[0] self.session = SSHChannel(node, user, pkey_filename='/root/.ssh/id_rsa_prvsnr') if v_type == "accessible": self.validate_bmc_accessibility(node, args[1], args[2], args[3]) elif v_type == "stonith": self.validate_bmc_stonith_config(node, args[1], args[2], args[3]) else: raise VError(errno.EINVAL, f"Action parameter '{v_type}' not supported") self.session.disconnect()
def __get_bmc_ip(self, node): """Get BMC IP along with status of command.""" cmd = "ipmitool lan print 1 | grep 'IP Address'" rc, output = self.session.execute(cmd) if rc != 0: msg = f"Failed to get BMC IP of {node}. Command: '{cmd}',\ Return Code: '{rc}'." msg += output raise VError(errno.EINVAL, msg) bmc_ip = output.split()[-1] return bmc_ip
def validate_network_drivers(self, driver, nodes): """Check if drivers (eg: Mellanox OFED) are proper.""" for node in nodes: cmd = f"ssh {node} rpm -qa | grep {driver}" cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = (f"Given Driver '{driver}' Is Not Installed on {node}. " f"Also, check if '{node}' is valid. " f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.EINVAL, res)
def validate_bmc_channel_over_lan(self, bmc_ip, bmc_user, bmc_passwd): """Get BMC channel information over lan (out-of-band).""" # check BMC ip is accessible over lan (out of band) cmd = f"ipmitool -H {bmc_ip} -U {bmc_user} -P {bmc_passwd} -I lan {self.channel_cmd}" rc, output = self.session.execute(cmd) if rc != 0: msg = f"Failed to get BMC channel info of '{bmc_ip}' over lan. Command: '{cmd}',\ Return Code: '{rc}'." msg += output raise VError(errno.EINVAL, msg) return True
def validate_controller_accessibility(self, ip, username, password): """Check contoller console is accessible to node.""" # Check if ssh connection is successful try: session = SSHChannel(host=ip, username=username, password=password) session.disconnect() except: err = traceback.format_exc() raise VError( errno.EINVAL, "Failed to create ssh connection to %s, Error: %s" % (ip, err)) # ping controller IP cmd = "ping -c 1 -W 1 %s" % ip cmd_proc = SimpleProcess(cmd) _, stderr, rc = cmd_proc.run() if rc != 0: msg = "Ping failed for IP '%s'. Command: '%s', Return Code: '%s'." % ( ip, cmd, rc) msg += stderr.decode("utf-8").replace('\r', '').replace('\n', '') raise VError(errno.EINVAL, msg)
def validate(self, v_type: str, index: str, keylist: list): """ Process confstore key validations. Usage (arguments to be provided): 1. exists keylist <i.e. ['k1', 'k2>k3', ...]> """ if v_type == "exists": return self.validate_keys(index, keylist) else: raise VError(errno.EINVAL, "Action parameter %s not supported" % v_type)
def __get_bmc_power_status(self, node): """Get BMC power status.""" cmd = "ipmitool chassis status | grep 'System Power'" rc, output = self.session.execute(cmd) if rc != 0: msg = f"Failed to get BMC power status of {node}. Command: '{cmd}',\ Return Code: '{rc}'." msg += output raise VError(errno.EINVAL, msg) pw_status = output.split()[-1] return pw_status
def validate_passwordless_ssh(self, user, nodes): """Check passwordless ssh.""" for node in nodes: cmd = ("ssh -o PreferredAuthentications=publickey " f"-o StrictHostKeyChecking=no {user}@{node} /bin/true") cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = (f"Passwordless ssh not configured for {node}." f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.ECONNREFUSED, res)
def validate_hba(self, provider, nodes): """Check HBA presence and ports.""" for node in nodes: if provider.lower() == "lsi": cmd = f"ssh {node} lspci -nn | grep 'SCSI'" # TO DO: Use paramiko for ssh. cmd_proc = SimpleProcess(cmd) run_result = cmd_proc.run() if run_result[1] or run_result[2]: res = ( "Host Bus Adapters (HBA) for " f"SAS channels not detected on {node}. " f"Also, check if '{node}' is valid. " f"CMD {cmd} failed. {run_result[0]}. {run_result[1]}") raise VError(errno.EINVAL, res) ports_cmd = f"ssh {node} ls /sys/class/scsi_host/ | wc -l" ports_cmd_proc = SimpleProcess(ports_cmd) ports_result = ports_cmd_proc.run() if ports_result[1] or ports_result[2]: res = ( "Host Bus Adapters (HBA) for " f"SAS channels not detected on {node}. " f"Also, check if '{node}' is valid. " f"CMD {cmd} failed. {ports_result[0]}. {ports_result[1]}" ) raise VError(errno.EINVAL, res) res = ports_result[0].decode('utf-8').strip() if int(res) == 0: res = ( "Host Bus Adapters (HBA) for " f"SAS channels not detected on {node}. " "For storage connectivity over SAS channels " "to JBOD/RBOD there is expectation for a PCIe HBA card " "to be present. Please check HW, if this system " "expects a connection to either JBOD or RBOD.") raise VError(errno.EINVAL, res)