def cmd_run(cmd, hostname, raise_on_error=True):
"""Glusto's command runner wrapper.
Args:
cmd (str): Shell command to run on the specified hostname.
hostname (str): hostname where Glusto should run specified command.
raise_on_error (bool): defines whether we should raise exception
in case command execution failed.
Returns:
str: Stripped shell command's stdout value if not None.
"""
ret, out, err = g.run(hostname, cmd, "root")
if ("no ssh connection" in err.lower() or
"tls handshake timeout" in err.lower()):
g.ssh_close_connection(hostname)
ret, out, err = g.run(hostname, cmd, "root")
msg = ("Failed to execute command '%s' on '%s' node. Got non-zero "
"return code '%s'. Err: %s" % (cmd, hostname, ret, err))
if int(ret) != 0:
g.log.error(msg)
if raise_on_error:
assert int(ret) == 0, msg
out = out.strip() if out else out
return out
def cmd_run(cmd, hostname, raise_on_error=True):
"""Glusto's command runner wrapper.
Args:
cmd (str): Shell command to run on the specified hostname.
hostname (str): hostname where Glusto should run specified command.
raise_on_error (bool): defines whether we should raise exception
in case command execution failed.
Returns:
str: Stripped shell command's stdout value if not None.
"""
ret, out, err = g.run(hostname, cmd, "root")
if ("no ssh connection" in err.lower()
or "tls handshake timeout" in err.lower()):
g.ssh_close_connection(hostname)
ret, out, err = g.run(hostname, cmd, "root")
msg = ("Failed to execute command '%s' on '%s' node. Got non-zero "
"return code '%s'. Err: %s" % (cmd, hostname, ret, err))
if int(ret) != 0:
g.log.error(msg)
if raise_on_error:
assert int(ret) == 0, msg
out = out.strip() if out else out
return out
def _node_reboot(self):
storage_hostname = (g.config["gluster_servers"]
[self.gluster_servers[0]]["storage"])
cmd = "sleep 3; /sbin/shutdown -r now 'Reboot triggered by Glusto'"
ret, out, err = g.run(storage_hostname, cmd)
self.addCleanup(self._wait_for_gluster_pod_to_be_ready)
if ret != 255:
err_msg = "failed to reboot host %s error: %s" % (
storage_hostname, err)
g.log.error(err_msg)
raise AssertionError(err_msg)
try:
g.ssh_close_connection(storage_hostname)
except Exception as e:
g.log.error("failed to close connection with host %s"
" with error: %s" % (storage_hostname, e))
raise
# added sleep as node will restart after 3 sec
time.sleep(3)
for w in Waiter(timeout=600, interval=10):
try:
if g.rpyc_get_connection(storage_hostname, user="******"):
g.rpyc_close_connection(storage_hostname, user="******")
break
except Exception as err:
g.log.info("exception while getting connection: '%s'" % err)
if w.expired:
error_msg = ("exceeded timeout 600 sec, node '%s' is "
"not reachable" % storage_hostname)
g.log.error(error_msg)
raise ExecutionError(error_msg)
# wait for the gluster pod to be in 'Running' state
self._wait_for_gluster_pod_to_be_ready()
# glusterd and gluster-blockd service should be up and running
service_names = ("glusterd", "gluster-blockd", "tcmu-runner")
for gluster_pod in self.gluster_pod_list:
for service in service_names:
g.log.info("gluster_pod - '%s' : gluster_service '%s'" % (
gluster_pod, service))
check_service_status_on_pod(
self.oc_node, gluster_pod, service, "running"
)
def node_reboot_by_command(node, timeout=600, wait_step=10):
"""Reboot node and wait to start for given timeout.
Args:
node (str) : Node which needs to be rebooted.
timeout (int) : Seconds to wait before node to be started.
wait_step (int): Interval in seconds to wait before checking
status of node again.
"""
cmd = "sleep 3; /sbin/shutdown -r now 'Reboot triggered by Glusto'"
ret, out, err = g.run(node, cmd)
if ret != 255:
err_msg = "failed to reboot host '%s' error %s" % (node, err)
g.log.error(err_msg)
raise AssertionError(err_msg)
try:
g.ssh_close_connection(node)
except Exception as e:
g.log.error("failed to close connection with host %s "
"with error: %s" % (node, e))
raise
# added sleep as node will restart after 3 sec
time.sleep(3)
for w in waiter.Waiter(timeout=timeout, interval=wait_step):
try:
if g.rpyc_get_connection(node, user="******"):
g.rpyc_close_connection(node, user="******")
return
except Exception as err:
g.log.info("exception while getting connection: '%s'" % err)
if w.expired:
error_msg = ("exceeded timeout %s sec, node '%s' is "
"not reachable" % (timeout, node))
g.log.error(error_msg)
raise exceptions.ExecutionError(error_msg)
