def sshfunc(job, cmd):
"""Function to mock the throwing of exception for a test."""
if cmd[0] == "module avail" and job["resource"] == "resource1":
raise exceptions.SSHError(
"Err", ("", "bash: module: command not found", 0))
def test_status_except1(mock_ssh):
"""
Test if SSH Error is handled.
"""
job = {"user": "******", "jobid": ""}
mock_ssh.side_effect = exceptions.SSHError("OUT", "ERR")
with pytest.raises(exceptions.SSHError):
status(job)
def test_testhandler_except(mock_ssh):
"""
Test that the correct exception is raised when nothing can be detected.
"""
job = {"modules": "", "resource": "test-machine", "handler": ""}
mock_ssh.side_effect = exceptions.SSHError("SSH Error", "Error")
with pytest.raises(exceptions.HandlercheckError):
_testhandler(job)
def test_delete_except1(mock_ssh):
"""
Test if jobdelete exception is triggered based on output from scheduler.
"""
job = {"jobid": "12345", "replicates": "1"}
mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "", 0))
with pytest.raises(exceptions.JobdeleteError):
delete(job)
def test_submit_except2(mock_ssh):
"""
Check that jobsubmit exception is raised on generic SSH failure.
"""
job = {"destdir": "/path/to/destdir", "subfile": "submit.file"}
mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "err", 0))
mock_ssh.return_value = ("success", "error", 0)
with pytest.raises(exceptions.JobsubmitError):
submit(job)
def test_submit_except1(mock_ssh):
"""
Test if Queuemax exception is triggered based on output from scheduler.
"""
job = {"destdir": "/path/to/destdir", "subfile": "submit.file"}
mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "limit", 0))
mock_ssh.return_value = ("success", "Error", 0)
with pytest.raises(exceptions.QueuemaxError):
submit(job)
def test_testhandler_detection2(mock_ssh):
"""
Test that a handler can be detected. It is hard to specify exactly which
to go for due to dictionaries being unordered. Throw in a failure event.
"""
job = {"modules": "", "resource": "test-machine", "handler": ""}
mock_ssh.side_effect = [exceptions.SSHError("SSH Error", "Error"), None]
_testhandler(job)
assert job["handler"] in ["aprun", "mpirun"]
def test_submit_except6(mock_ssh):
"""
Check that the jobsubmit error is triggered based on certain output.
"""
job = {"destdir": "/path/to/destdir", "subfile": "submit.file"}
mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "budget", 0))
mock_ssh.return_value = ("success", "Error", 0)
with pytest.raises(exceptions.JobsubmitError):
submit(job)
def test_remotedelete_exceptiontest(mock_sendtossh):
"""
Check that the SSH exception is percolated properly.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
"destdir": "~/source/directory/path"
}
mock_sendtossh.side_effect = exceptions.SSHError("SSHError", "Error")
with pytest.raises(exceptions.RemotedeleteError):
remotedelete(job)
def test_stage_upstream_sshexcept(mock_ssh):
"""
Test if the SSH exception is raised if passed up from the SSH call.
"""
jobs = {
"jobone": {
"destdir": "/path/to/jobone12484",
"resource": "test-machine"
}
}
mock_ssh.side_effect = exceptions.SSHError("SSH Error", "output")
with pytest.raises(exceptions.SSHError):
stage_upstream(jobs)
def test_submit_except1(mock_ssh):
"""
Test if Queuemax exception is triggered based on output from scheduler.
"""
job = {
"destdir": "/path/to/destdir",
"subfile": "submit.file"
}
mock_ssh.side_effect = exceptions.SSHError(
"Error", ("out", "Batch job submission failed: Job violates accounting"
"/QOS policy (job submit limit, user's size and/or time "
"limits)", 0))
mock_ssh.return_value = ("success", "Error", 0)
with pytest.raises(exceptions.QueuemaxError):
submit(job)
def test_testapp_except(m_sendtossh):
"""
Test that modules to be loaded are appended into the executable query
string.
"""
jobs = {
"jobone": {
"resource": "res1",
"executable": "exec1",
"modules": "intel, amber",
"nochecks": "false"
}
}
m_sendtossh.side_effect = ex.SSHError("Error", ("stdout", "stderr", 1))
with pytest.raises(ex.ExecutableError):
checkapp(jobs)
def test_remotecopy_exceptiontest(mock_sendtossh):
"""
Check that the SSH exception is percolated properly.
"""
job = {
"port": "22",
"user": "******",
"host": "massive-machine",
}
src = "~/source/directory/path"
dst = "~/destination/directory/path"
mock_sendtossh.side_effect = exceptions.SSHError("SSHError", "Error")
with pytest.raises(exceptions.RemotecopyError):
remotecopy(job, src, dst)
def test_testconnections_sshexcept(mock_sendtossh):
"""
Test to see that if the underlying SSH call fails, the resulting
SSHError is passed up the chain. This is important!
"""
jobs = {
"LongbowJob1": {
"resource": "resource1"
},
"LongbowJob2": {
"resource": "resource2"
},
"LongbowJob3": {
"resource": "resource1"
}
}
mock_sendtossh.side_effect = exceptions.SSHError("SSH Error", "output")
with pytest.raises(exceptions.SSHError):
checkconnections(jobs)
def sendtossh(job, args):
"""Construct SSH commands and hand them off to the shell.
This method constructs a string containing commands to be executed via SSH.
This string is then handed off to the sendtoshell() method for execution.
Required arguments are:
job (dictionary) - A single job dictionary, this is often simply passed in
as a subset of the main jobs dictionary.
args (list) - A list containing commands to be sent to SSH, multiple
commands should each be an entry in the list.
Return parameters are:
shellout (tuple of strings) - Contains the three strings returned from the
sendtoshell() method. These are standard
output, standard error and the exit code.
"""
# basic ssh command.
cmd = ["ssh", "-p " + job["port"], job["user"] + "@" + job["host"]]
# Source the /etc/profile on machines where problems have been detected
# with the environment.
if job["env-fix"] == "true":
cmd.append("source /etc/profile;")
# add the commands to be sent to ssh.
cmd.extend(args)
i = 0
# This loop is essentially so we can do 3 retries on commands that fail,
# this is to catch when things go wrong over SSH like dropped connections,
# issues with latency etc.
while i is not 3:
# Send to ssh.
shellout = sendtoshell(cmd)
errorstate = shellout[2]
# If no error exit loop, if errorcode is not 0 raise exception unless
# code is 255
if errorstate is 0:
break
elif errorstate is 255:
i = i + 1
else:
raise exceptions.SSHError(
"SSH failed, make sure a normal terminal can connect to SSH "
"to be sure there are no connection issues.", shellout)
# If number of retries hits 3 then give up.
if i is 3:
raise exceptions.SSHError(
"SSH failed, make sure a normal terminal can connect to SSH "
"to be sure there are no connection issues.", shellout)
LOG.debug("Retry SSH after 10 second wait.")
# Wait 10 seconds to see if problem goes away before trying again.
time.sleep(10)
return shellout
