def sshfunc(job, cmd): """Function to mock the throwing of exception for a test.""" if cmd[0] == "module avail" and job["resource"] == "resource1": raise exceptions.SSHError( "Err", ("", "bash: module: command not found", 0))
def test_status_except1(mock_ssh): """ Test if SSH Error is handled. """ job = {"user": "******", "jobid": ""} mock_ssh.side_effect = exceptions.SSHError("OUT", "ERR") with pytest.raises(exceptions.SSHError): status(job)
def test_testhandler_except(mock_ssh): """ Test that the correct exception is raised when nothing can be detected. """ job = {"modules": "", "resource": "test-machine", "handler": ""} mock_ssh.side_effect = exceptions.SSHError("SSH Error", "Error") with pytest.raises(exceptions.HandlercheckError): _testhandler(job)
def test_delete_except1(mock_ssh): """ Test if jobdelete exception is triggered based on output from scheduler. """ job = {"jobid": "12345", "replicates": "1"} mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "", 0)) with pytest.raises(exceptions.JobdeleteError): delete(job)
def test_submit_except2(mock_ssh): """ Check that jobsubmit exception is raised on generic SSH failure. """ job = {"destdir": "/path/to/destdir", "subfile": "submit.file"} mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "err", 0)) mock_ssh.return_value = ("success", "error", 0) with pytest.raises(exceptions.JobsubmitError): submit(job)
def test_submit_except1(mock_ssh): """ Test if Queuemax exception is triggered based on output from scheduler. """ job = {"destdir": "/path/to/destdir", "subfile": "submit.file"} mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "limit", 0)) mock_ssh.return_value = ("success", "Error", 0) with pytest.raises(exceptions.QueuemaxError): submit(job)
def test_testhandler_detection2(mock_ssh): """ Test that a handler can be detected. It is hard to specify exactly which to go for due to dictionaries being unordered. Throw in a failure event. """ job = {"modules": "", "resource": "test-machine", "handler": ""} mock_ssh.side_effect = [exceptions.SSHError("SSH Error", "Error"), None] _testhandler(job) assert job["handler"] in ["aprun", "mpirun"]
def test_submit_except6(mock_ssh): """ Check that the jobsubmit error is triggered based on certain output. """ job = {"destdir": "/path/to/destdir", "subfile": "submit.file"} mock_ssh.side_effect = exceptions.SSHError("Error", ("out", "budget", 0)) mock_ssh.return_value = ("success", "Error", 0) with pytest.raises(exceptions.JobsubmitError): submit(job)
def test_remotedelete_exceptiontest(mock_sendtossh): """ Check that the SSH exception is percolated properly. """ job = { "port": "22", "user": "******", "host": "massive-machine", "destdir": "~/source/directory/path" } mock_sendtossh.side_effect = exceptions.SSHError("SSHError", "Error") with pytest.raises(exceptions.RemotedeleteError): remotedelete(job)
def test_stage_upstream_sshexcept(mock_ssh): """ Test if the SSH exception is raised if passed up from the SSH call. """ jobs = { "jobone": { "destdir": "/path/to/jobone12484", "resource": "test-machine" } } mock_ssh.side_effect = exceptions.SSHError("SSH Error", "output") with pytest.raises(exceptions.SSHError): stage_upstream(jobs)
def test_submit_except1(mock_ssh): """ Test if Queuemax exception is triggered based on output from scheduler. """ job = { "destdir": "/path/to/destdir", "subfile": "submit.file" } mock_ssh.side_effect = exceptions.SSHError( "Error", ("out", "Batch job submission failed: Job violates accounting" "/QOS policy (job submit limit, user's size and/or time " "limits)", 0)) mock_ssh.return_value = ("success", "Error", 0) with pytest.raises(exceptions.QueuemaxError): submit(job)
def test_testapp_except(m_sendtossh): """ Test that modules to be loaded are appended into the executable query string. """ jobs = { "jobone": { "resource": "res1", "executable": "exec1", "modules": "intel, amber", "nochecks": "false" } } m_sendtossh.side_effect = ex.SSHError("Error", ("stdout", "stderr", 1)) with pytest.raises(ex.ExecutableError): checkapp(jobs)
def test_remotecopy_exceptiontest(mock_sendtossh): """ Check that the SSH exception is percolated properly. """ job = { "port": "22", "user": "******", "host": "massive-machine", } src = "~/source/directory/path" dst = "~/destination/directory/path" mock_sendtossh.side_effect = exceptions.SSHError("SSHError", "Error") with pytest.raises(exceptions.RemotecopyError): remotecopy(job, src, dst)
def test_testconnections_sshexcept(mock_sendtossh): """ Test to see that if the underlying SSH call fails, the resulting SSHError is passed up the chain. This is important! """ jobs = { "LongbowJob1": { "resource": "resource1" }, "LongbowJob2": { "resource": "resource2" }, "LongbowJob3": { "resource": "resource1" } } mock_sendtossh.side_effect = exceptions.SSHError("SSH Error", "output") with pytest.raises(exceptions.SSHError): checkconnections(jobs)
def sendtossh(job, args): """Construct SSH commands and hand them off to the shell. This method constructs a string containing commands to be executed via SSH. This string is then handed off to the sendtoshell() method for execution. Required arguments are: job (dictionary) - A single job dictionary, this is often simply passed in as a subset of the main jobs dictionary. args (list) - A list containing commands to be sent to SSH, multiple commands should each be an entry in the list. Return parameters are: shellout (tuple of strings) - Contains the three strings returned from the sendtoshell() method. These are standard output, standard error and the exit code. """ # basic ssh command. cmd = ["ssh", "-p " + job["port"], job["user"] + "@" + job["host"]] # Source the /etc/profile on machines where problems have been detected # with the environment. if job["env-fix"] == "true": cmd.append("source /etc/profile;") # add the commands to be sent to ssh. cmd.extend(args) i = 0 # This loop is essentially so we can do 3 retries on commands that fail, # this is to catch when things go wrong over SSH like dropped connections, # issues with latency etc. while i is not 3: # Send to ssh. shellout = sendtoshell(cmd) errorstate = shellout[2] # If no error exit loop, if errorcode is not 0 raise exception unless # code is 255 if errorstate is 0: break elif errorstate is 255: i = i + 1 else: raise exceptions.SSHError( "SSH failed, make sure a normal terminal can connect to SSH " "to be sure there are no connection issues.", shellout) # If number of retries hits 3 then give up. if i is 3: raise exceptions.SSHError( "SSH failed, make sure a normal terminal can connect to SSH " "to be sure there are no connection issues.", shellout) LOG.debug("Retry SSH after 10 second wait.") # Wait 10 seconds to see if problem goes away before trying again. time.sleep(10) return shellout