def test_future_no_retry_failure():
# Verify that with no retries, job failures are raised as
# exceptions for the user
c = _setup_client_mock(failure_is_error=False)
fut = ContainerFuture(-10, 100, polling_interval=0.001, client=c)
with pytest.raises(CivisJobFailure):
fut.result()
def test_future_no_retry_error():
# Verify that with no retries, exceptions on job polling
# are raised to the user
c = _setup_client_mock(failure_is_error=True)
fut = ContainerFuture(-10, 100, polling_interval=0.001, client=c)
with pytest.raises(CivisAPIError):
fut.result()
def test_container_exception_no_result_logs(m_sleep):
# If the job errored with no output but with logs,
# we should return error logs with the future exception.
mem_msg = ('Run used approximately 2 millicores '
'of its 256 millicore CPU limit')
failed_msg = 'Failed: The job container failed. Exit code 1'
logs = [{'id': 111, 'created_at': 'abc',
'message': mem_msg,
'level': 'info'},
{'id': 222,
'created_at': 'def',
'message': failed_msg,
'level': 'error'}]
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed',
run_outputs=[],
log_outputs=logs)
fut = ContainerFuture(1, 2, client=mock_client)
with pytest.raises(CivisJobFailure) as err:
fut.result()
expected_msg = (
"(From job 1 / run 2) " + '\n'.join([failed_msg, mem_msg, '']))
assert expected_msg == str(fut._exception.error_message)
assert str(err.value) == expected_msg
def test_container_exception_memory_error(m_sleep):
err_msg = ('Process ran out of its allowed 3000 MiB of '
'memory and was killed.')
logs = [{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10005,
'level': 'error',
'message': 'Failed'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10003,
'level': 'error',
'message': 'Error on job: Process ended with an '
'error, exiting: 137.'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10000,
'level': 'error',
'message': err_msg}]
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed',
run_outputs=[],
log_outputs=logs)
fut = ContainerFuture(1, 2, client=mock_client)
with pytest.raises(MemoryError) as err:
fut.result()
assert str(err.value) == f"(From job 1 / run 2) {err_msg}"
def test_future_not_enough_retry_error():
# Verify that if polling the run is still erroring after all retries
# are exhausted, the error will be raised for the user.
c = _setup_client_mock(failure_is_error=True)
fut = ContainerFuture(-10, 100, max_n_retries=3, polling_interval=0.01,
client=c)
with pytest.raises(CivisAPIError):
fut.result()
def test_future_not_enough_retry_failure():
# Verify that if the job is still failing after all retries
# are exhausted, the job failure will be raised for the user.
c = _setup_client_mock(failure_is_error=False)
fut = ContainerFuture(-10, 100, max_n_retries=3, polling_interval=0.01,
client=c)
with pytest.raises(CivisJobFailure):
fut.result()
def test_result_callback_no_get(mock_civis):
# Test that the completed callback happens even if we don't call `get`
callback = mock.MagicMock()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam')
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
civis.parallel._CivisBackendResult(fut, callback)
assert callback.call_count == 1
def test_future_retry_error():
# Verify that we can retry through job failures until it succeeds
c = _setup_client_mock(failure_is_error=True)
fut = ContainerFuture(-10,
100,
max_n_retries=10,
polling_interval=0.01,
client=c)
assert fut.result().state == 'succeeded'
def test_result_success(mock_civis):
# Test that we can get a result back from a succeeded job.
callback = mock.MagicMock()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam')
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
res = civis.parallel._CivisBackendResult(fut, callback)
assert res.get() == 'spam'
assert callback.call_count == 1
def test_result_exception(mock_civis):
# An error in the job should be raised by the result
callback = mock.MagicMock()
exc = ZeroDivisionError()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock(exc)
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut._set_api_exception(Response({'state': 'failed'}))
res = civis.parallel._CivisBackendResult(fut, callback)
with pytest.raises(ZeroDivisionError):
res.get()
assert callback.call_count == 0
def test_result_eventual_success(mock_civis):
# Test that we can get a result back from a succeeded job,
# even if we need to retry a few times to succeed with the download.
callback = mock.MagicMock()
exc = requests.ConnectionError()
se = make_to_file_mock('spam', max_n_err=2, exc=exc)
mock_civis.io.civis_to_file.side_effect = se
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
res = civis.parallel._CivisBackendResult(fut, callback)
assert res.get() == 'spam'
assert callback.call_count == 1
def test_result_eventual_failure(mock_civis):
# We will retry a connection error up to 5 times. Make sure
# that we will get an error if it persists forever.
callback = mock.MagicMock()
exc = requests.ConnectionError()
se = make_to_file_mock('spam', max_n_err=10, exc=exc)
mock_civis.io.civis_to_file.side_effect = se
fut = ContainerFuture(1, 2, client=mock.MagicMock())
fut.set_result(Response({'state': 'success'}))
res = civis.parallel._CivisBackendResult(fut, callback)
with pytest.raises(requests.ConnectionError):
res.get()
assert callback.call_count == 0
def test_result_callback_exception(mock_civis):
# An error in the result retrieval should be raised by .get
callback = mock.MagicMock()
exc = ZeroDivisionError()
mock_civis.io.civis_to_file.side_effect = exc
fut = ContainerFuture(1, 2, client=mock.MagicMock())
# We're simulating a job which succeeded but generated an
# exception when we try to download the outputs.
fut._set_api_exception(Response({'state': 'succeeded'}))
res = civis.parallel._CivisBackendResult(fut, callback)
with pytest.raises(ZeroDivisionError):
res.get()
assert callback.call_count == 0
def test_result_running_and_cancel_requested(mock_civis):
# When scripts request cancellation, they remain in a running
# state. Make sure these are treated as cancelled runs.
response = Response({'is_cancel_requested': True, 'state': 'running'})
mock_client = create_client_mock_for_container_tests(
1, 2, state='running', run_outputs=mock.MagicMock())
mock_client.scripts.post_cancel.return_value = response
fut = ContainerFuture(1, 2, client=mock_client)
callback = mock.MagicMock()
# When a _CivisBackendResult created by the Civis joblib backend completes
# successfully, a callback is executed. When cancelled, this callback
# shouldn't be run
civis.parallel._CivisBackendResult(fut, callback)
fut.cancel()
assert callback.call_count == 0
def test_result_exception_no_result(m_sleep):
# If the job errored but didn't write an output, we should get
# a generic TransportableException back.
callback = mock.MagicMock()
mock_client = create_client_mock_for_container_tests(1,
2,
state='failed',
run_outputs=[])
fut = ContainerFuture(1, 2, client=mock_client)
res = civis.parallel._CivisBackendResult(fut, callback)
fut._set_api_exception(CivisJobFailure(Response({'state': 'failed'})))
with pytest.raises(TransportableException) as exc:
res.get()
assert "{'state': 'failed'}" in str(exc.value)
assert callback.call_count == 0
def test_result_exception_no_result():
# If the job errored but didn't write an output, we should get
# a generic TransportableException back.
callback = mock.MagicMock()
# Passing the client mock as an argument instead of globally
# patching the client tests that the _CivisBackendResult
# uses the client object on the input CivisFuture.
mock_client = mock.MagicMock().APIClient()
mock_client.scripts.list_containers_runs_outputs.return_value = []
fut = ContainerFuture(1, 2, client=mock_client)
fut._set_api_exception(Response({'state': 'failed'}))
res = civis.parallel._CivisBackendResult(fut, callback)
with pytest.raises(TransportableException) as exc:
res.get()
assert "{'state': 'failed'}" in str(exc.value)
assert callback.call_count == 0
def test_result_callback_no_get(mock_civis):
# Test that the completed callback happens even if we don't call `get`
callback = mock.MagicMock()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam')
mock_client = create_client_mock_for_container_tests(
1, 2, state='success', run_outputs=mock.MagicMock())
fut = ContainerFuture(1, 2, client=mock_client)
civis.parallel._CivisBackendResult(fut, callback)
assert callback.call_count == 1
def test_container_future_job_id_run_id():
job_id, run_id = 123, 456
result = ContainerFuture(
job_id=job_id,
run_id=run_id,
client=create_client_mock_for_container_tests(),
)
assert result.job_id == job_id
assert result.run_id == run_id
def test_result_success(mock_civis):
# Test that we can get a result back from a succeeded job.
callback = mock.MagicMock()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam')
mock_client = create_client_mock_for_container_tests(
1, 2, state='success', run_outputs=mock.MagicMock())
fut = ContainerFuture(1, 2, client=mock_client)
res = civis.parallel._CivisBackendResult(fut, callback)
assert res.get() == 'spam'
assert callback.call_count == 1
def test_cancel_finished_job():
# If we try to cancel a completed job, we get a 404 error.
# That shouldn't be sent to the user.
# Set up a mock client which will give an exception when
# you try to cancel any job.
c = _setup_client_mock()
err_resp = response.Response({
'status_code': 404,
'error': 'not_found',
'errorDescription': 'The requested resource could not be found.',
'content': True})
err_resp.json = lambda: err_resp.json_data
c.scripts.post_cancel.side_effect = CivisAPIError(err_resp)
c.scripts.post_containers_runs.return_value.state = 'running'
fut = ContainerFuture(-10, 100, polling_interval=1, client=c,
poll_on_creation=False)
assert not fut.done()
assert fut.cancel() is False
def test_result_exception(m_sleep, mock_civis):
# An error in the job should be raised by the result
callback = mock.MagicMock()
exc = ZeroDivisionError()
mock_civis.io.civis_to_file.side_effect = make_to_file_mock(exc)
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed', run_outputs=mock.MagicMock())
fut = ContainerFuture(1, 2, client=mock_client)
res = civis.parallel._CivisBackendResult(fut, callback)
with pytest.raises(ZeroDivisionError):
res.get()
assert callback.call_count == 0
