def test_container_exception_no_result_logs(m_sleep):
# If the job errored with no output but with logs,
# we should return error logs with the future exception.
mem_msg = ('Run used approximately 2 millicores '
'of its 256 millicore CPU limit')
failed_msg = 'Failed: The job container failed. Exit code 1'
logs = [{'id': 111, 'created_at': 'abc',
'message': mem_msg,
'level': 'info'},
{'id': 222,
'created_at': 'def',
'message': failed_msg,
'level': 'error'}]
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed',
run_outputs=[],
log_outputs=logs)
fut = ContainerFuture(1, 2, client=mock_client)
with pytest.raises(CivisJobFailure) as err:
fut.result()
expected_msg = (
"(From job 1 / run 2) " + '\n'.join([failed_msg, mem_msg, '']))
assert expected_msg == str(fut._exception.error_message)
assert str(err.value) == expected_msg
def test_container_exception_memory_error(m_sleep):
err_msg = ('Process ran out of its allowed 3000 MiB of '
'memory and was killed.')
logs = [{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10005,
'level': 'error',
'message': 'Failed'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10003,
'level': 'error',
'message': 'Error on job: Process ended with an '
'error, exiting: 137.'},
{'created_at': '2017-05-10T12:00:00.000Z',
'id': 10000,
'level': 'error',
'message': err_msg}]
mock_client = create_client_mock_for_container_tests(
1, 2, state='failed',
run_outputs=[],
log_outputs=logs)
fut = ContainerFuture(1, 2, client=mock_client)
with pytest.raises(MemoryError) as err:
fut.result()
assert str(err.value) == f"(From job 1 / run 2) {err_msg}"
def test_future_no_retry_failure():
# Verify that with no retries, job failures are raised as
# exceptions for the user
c = _setup_client_mock(failure_is_error=False)
fut = ContainerFuture(-10, 100, polling_interval=0.001, client=c)
with pytest.raises(CivisJobFailure):
fut.result()
def test_future_no_retry_error():
# Verify that with no retries, exceptions on job polling
# are raised to the user
c = _setup_client_mock(failure_is_error=True)
fut = ContainerFuture(-10, 100, polling_interval=0.001, client=c)
with pytest.raises(CivisAPIError):
fut.result()
def test_future_not_enough_retry_failure():
# Verify that if the job is still failing after all retries
# are exhausted, the job failure will be raised for the user.
c = _setup_client_mock(failure_is_error=False)
fut = ContainerFuture(-10, 100, max_n_retries=3, polling_interval=0.01,
client=c)
with pytest.raises(CivisJobFailure):
fut.result()
def test_future_not_enough_retry_error():
# Verify that if polling the run is still erroring after all retries
# are exhausted, the error will be raised for the user.
c = _setup_client_mock(failure_is_error=True)
fut = ContainerFuture(-10, 100, max_n_retries=3, polling_interval=0.01,
client=c)
with pytest.raises(CivisAPIError):
fut.result()
def test_future_retry_error():
# Verify that we can retry through job failures until it succeeds
c = _setup_client_mock(failure_is_error=True)
fut = ContainerFuture(-10,
100,
max_n_retries=10,
polling_interval=0.01,
client=c)
assert fut.result().state == 'succeeded'