def test_future_no_retry_failure(): # Verify that with no retries, job failures are raised as # exceptions for the user c = _setup_client_mock(failure_is_error=False) fut = ContainerFuture(-10, 100, polling_interval=0.001, client=c) with pytest.raises(CivisJobFailure): fut.result()
def test_future_no_retry_error(): # Verify that with no retries, exceptions on job polling # are raised to the user c = _setup_client_mock(failure_is_error=True) fut = ContainerFuture(-10, 100, polling_interval=0.001, client=c) with pytest.raises(CivisAPIError): fut.result()
def test_container_exception_no_result_logs(m_sleep): # If the job errored with no output but with logs, # we should return error logs with the future exception. mem_msg = ('Run used approximately 2 millicores ' 'of its 256 millicore CPU limit') failed_msg = 'Failed: The job container failed. Exit code 1' logs = [{'id': 111, 'created_at': 'abc', 'message': mem_msg, 'level': 'info'}, {'id': 222, 'created_at': 'def', 'message': failed_msg, 'level': 'error'}] mock_client = create_client_mock_for_container_tests( 1, 2, state='failed', run_outputs=[], log_outputs=logs) fut = ContainerFuture(1, 2, client=mock_client) with pytest.raises(CivisJobFailure) as err: fut.result() expected_msg = ( "(From job 1 / run 2) " + '\n'.join([failed_msg, mem_msg, ''])) assert expected_msg == str(fut._exception.error_message) assert str(err.value) == expected_msg
def test_container_exception_memory_error(m_sleep): err_msg = ('Process ran out of its allowed 3000 MiB of ' 'memory and was killed.') logs = [{'created_at': '2017-05-10T12:00:00.000Z', 'id': 10005, 'level': 'error', 'message': 'Failed'}, {'created_at': '2017-05-10T12:00:00.000Z', 'id': 10003, 'level': 'error', 'message': 'Error on job: Process ended with an ' 'error, exiting: 137.'}, {'created_at': '2017-05-10T12:00:00.000Z', 'id': 10000, 'level': 'error', 'message': err_msg}] mock_client = create_client_mock_for_container_tests( 1, 2, state='failed', run_outputs=[], log_outputs=logs) fut = ContainerFuture(1, 2, client=mock_client) with pytest.raises(MemoryError) as err: fut.result() assert str(err.value) == f"(From job 1 / run 2) {err_msg}"
def test_future_not_enough_retry_error(): # Verify that if polling the run is still erroring after all retries # are exhausted, the error will be raised for the user. c = _setup_client_mock(failure_is_error=True) fut = ContainerFuture(-10, 100, max_n_retries=3, polling_interval=0.01, client=c) with pytest.raises(CivisAPIError): fut.result()
def test_future_not_enough_retry_failure(): # Verify that if the job is still failing after all retries # are exhausted, the job failure will be raised for the user. c = _setup_client_mock(failure_is_error=False) fut = ContainerFuture(-10, 100, max_n_retries=3, polling_interval=0.01, client=c) with pytest.raises(CivisJobFailure): fut.result()
def test_result_callback_no_get(mock_civis): # Test that the completed callback happens even if we don't call `get` callback = mock.MagicMock() mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam') fut = ContainerFuture(1, 2, client=mock.MagicMock()) fut.set_result(Response({'state': 'success'})) civis.parallel._CivisBackendResult(fut, callback) assert callback.call_count == 1
def test_future_retry_error(): # Verify that we can retry through job failures until it succeeds c = _setup_client_mock(failure_is_error=True) fut = ContainerFuture(-10, 100, max_n_retries=10, polling_interval=0.01, client=c) assert fut.result().state == 'succeeded'
def test_result_success(mock_civis): # Test that we can get a result back from a succeeded job. callback = mock.MagicMock() mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam') fut = ContainerFuture(1, 2, client=mock.MagicMock()) fut.set_result(Response({'state': 'success'})) res = civis.parallel._CivisBackendResult(fut, callback) assert res.get() == 'spam' assert callback.call_count == 1
def test_result_exception(mock_civis): # An error in the job should be raised by the result callback = mock.MagicMock() exc = ZeroDivisionError() mock_civis.io.civis_to_file.side_effect = make_to_file_mock(exc) fut = ContainerFuture(1, 2, client=mock.MagicMock()) fut._set_api_exception(Response({'state': 'failed'})) res = civis.parallel._CivisBackendResult(fut, callback) with pytest.raises(ZeroDivisionError): res.get() assert callback.call_count == 0
def test_result_eventual_success(mock_civis): # Test that we can get a result back from a succeeded job, # even if we need to retry a few times to succeed with the download. callback = mock.MagicMock() exc = requests.ConnectionError() se = make_to_file_mock('spam', max_n_err=2, exc=exc) mock_civis.io.civis_to_file.side_effect = se fut = ContainerFuture(1, 2, client=mock.MagicMock()) fut.set_result(Response({'state': 'success'})) res = civis.parallel._CivisBackendResult(fut, callback) assert res.get() == 'spam' assert callback.call_count == 1
def test_result_eventual_failure(mock_civis): # We will retry a connection error up to 5 times. Make sure # that we will get an error if it persists forever. callback = mock.MagicMock() exc = requests.ConnectionError() se = make_to_file_mock('spam', max_n_err=10, exc=exc) mock_civis.io.civis_to_file.side_effect = se fut = ContainerFuture(1, 2, client=mock.MagicMock()) fut.set_result(Response({'state': 'success'})) res = civis.parallel._CivisBackendResult(fut, callback) with pytest.raises(requests.ConnectionError): res.get() assert callback.call_count == 0
def test_result_callback_exception(mock_civis): # An error in the result retrieval should be raised by .get callback = mock.MagicMock() exc = ZeroDivisionError() mock_civis.io.civis_to_file.side_effect = exc fut = ContainerFuture(1, 2, client=mock.MagicMock()) # We're simulating a job which succeeded but generated an # exception when we try to download the outputs. fut._set_api_exception(Response({'state': 'succeeded'})) res = civis.parallel._CivisBackendResult(fut, callback) with pytest.raises(ZeroDivisionError): res.get() assert callback.call_count == 0
def test_result_running_and_cancel_requested(mock_civis): # When scripts request cancellation, they remain in a running # state. Make sure these are treated as cancelled runs. response = Response({'is_cancel_requested': True, 'state': 'running'}) mock_client = create_client_mock_for_container_tests( 1, 2, state='running', run_outputs=mock.MagicMock()) mock_client.scripts.post_cancel.return_value = response fut = ContainerFuture(1, 2, client=mock_client) callback = mock.MagicMock() # When a _CivisBackendResult created by the Civis joblib backend completes # successfully, a callback is executed. When cancelled, this callback # shouldn't be run civis.parallel._CivisBackendResult(fut, callback) fut.cancel() assert callback.call_count == 0
def test_result_exception_no_result(m_sleep): # If the job errored but didn't write an output, we should get # a generic TransportableException back. callback = mock.MagicMock() mock_client = create_client_mock_for_container_tests(1, 2, state='failed', run_outputs=[]) fut = ContainerFuture(1, 2, client=mock_client) res = civis.parallel._CivisBackendResult(fut, callback) fut._set_api_exception(CivisJobFailure(Response({'state': 'failed'}))) with pytest.raises(TransportableException) as exc: res.get() assert "{'state': 'failed'}" in str(exc.value) assert callback.call_count == 0
def test_result_exception_no_result(): # If the job errored but didn't write an output, we should get # a generic TransportableException back. callback = mock.MagicMock() # Passing the client mock as an argument instead of globally # patching the client tests that the _CivisBackendResult # uses the client object on the input CivisFuture. mock_client = mock.MagicMock().APIClient() mock_client.scripts.list_containers_runs_outputs.return_value = [] fut = ContainerFuture(1, 2, client=mock_client) fut._set_api_exception(Response({'state': 'failed'})) res = civis.parallel._CivisBackendResult(fut, callback) with pytest.raises(TransportableException) as exc: res.get() assert "{'state': 'failed'}" in str(exc.value) assert callback.call_count == 0
def test_result_callback_no_get(mock_civis): # Test that the completed callback happens even if we don't call `get` callback = mock.MagicMock() mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam') mock_client = create_client_mock_for_container_tests( 1, 2, state='success', run_outputs=mock.MagicMock()) fut = ContainerFuture(1, 2, client=mock_client) civis.parallel._CivisBackendResult(fut, callback) assert callback.call_count == 1
def test_container_future_job_id_run_id(): job_id, run_id = 123, 456 result = ContainerFuture( job_id=job_id, run_id=run_id, client=create_client_mock_for_container_tests(), ) assert result.job_id == job_id assert result.run_id == run_id
def test_result_success(mock_civis): # Test that we can get a result back from a succeeded job. callback = mock.MagicMock() mock_civis.io.civis_to_file.side_effect = make_to_file_mock('spam') mock_client = create_client_mock_for_container_tests( 1, 2, state='success', run_outputs=mock.MagicMock()) fut = ContainerFuture(1, 2, client=mock_client) res = civis.parallel._CivisBackendResult(fut, callback) assert res.get() == 'spam' assert callback.call_count == 1
def test_cancel_finished_job(): # If we try to cancel a completed job, we get a 404 error. # That shouldn't be sent to the user. # Set up a mock client which will give an exception when # you try to cancel any job. c = _setup_client_mock() err_resp = response.Response({ 'status_code': 404, 'error': 'not_found', 'errorDescription': 'The requested resource could not be found.', 'content': True}) err_resp.json = lambda: err_resp.json_data c.scripts.post_cancel.side_effect = CivisAPIError(err_resp) c.scripts.post_containers_runs.return_value.state = 'running' fut = ContainerFuture(-10, 100, polling_interval=1, client=c, poll_on_creation=False) assert not fut.done() assert fut.cancel() is False
def test_result_exception(m_sleep, mock_civis): # An error in the job should be raised by the result callback = mock.MagicMock() exc = ZeroDivisionError() mock_civis.io.civis_to_file.side_effect = make_to_file_mock(exc) mock_client = create_client_mock_for_container_tests( 1, 2, state='failed', run_outputs=mock.MagicMock()) fut = ContainerFuture(1, 2, client=mock_client) res = civis.parallel._CivisBackendResult(fut, callback) with pytest.raises(ZeroDivisionError): res.get() assert callback.call_count == 0