def test_template_submit(mock_file, mock_result, mock_pool):
# Verify that creating child jobs from a template looks like we expect
file_id = 17
mock_client = mock.Mock()
mock_file.return_value = file_id
factory = civis.parallel.make_backend_template_factory(
from_template_id=1234, client=mock_client)
n_calls = 3
register_parallel_backend('civis', factory)
with parallel_backend('civis'):
parallel = Parallel(n_jobs=5, pre_dispatch='n_jobs')
parallel(delayed(sqrt)(i**2) for i in range(n_calls))
assert mock_file.call_count == 3, "Upload 3 functions to run"
assert mock_pool().submit.call_count == n_calls, "Run 3 functions"
for this_call in mock_pool().submit.call_args_list:
assert this_call == mock.call(JOBLIB_FUNC_FILE_ID=file_id)
assert mock_result.call_count == 3, "Create 3 results"
def _test_retries_helper(num_failures, max_submit_retries, should_fail,
from_template_id, mock_file_to_civis, mock_result_cls,
mock_custom_exec_cls, mock_executor_cls):
mock_file_to_civis.return_value = 0
mock_result_cls.return_value.get.return_value = [123]
# A function to raise fake API errors the first num_failures times it is
# called.
counter = {'n_failed': 0}
def mock_submit(fn='', *args, **kwargs):
if counter['n_failed'] < num_failures:
counter['n_failed'] += 1
raise CivisAPIError(mock.MagicMock())
else:
return mock.MagicMock(spec=ContainerFuture)
mock_custom_exec_cls.return_value.submit.side_effect = mock_submit
mock_executor_cls.return_value.submit.side_effect = mock_submit
if from_template_id:
factory = civis.parallel.make_backend_template_factory(
from_template_id=from_template_id,
max_submit_retries=max_submit_retries,
client=create_client_mock())
else:
factory = civis.parallel.make_backend_factory(
max_submit_retries=max_submit_retries, client=create_client_mock())
register_parallel_backend('civis', factory)
with parallel_backend('civis'):
# NB: joblib >v0.11 relies on callbacks from the result object to
# decide when it's done consuming inputs. We've mocked the result
# object here, so Parallel must be called either with n_jobs=1 or
# pre_dispatch='all' to consume the inputs all at once.
parallel = Parallel(n_jobs=1, pre_dispatch='n_jobs')
if should_fail:
with pytest.raises(civis.parallel.JobSubmissionError):
parallel(delayed(sqrt)(i**2) for i in range(3))
else:
parallel(delayed(sqrt)(i**2) for i in range(3))
def _test_retries_helper(num_failures, max_submit_retries,
should_fail, from_template_id,
mock_file_to_civis, mock_result_cls,
mock_custom_exec_cls, mock_executor_cls):
mock_file_to_civis.return_value = 0
mock_result_cls.get.return_value = 123
# A function to raise fake API errors the first num_failures times it is
# called.
counter = {'n_failed': 0}
def mock_submit(fn='', *args, **kwargs):
if counter['n_failed'] < num_failures:
counter['n_failed'] += 1
raise CivisAPIError(mock.MagicMock())
else:
return mock.MagicMock(spec=ContainerFuture)
mock_custom_exec_cls.return_value.submit.side_effect = mock_submit
mock_executor_cls.return_value.submit.side_effect = mock_submit
if from_template_id:
factory = civis.parallel.make_backend_template_factory(
from_template_id=from_template_id,
max_submit_retries=max_submit_retries,
client=mock.Mock())
else:
factory = civis.parallel.make_backend_factory(
max_submit_retries=max_submit_retries, client=mock.Mock())
register_parallel_backend('civis', factory)
with parallel_backend('civis'):
parallel = Parallel(n_jobs=5, pre_dispatch='n_jobs')
if should_fail:
with pytest.raises(civis.parallel.JobSubmissionError):
parallel(delayed(sqrt)(i ** 2) for i in range(3))
else:
parallel(delayed(sqrt)(i ** 2) for i in range(3))
def test_template_submit(mock_file, mock_result, mock_pool):
# Verify that creating child jobs from a template looks like we expect
file_id = 17
mock_client = create_client_mock()
mock_file.return_value = file_id
factory = civis.parallel.make_backend_template_factory(
from_template_id=1234, client=mock_client)
n_calls = 3
register_parallel_backend('civis', factory)
with parallel_backend('civis'):
# NB: joblib >v0.11 relies on callbacks from the result object to
# decide when it's done consuming inputs. We've mocked the result
# object here, so Parallel must be called either with n_jobs=1 or
# pre_dispatch='all' to consume the inputs all at once.
parallel = Parallel(n_jobs=1, pre_dispatch='n_jobs')
parallel(delayed(sqrt)(i**2) for i in range(n_calls))
assert mock_file.call_count == 3, "Upload 3 functions to run"
assert mock_pool().submit.call_count == n_calls, "Run 3 functions"
for this_call in mock_pool().submit.call_args_list:
assert this_call == mock.call(JOBLIB_FUNC_FILE_ID=file_id)
assert mock_result.call_count == 3, "Create 3 results"
