def test_transient_result(self):
ft = FutureTask(guid=guid, tuid=tuid, client=self.client)
self.mock_response(responses.GET, {}, status=404)
with pytest.raises(TransientResultError):
ft.get_result(wait=False)
def test_get_result(self):
ft = FutureTask(guid=guid, tuid=tuid, client=self.client)
self.mock_response(responses.GET, {}, status=404)
with pytest.raises(TimeoutError):
ft.get_result(wait=True, timeout=1)
def test_from_guid_tuid(self):
ft = FutureTask(guid, tuid)
ft2 = FutureTask(guid, tuid)
self.assertEqual(ft, ft2)
self.assertIsInstance(ft, FutureTask)
self.assertIsInstance(ft, object)
def test_from_guid_tuid(self):
ft = FutureTask(guid, tuid)
ft2 = FutureTask(guid, tuid)
assert ft == ft2
assert isinstance(ft, FutureTask)
assert isinstance(ft, object)
def __init__(self,
guid,
tuid=None,
client=None,
upload_id=None,
result_attrs=None):
FutureTask.__init__(self, guid, tuid, client)
self._upload_id = upload_id
self._task_result = result_attrs
def test_getattr_access(self):
ft = FutureTask(guid=guid, tuid=tuid)
assert ft.guid == guid
assert ft.tuid == tuid
with pytest.raises(AttributeError):
ft.nonexistent
def test_ready(self):
ft = FutureTask(guid=guid, tuid=tuid, client=self.client)
self.mock_response(responses.GET, {}, status=404)
self.mock_response(responses.GET, {'id': tuid, 'result_type': 'json'})
self.assertFalse(ft.ready)
self.assertTrue(ft.ready)
def test_getattr_access(self):
ft = FutureTask(guid=guid, tuid=tuid)
self.assertEqual(ft.guid, guid)
self.assertEqual(ft.tuid, tuid)
with self.assertRaises(AttributeError):
ft.nonexistent
def test_ready(self):
ft = FutureTask(guid=guid, tuid=tuid, client=self.client)
self.mock_response(responses.GET, {}, status=404)
self.mock_response(responses.GET, {"id": tuid, "result_type": "json"})
assert not ft.ready
assert ft.ready
def test_as_completed(self):
tasks = [
FutureTask("group_id", str(n), client=self.client)
for n in range(5)
]
response1 = [{"id": str(n), "result_type": "json"} for n in range(3)]
response2 = [{
"id": str(n),
"result_type": "json"
} for n in range(3, 5)]
self.mock_response(
responses.GET,
{
"id": "foo",
"queue": {
"pending": 3,
"successes": 0,
"failures": 0
},
"status": "running",
},
)
self.mock_response(
responses.GET,
{
"id": "foo",
"queue": {
"pending": 3,
"successes": 0,
"failures": 0
},
"status": "running",
},
)
self.mock_response(responses.POST, {"results": response1})
self.mock_response(responses.POST, {"results": response2})
completed_tasks = list(as_completed(tasks, show_progress=False))
assert 5 == len(completed_tasks)
assert list(
range(5)) == [int(r._task_result["id"]) for r in completed_tasks]
def test_as_completed_exception(self):
tasks = [
FutureTask("group_id", str(n), client=self.client)
for n in range(5)
]
response1 = [{"id": str(n), "result_type": "json"} for n in range(3)]
response2 = [{
"id": str(n),
"result_type": "json"
} for n in range(3, 5)]
self.mock_response(
responses.GET,
{
"id": "foo",
"queue": {
"pending": 3,
"successes": 0,
"failures": 0
},
"status": "running",
},
)
self.mock_response(
responses.GET,
{
"id": "foo",
"queue": {
"pending": 3,
"successes": 0,
"failures": 0
},
"status": "terminated",
},
)
self.mock_response(responses.POST, {"results": response1})
self.mock_response(responses.POST, {"results": response2})
with pytest.raises(GroupTerminalException):
list(as_completed(tasks, show_progress=False))
def test_as_completed(self):
tasks = [
FutureTask("group_id", str(n), client=self.client)
for n in range(5)
]
response1 = [{'id': str(n), 'result_type': 'json'} for n in range(3)]
response2 = [{
'id': str(n),
'result_type': 'json'
} for n in range(3, 5)]
self.mock_response(responses.POST, {'results': response1})
self.mock_response(responses.POST, {'results': response2})
completed_tasks = list(as_completed(tasks, show_progress=False))
self.assertEqual(5, len(completed_tasks))
self.assertEqual(list(range(5)),
[int(r._task_result['id']) for r in completed_tasks])
def map(self, args, *iterargs):
"""
Submits multiple tasks efficiently with positional argument to each function
call, mimicking the behaviour of the builtin `map()` function. When
submitting multiple tasks this is preferred over calling the function
repeatedly.
All positional arguments must be JSON-serializable (i.e., booleans, numbers,
strings, lists, dictionaries).
:param iterable args: An iterable of arguments. A task will be submitted
with each element of the iterable as the first positional argument
to the function.
:param list(iterable) iterargs: If additional iterable arguments are
passed, the function must take that many arguments and is applied
to the items from all iterables in parallel (mimicking builtin
`map()` behaviour).
:return: A list of :class:`FutureTask` for all submitted tasks.
"""
arguments = zip_longest(args, *iterargs)
futures = []
batch = list(itertools.islice(arguments, self.TASK_SUBMIT_SIZE))
while batch:
tasks_info = self.client.new_tasks(
self.group_id,
list_of_arguments=batch,
retry_count=self.retry_count,
)
futures += [
FutureTask(self.group_id,
task_info.id,
client=self.client,
args=task_args)
for task_info, task_args in zip(tasks_info.tasks, batch)
]
batch = list(itertools.islice(arguments, self.TASK_SUBMIT_SIZE))
return futures
def __call__(self, *args, **kwargs):
"""
Submits a task calling the function with the given positional and
keyword arguments.
All positional and keyword arguments must be JSON-serializable (i.e.,
booleans, numbers, strings, lists, dictionaries).
:return: A :class:`FutureTask` for the submitted task.
"""
tasks = self.client.new_task(
self.group_id,
arguments=args,
parameters=kwargs,
retry_count=self.retry_count,
)
task_info = tasks.tasks[0]
return FutureTask(self.group_id,
task_info.id,
client=self.client,
args=args,
kwargs=kwargs)
def test_as_completed_exception(self):
tasks = [
FutureTask("group_id", str(n), client=self.client)
for n in range(5)
]
response1 = [{'id': str(n), 'result_type': 'json'} for n in range(3)]
response2 = [{
'id': str(n),
'result_type': 'json'
} for n in range(3, 5)]
self.mock_response(
responses.GET, {
'id': 'foo',
'queue': {
'pending': 3,
'successes': 0,
'failures': 0
},
'status': 'running'
})
self.mock_response(
responses.GET, {
'id': 'foo',
'queue': {
'pending': 3,
'successes': 0,
'failures': 0
},
'status': 'terminated'
})
self.mock_response(responses.POST, {'results': response1})
self.mock_response(responses.POST, {'results': response2})
with self.assertRaises(GroupTerminalException):
list(as_completed(tasks, show_progress=False))
def test_getattr_access(self):
ft = FutureTask(guid=guid, tuid=tuid, client=self.client)
with self.assertRaises(TimeoutError):
ft.get_result(wait=True, timeout=1)
def test_transient_result(self):
ft = FutureTask(guid=guid, tuid=tuid, client=self.client)
with self.assertRaises(TransientResultError):
ft.get_result(wait=False)
def test_without_guid_tuid(self):
with self.assertRaises(TypeError):
ft = FutureTask()
def test_without_guid_tuid(self):
with pytest.raises(TypeError):
ft = FutureTask()