def test_groupresult_save_restore_nested(self):
"""Test if we can save and restore a nested GroupResult"""
group_id = uuid()
async_result = AsyncResult(id=uuid())
nested_results = [AsyncResult(id=uuid()), AsyncResult(id=uuid())]
nested_group = GroupResult(id=uuid(), results=nested_results)
group = GroupResult(id=group_id, results=[nested_group, async_result])
group.save(backend=self.b)
restored_group = self.b.restore_group(group_id=group_id)
assert restored_group == group
def test_join_native_with_group_chain_group(self):
"""Test group(chain(group)) case, join_native can be run correctly.
In group(chain(group)) case, GroupResult has no _cache property, and
AsyncBackendMixin.iter_native returns a node instead of node._cache,
this test make sure ResultSet.join_native can process correctly both
values of AsyncBackendMixin.iter_native returns.
"""
def _get_meta(tid, result=None, children=None):
return {
'status': states.SUCCESS,
'result': result,
'children': children,
'task_id': tid,
}
results = [self.app.AsyncResult(t) for t in [1, 2, 3]]
values = [(_.id, _get_meta(_.id, _)) for _ in results]
g_res = GroupResult(6, [self.app.AsyncResult(t) for t in [4, 5]])
results += [g_res]
values += [(6, g_res.children)]
x = self.app.ResultSet(results)
x.results[0].backend = Mock()
x.results[0].backend.join = Mock()
x.results[3][0].get = Mock()
x.results[3][0].get.return_value = g_res.results[0]
x.results[3][1].get = Mock()
x.results[3][1].get.return_value = g_res.results[1]
x.iter_native = Mock()
x.iter_native.return_value = values.__iter__()
x.join_native()
x.iter_native.assert_called()
def test_forget(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = GroupResult(uuid(), subs)
ts.forget()
for sub in subs:
self.assertTrue(sub.forgotten)
def test_iter_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend)
for i in range(10)]
ts = GroupResult(uuid(), subtasks)
backend.ids = [subtask.id for subtask in subtasks]
self.assertEqual(len(list(ts.iter_native())), 10)
def test_iterate_yields(self):
ar = MockAsyncResultSuccess(uuid())
ar2 = MockAsyncResultSuccess(uuid())
ts = GroupResult(uuid(), [ar, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def test_join_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend) for i in range(10)]
ts = GroupResult(uuid(), subtasks)
backend.ids = [subtask.id for subtask in subtasks]
res = ts.join_native()
self.assertEqual(res, range(10))
def test_join_timeout(self):
ar = MockAsyncResultSuccess(uuid())
ar2 = MockAsyncResultSuccess(uuid())
ar3 = AsyncResult(uuid())
ts = GroupResult(uuid(), [ar, ar2, ar3])
with self.assertRaises(TimeoutError):
ts.join(timeout=0.0000001)
def test_iterate_eager(self):
ar1 = EagerResult(uuid(), 42, states.SUCCESS)
ar2 = EagerResult(uuid(), 42, states.SUCCESS)
ts = GroupResult(uuid(), [ar1, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def setup(self):
self.size = 11
subtasks = make_mock_group(10)
failed = mock_task('ts11', states.FAILURE, KeyError('Baz'))
save_result(failed)
failed_res = AsyncResult(failed['id'])
self.ts = GroupResult(uuid(), subtasks + [failed_res])
def join_map_jobs(task_ids):
"""Test reduce function that manually joins all mapped jobs."""
print(("task_ids: {}".format(json.dumps(task_ids, indent=2))))
res = GroupResult(id=uuid.uuid4().bytes, results=[
AsyncResult(id[0]) for id in task_ids])
while True:
ready = res.ready()
if ready:
break
time.sleep(5)
results = []
for r in res.join(timeout=10.):
# deduped job?
if isinstance(r, (list, tuple)):
# build resolvable result
task_id = r[0]
results.append({'uuid': task_id,
'job_id': task_id,
'payload_id': task_id,
'status': 'job-deduped'})
else:
results.append(r)
args = [result['payload_id'] for result in results]
return args
def test_on_chord_part_return(self):
"""Test if the ChordCounter is properly decremented and the callback is
triggered after all chord parts have returned"""
gid = uuid()
tid1 = uuid()
tid2 = uuid()
subtasks = [AsyncResult(tid1), AsyncResult(tid2)]
group = GroupResult(id=gid, results=subtasks)
self.b.apply_chord(group, self.add.s())
chord_counter = ChordCounter.objects.get(group_id=gid)
assert chord_counter.count == 2
request = mock.MagicMock()
request.id = subtasks[0].id
request.group = gid
request.task = "my_task"
request.args = ["a", 1, "password"]
request.kwargs = {"c": 3, "d": "e", "password": "******"}
request.argsrepr = "argsrepr"
request.kwargsrepr = "kwargsrepr"
request.hostname = "celery@ip-0-0-0-0"
result = {"foo": "baz"}
self.b.mark_as_done(tid1, result, request=request)
chord_counter.refresh_from_db()
assert chord_counter.count == 1
self.b.mark_as_done(tid2, result, request=request)
with pytest.raises(ChordCounter.DoesNotExist):
ChordCounter.objects.get(group_id=gid)
request.chord.delay.assert_called_once()
def test_save_restore_delete_group(self):
tid = uuid()
tsr = GroupResult(tid, [AsyncResult(uuid()) for _ in range(10)])
self.b.save_group(tid, tsr)
self.b.restore_group(tid)
self.assertEqual(self.b.restore_group(tid), tsr)
self.b.delete_group(tid)
self.assertIsNone(self.b.restore_group(tid))
def test_save_restore(self):
subs = [MockAsyncResultSuccess(uuid()), MockAsyncResultSuccess(uuid())]
ts = GroupResult(uuid(), subs)
ts.save()
with self.assertRaises(AttributeError):
ts.save(backend=object())
self.assertEqual(GroupResult.restore(ts.id).subtasks, ts.subtasks)
ts.delete()
self.assertIsNone(GroupResult.restore(ts.id))
def test_groupresult_save_restore(self):
"""Test if we can save and restore a GroupResult"""
group_id = uuid()
results = [AsyncResult(id=uuid())]
group = GroupResult(id=group_id, results=results)
group.save(backend=self.b)
restored_group = self.b.restore_group(group_id=group_id)
assert restored_group == group
def test_apply_chord_header_result_arg(self):
"""Test if apply_chord can handle Celery <= 5.1 call signature"""
gid = uuid()
tid1 = uuid()
tid2 = uuid()
subtasks = [AsyncResult(tid1), AsyncResult(tid2)]
group = GroupResult(id=gid, results=subtasks)
# Celery < 5.1
self.b.apply_chord(group, self.add.s())
# Celery 5.1
self.b.apply_chord((uuid(), subtasks), self.add.s())
def cancel(request, instance, import_type, import_event_id):
ie = _get_import_event(instance, import_type, import_event_id)
ie.status = GenericImportEvent.CANCELED
ie.save()
# If verifications tasks are still scheduled, we need to revoke them
if ie.task_id:
GroupResult(ie.task_id).revoke()
return list_imports(request, instance)
def _freeze(self, _id=None):
opts = self.options
try:
gid = opts['group']
except KeyError:
gid = opts['group'] = uuid()
new_tasks, results = [], []
for task in self.tasks:
task = maybe_subtask(task).clone()
results.append(task._freeze())
new_tasks.append(task)
self.tasks = self.kwargs['tasks'] = new_tasks
return GroupResult(gid, results)
def group_result(self, app=None):
"""Return the GroupResult of self.
Arguments:
---------
app (Celery): app instance to create the GroupResult with.
"""
return GroupResult(
self.group_id,
[result_from_tuple(r, app=app)
for r in json.loads(self.sub_tasks)],
app=app,
)
def post(self):
task_ids = request.form.getlist('task_ids')
results = [AsyncResult(tid, app=app.celery) for tid in task_ids]
group_result = GroupResult(id=str(uuid.uuid4()), results=results)
group_result.save()
successful = all_finished(group_result.results)
return {
'status':
'SUCCESS' if successful else 'PENDING',
'bulk_task_id':
str(group_result.id),
'bulk_task_url':
app.api.url_for(TaskSubscriptionAPI,
bulk_task_id=str(group_result.id),
_external=True),
'task_results':
get_all_task_results(group_result.results)
}
def test_callback_failure(self):
"""Test if a failure in the chord callback is properly handled"""
gid = uuid()
tid1 = uuid()
tid2 = uuid()
cid = uuid()
subtasks = [AsyncResult(tid1), AsyncResult(tid2)]
group = GroupResult(id=gid, results=subtasks)
self.b.apply_chord(group, self.add.s())
chord_counter = ChordCounter.objects.get(group_id=gid)
assert chord_counter.count == 2
request = mock.MagicMock()
request.id = subtasks[0].id
request.group = gid
request.task = "my_task"
request.args = ["a", 1, "password"]
request.kwargs = {"c": 3, "d": "e", "password": "******"}
request.argsrepr = "argsrepr"
request.kwargsrepr = "kwargsrepr"
request.hostname = "celery@ip-0-0-0-0"
request.properties = {"periodic_task_name": "my_periodic_task"}
request.ignore_result = False
request.chord.id = cid
result = {"foo": "baz"}
# Trigger an exception when the callback is triggered
request.chord.delay.side_effect = ValueError()
self.b.mark_as_done(tid1, result, request=request)
chord_counter.refresh_from_db()
assert chord_counter.count == 1
self.b.mark_as_done(tid2, result, request=request)
with pytest.raises(ChordCounter.DoesNotExist):
ChordCounter.objects.get(group_id=gid)
request.chord.delay.assert_called_once()
assert TaskResult.objects.get(task_id=cid).status == states.FAILURE
def stop_jobs(jobs: List[Job], job_options: Dict[str, Any]):
raise NotImplementedError
# TODO: not sure whether we should .revoke(terminate=True)
# TODO: one of the options below should work, but I don't have time to test it right now...
# http://docs.celeryproject.org/en/latest/userguide/workers.html#worker-persistent-revokes
from celery.result import AsyncResult
AsyncResult(job_options['command_id']).revoke()
# https://docs.celeryproject.org/en/stable/reference/celery.result.html
from celery.result import GroupResult
g = GroupResult(id=job_options['command_id'])
# https://stackoverflow.com/questions/13685344/retrieving-groupresult-from-taskset-id-in-celery
# We may need to call result.save() in the task above for it to work...
from celery.result import GroupResult
result = GroupResult.restore(job_options['command_id'])
result.revoke()
from celery.task.control import revoke
revoke(job_options['command_id'])
def test_on_chord_part_return_failure(self):
"""Test if a failure in one of the chord header tasks is properly handled
and the callback was not triggered
"""
gid = uuid()
tid1 = uuid()
tid2 = uuid()
cid = uuid()
subtasks = [AsyncResult(tid1), AsyncResult(tid2)]
group = GroupResult(id=gid, results=subtasks)
self.b.apply_chord(group, self.add.s())
chord_counter = ChordCounter.objects.get(group_id=gid)
assert chord_counter.count == 2
request = mock.MagicMock()
request.id = tid1
request.group = gid
request.task = "my_task"
request.args = ["a", 1, "password"]
request.kwargs = {"c": 3, "d": "e", "password": "******"}
request.argsrepr = "argsrepr"
request.kwargsrepr = "kwargsrepr"
request.hostname = "celery@ip-0-0-0-0"
request.properties = {"periodic_task_name": "my_periodic_task"}
request.chord.id = cid
result = {"foo": "baz"}
self.b.mark_as_done(tid1, result, request=request)
chord_counter.refresh_from_db()
assert chord_counter.count == 1
request.id = tid2
self.b.mark_as_failure(tid2, ValueError(), request=request)
with pytest.raises(ChordCounter.DoesNotExist):
ChordCounter.objects.get(group_id=gid)
request.chord.delay.assert_not_called()
def test_GroupResult(self):
x = GroupResult(uuid(), [AsyncResult(uuid()) for _ in range(10)])
self.assertEqual(x, from_serializable(x.serializable()))
self.assertEqual(x, from_serializable(x))
def setup(self):
self.ts = GroupResult(
uuid(),
[AsyncResult(uuid()), AsyncResult(uuid())])
def group_status(group_id, result_ids):
results = [AsyncResult(r_id) for r_id in result_ids]
gr = GroupResult(id, results)
return gr.successful()
class test_RedisBackend_chords_complex(basetest_RedisBackend):
@pytest.fixture(scope="function", autouse=True)
def complex_header_result(self):
with patch("celery.result.GroupResult.restore") as p:
yield p
@pytest.mark.parametrize(['results', 'assert_save_called'], [
# No results in the header at all - won't call `save()`
(tuple(), False),
# Simple results in the header - won't call `save()`
((AsyncResult("foo"), ), False),
# Many simple results in the header - won't call `save()`
((AsyncResult("foo"), ) * 42, False),
# A single complex result in the header - will call `save()`
((GroupResult("foo", []),), True),
# Many complex results in the header - will call `save()`
((GroupResult("foo"), ) * 42, True),
# Mixed simple and complex results in the header - will call `save()`
(itertools.islice(
itertools.cycle((
AsyncResult("foo"), GroupResult("foo"),
)), 42,
), True),
])
def test_apply_chord_complex_header(self, results, assert_save_called):
mock_group_result = Mock()
mock_group_result.return_value.results = results
self.app.GroupResult = mock_group_result
header_result_args = ("gid11", results)
self.b.apply_chord(header_result_args, None)
if assert_save_called:
mock_group_result.return_value.save.assert_called_once_with(backend=self.b)
else:
mock_group_result.return_value.save.assert_not_called()
def test_on_chord_part_return_timeout(self, complex_header_result):
tasks = [self.create_task(i) for i in range(10)]
random.shuffle(tasks)
try:
self.app.conf.result_chord_join_timeout += 1.0
for task, result_val in zip(tasks, itertools.cycle((42, ))):
self.b.on_chord_part_return(
task.request, states.SUCCESS, result_val,
)
finally:
self.app.conf.result_chord_join_timeout -= 1.0
join_func = complex_header_result.return_value.join_native
join_func.assert_called_once_with(timeout=4.0, propagate=True)
@pytest.mark.parametrize("supports_native_join", (True, False))
def test_on_chord_part_return(
self, complex_header_result, supports_native_join,
):
mock_result_obj = complex_header_result.return_value
mock_result_obj.supports_native_join = supports_native_join
tasks = [self.create_task(i) for i in range(10)]
random.shuffle(tasks)
with self.chord_context(10) as (tasks, request, callback):
for task, result_val in zip(tasks, itertools.cycle((42, ))):
self.b.on_chord_part_return(
task.request, states.SUCCESS, result_val,
)
# Confirm that `zadd` was called even though we won't end up
# using the data pushed into the sorted set
assert self.b.client.zadd.call_count == 1
self.b.client.zadd.reset_mock()
# Confirm that neither `zrange` not `lrange` were called
self.b.client.zrange.assert_not_called()
self.b.client.lrange.assert_not_called()
# Confirm that the `GroupResult.restore` mock was called
complex_header_result.assert_called_once_with(request.group)
# Confirm the the callback was called with the `join()`ed group result
if supports_native_join:
expected_join = mock_result_obj.join_native
else:
expected_join = mock_result_obj.join
callback.delay.assert_called_once_with(expected_join())
def test_getitem(self):
subs = [MockAsyncResultSuccess(uuid()), MockAsyncResultSuccess(uuid())]
ts = GroupResult(uuid(), subs)
self.assertIs(ts[0], subs[0])
def test_iterate_raises(self):
ar = MockAsyncResultFailure(uuid())
ts = GroupResult(uuid(), [ar])
it = iter(ts)
with self.assertRaises(KeyError):
it.next()
def setup(self):
self.size = 10
self.ts = GroupResult(uuid(), make_mock_group(self.size))
def _run(self):
"""Submit the parallel work jobs, wait for all to complete, and return the results."""
# get map and work functions
workFunc = getFunction(self._call)
# get list of jobs
jobs = []
if not isinstance(self._args[0], (list, tuple)):
raise ParMapWorkUnitError(
"Invalid type for ParWorkUnit argument 1: %s\n%s" %
(type(self._args[0]), self._args[0]))
for i, arg in enumerate(self._args[0]):
workArgs = [arg]
for mapArg in self._args[1:]:
if isinstance(mapArg, (list, tuple)) and len(mapArg) == len(
self._args[0]):
workArgs.append(mapArg[i])
else:
workArgs.append(mapArg)
# append work unit id and job number for job tracking
job = workFunc(*workArgs, **{'wuid': self._wuid, 'job_num': i})
# update context in job payload
job.setdefault('context', {}).update(self._ctx)
# propagate job/container configs from HySDS context
if 'priority' not in job:
job['priority'] = int(self._ctx.get('job_priority', 0))
if 'username' not in job:
job['username'] = self._ctx.get('username', None)
if 'container_image_name' not in job:
job['container_image_name'] = self._ctx.get(
'container_image_name', None)
if 'container_image_url' not in job:
job['container_image_url'] = self._ctx.get(
'container_image_url', None)
if 'container_mappings' not in job:
job['container_mappings'] = self._ctx.get(
'container_mappings', {})
# set tag from HySDS context
if 'tag' not in job and 'tag' in self._ctx:
job['tag'] = self._ctx['tag']
jobs.append(job)
# submit jobs and wait for execution
group_res = group(
submit_job.s(job).set(queue=self._job_queue) for job in jobs)()
while True:
ready = group_res.ready()
if ready:
break
time.sleep(5)
task_ids = group_res.join(timeout=10.)
# if async, return task IDs; otherwise harvest results in a group then return
if self._async:
return [id for id in task_ids]
else:
res = GroupResult(id=uuid.uuid4().bytes,
results=[AsyncResult(id[0]) for id in task_ids])
while True:
ready = res.ready()
if ready:
break
time.sleep(5)
results = []
for r in res.join(timeout=10.):
# deduped job?
if isinstance(r, (list, tuple)):
# build resolvable result
task_id = r[0]
results.append({
'uuid': task_id,
'job_id': task_id,
'payload_id': task_id,
'status': 'job-deduped'
})
else:
results.append(r)
return results
