def test_iter_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend)
for i in range(10)]
ts = TaskSetResult(uuid(), subtasks)
backend.ids = [subtask.task_id for subtask in subtasks]
self.assertEqual(len(list(ts.iter_native())), 10)
def test_join_timeout(self):
ar = MockAsyncResultSuccess(uuid())
ar2 = MockAsyncResultSuccess(uuid())
ar3 = AsyncResult(uuid())
ts = TaskSetResult(uuid(), [ar, ar2, ar3])
with self.assertRaises(TimeoutError):
ts.join(timeout=0.0000001)
def test_join_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend) for i in range(10)]
ts = TaskSetResult(uuid(), subtasks)
backend.ids = [subtask.id for subtask in subtasks]
res = ts.join_native()
self.assertEqual(res, range(10))
def test_forget(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = TaskSetResult(uuid(), subs)
ts.forget()
for sub in subs:
self.assertTrue(sub.forgotten)
def test_forget(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = TaskSetResult(uuid(), subs)
ts.forget()
for sub in subs:
self.assertTrue(sub.forgotten)
def setUp(self):
self.size = 11
subtasks = make_mock_taskset(10)
failed = mock_task("ts11", states.FAILURE, KeyError("Baz"))
save_result(failed)
failed_res = AsyncResult(failed["id"])
self.ts = TaskSetResult(gen_unique_id(), subtasks + [failed_res])
def test_iter_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend)
for i in range(10)]
ts = TaskSetResult(uuid(), subtasks)
backend.ids = [subtask.task_id for subtask in subtasks]
self.assertEqual(len(list(ts.iter_native())), 10)
def test_join_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend)
for i in range(10)]
ts = TaskSetResult(uuid(), subtasks)
backend.ids = [subtask.task_id for subtask in subtasks]
res = ts.join_native()
self.assertEqual(res, range(10))
def test_save_restore(self):
subs = [MockAsyncResultSuccess(gen_unique_id()),
MockAsyncResultSuccess(gen_unique_id())]
ts = TaskSetResult(gen_unique_id(), subs)
ts.save()
self.assertRaises(AttributeError, ts.save, backend=object())
self.assertEqual(TaskSetResult.restore(ts.taskset_id).subtasks,
ts.subtasks)
self.assertRaises(AttributeError,
TaskSetResult.restore, ts.taskset_id,
backend=object())
def test_save_restore(self):
subs = [
MockAsyncResultSuccess(gen_unique_id()),
MockAsyncResultSuccess(gen_unique_id())
]
ts = TaskSetResult(gen_unique_id(), subs)
ts.save()
self.assertRaises(AttributeError, ts.save, backend=object())
self.assertEqual(
TaskSetResult.restore(ts.taskset_id).subtasks, ts.subtasks)
self.assertRaises(AttributeError,
TaskSetResult.restore,
ts.taskset_id,
backend=object())
def test_iterate_yields(self):
ar = MockAsyncResultSuccess(gen_unique_id())
ar2 = MockAsyncResultSuccess(gen_unique_id())
ts = TaskSetResult(gen_unique_id(), [ar, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def test_iterate_eager(self):
ar1 = EagerResult(gen_unique_id(), 42, states.SUCCESS)
ar2 = EagerResult(gen_unique_id(), 42, states.SUCCESS)
ts = TaskSetResult(gen_unique_id(), [ar1, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def apply(self):
"""Applies the taskset locally."""
taskset_id = gen_unique_id()
# This will be filled with EagerResults.
return TaskSetResult(taskset_id, [task.apply(taskset_id=taskset_id)
for task in self.tasks])
def setUp(self):
self.size = 11
subtasks = make_mock_taskset(10)
failed = mock_task("ts11", states.FAILURE, KeyError("Baz"))
save_result(failed)
failed_res = AsyncResult(failed["id"])
self.ts = TaskSetResult(gen_unique_id(), subtasks + [failed_res])
def task_status(request, taskid):
"""
Return task status based on:
djcelery.views.task_status
"""
result = TaskSetResult.restore(taskid)
percent = "0.0"
status = 'PROGRESS'
results = []
if result.ready():
finished = True
results = result.join()
else:
finished = False
percent = "%.1f" % (
(1.0 * int(result.completed_count()) / int(result.total)) * 100)
rdict = {
'taskid': taskid,
'finished': finished,
'results': results,
'status': status,
'completed': percent
}
if request.is_ajax():
response = anyjson.dumps(rdict)
return HttpResponse(
response, content_type='application/javascript; charset=utf-8')
return render_to_response('mail/messages/task_status.html',
rdict,
context_instance=RequestContext(request))
def _unlock_chord(setid, callback, interval=1, max_retries=None):
result = TaskSetResult.restore(setid)
if result.ready():
subtask(callback).delay(result.join())
result.delete()
else:
_unlock_chord.retry(countdown=interval, max_retries=max_retries)
def _unlock_chord(setid, callback, interval=1, max_retries=None):
result = TaskSetResult.restore(setid)
if result.ready():
subtask(callback).delay(result.join())
result.delete()
else:
_unlock_chord.retry(countdown=interval, max_retries=max_retries)
def test_getitem(self):
subs = [
MockAsyncResultSuccess(gen_unique_id()),
MockAsyncResultSuccess(gen_unique_id())
]
ts = TaskSetResult(gen_unique_id(), subs)
self.assertIs(ts[0], subs[0])
def on_chord_part_return(self, task, keyprefix="chord-unlock-%s"):
setid = task.request.taskset
key = keyprefix % setid
deps = TaskSetResult.restore(setid, backend=task.backend)
if self.client.incr(key) >= deps.total:
subtask(task.request.chord).delay(deps.join())
deps.delete()
self.client.expire(key, 86400)
def on_chord_part_return(self, task, keyprefix="chord-unlock-%s"):
setid = task.request.taskset
key = keyprefix % setid
deps = TaskSetResult.restore(setid, backend=task.backend)
if self.client.incr(key) >= deps.total:
subtask(task.request.chord).delay(deps.join())
deps.delete()
self.client.expire(key, 86400)
def test_save_restore_delete_taskset(self):
tid = uuid()
tsr = TaskSetResult(tid, [AsyncResult(uuid()) for _ in range(10)])
self.b.save_taskset(tid, tsr)
stored = self.b.restore_taskset(tid)
print(stored)
self.assertEqual(self.b.restore_taskset(tid), tsr)
self.b.delete_taskset(tid)
self.assertIsNone(self.b.restore_taskset(tid))
class TestTaskSetPending(unittest.TestCase):
def setUp(self):
self.ts = TaskSetResult(gen_unique_id(), [
AsyncResult(gen_unique_id()),
AsyncResult(gen_unique_id())])
def test_completed_count(self):
self.assertEquals(self.ts.completed_count(), 0)
def test_ready(self):
self.assertFalse(self.ts.ready())
def test_waiting(self):
self.assertTrue(self.ts.waiting())
def x_join(self):
self.assertRaises(TimeoutError, self.ts.join, timeout=0.001)
def join_taskset(setid, callback, interval=10, max_retries=None, propagate=True):
'''
Task to poll if the TaskSet ``setid`` has finished.
Pass results of the TaskSet to ``callback``.
'''
result = TaskSetResult.restore(setid)
if result.ready():
return subtask(callback).delay(result.join(propagate=propagate))
join_taskset.retry(countdown=interval, max_retries=max_retries)
def on_chord_part_return(self, task, propagate=False,
keyprefix="chord-unlock-%s"):
from celery.task.sets import subtask
from celery.result import TaskSetResult
setid = task.request.taskset
key = keyprefix % setid
deps = TaskSetResult.restore(setid, backend=task.backend)
if self.client.incr(key) >= deps.total:
subtask(task.request.chord).delay(deps.join(propagate=propagate))
deps.delete()
self.client.expire(key, 86400)
def on_chord_part_return(self, task, propagate=False):
from celery import subtask
from celery.result import TaskSetResult
setid = task.request.taskset
if not setid:
return
key = self.get_key_for_chord(setid)
deps = TaskSetResult.restore(setid, backend=task.backend)
if self.client.incr(key) >= deps.total:
subtask(task.request.chord).delay(deps.join(propagate=propagate))
deps.delete()
self.client.delete(key)
class TestFailedTaskSetResult(TestTaskSetResult):
def setup(self):
self.size = 11
subtasks = make_mock_taskset(10)
failed = mock_task("ts11", states.FAILURE, KeyError("Baz"))
save_result(failed)
failed_res = AsyncResult(failed["id"])
self.ts = TaskSetResult(uuid(), subtasks + [failed_res])
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i in xrange(self.size - 1):
t = it.next()
self.assertEqual(t.get(), i)
with self.assertRaises(KeyError):
t = it.next() # need to do this in two lines or 2to3 borks.
t.get()
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), self.ts.total - 1)
def test___iter__(self):
it = iter(self.ts)
def consume():
return list(it)
self.assertRaises(KeyError, consume)
def test_join(self):
self.assertRaises(KeyError, self.ts.join)
def test_successful(self):
self.assertFalse(self.ts.successful())
def test_failed(self):
self.assertTrue(self.ts.failed())
class TestTaskSetPending(unittest.TestCase):
def setUp(self):
self.ts = TaskSetResult(
gen_unique_id(),
[AsyncResult(gen_unique_id()),
AsyncResult(gen_unique_id())])
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), 0)
def test_ready(self):
self.assertFalse(self.ts.ready())
def test_waiting(self):
self.assertTrue(self.ts.waiting())
def x_join(self):
self.assertRaises(TimeoutError, self.ts.join, timeout=0.001)
@skip_if_quick
def x_join_longer(self):
self.assertRaises(TimeoutError, self.ts.join, timeout=1)
def save(self):
if not self.id:
self.id = utils.random_id()
if self.taskset_id:
tsr = TaskSetResult.restore(self.taskset_id)
if tsr is not None and hasattr(tsr,"taskset_id"):
subtasks = tsr.subtasks
if subtasks:
for st in subtasks:
jasr = JobASyncResult(async_id=st.task_id,job=self)
jasr.save()
super(Job, self).save()
def on_chord_part_return(self,
task,
propagate=False,
keyprefix="chord-unlock-%s"):
from celery.task.sets import subtask
from celery.result import TaskSetResult
setid = task.request.taskset
key = keyprefix % setid
deps = TaskSetResult.restore(setid, backend=task.backend)
if self.client.incr(key) >= deps.total:
subtask(task.request.chord).delay(deps.join(propagate=propagate))
deps.delete()
self.client.expire(key, 86400)
class TestTaskSetPending(AppCase):
def setup(self):
self.ts = TaskSetResult(uuid(), [
AsyncResult(uuid()),
AsyncResult(uuid())])
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), 0)
def test_ready(self):
self.assertFalse(self.ts.ready())
def test_waiting(self):
self.assertTrue(self.ts.waiting())
def x_join(self):
self.assertRaises(TimeoutError, self.ts.join, timeout=0.001)
@skip_if_quick
def x_join_longer(self):
self.assertRaises(TimeoutError, self.ts.join, timeout=1)
class TestFailedTaskSetResult(TestTaskSetResult):
def setUp(self):
self.size = 11
subtasks = make_mock_taskset(10)
failed = mock_task("ts11", "FAILED", KeyError("Baz"))
save_result(failed)
failed_res = AsyncResult(failed["id"])
self.ts = TaskSetResult(gen_unique_id(), subtasks + [failed_res])
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i in xrange(self.size - 1):
t = it.next()
self.assertEquals(t.get(), i)
self.assertRaises(KeyError, it.next().get)
def test_completed_count(self):
self.assertEquals(self.ts.completed_count(), self.ts.total - 1)
def test___iter__(self):
it = iter(self.ts)
def consume():
return list(it)
self.assertRaises(KeyError, consume)
def test_join(self):
self.assertRaises(KeyError, self.ts.join)
def test_successful(self):
self.assertFalse(self.ts.successful())
def test_failed(self):
self.assertTrue(self.ts.failed())
class TestFailedTaskSetResult(TestTaskSetResult):
def setUp(self):
self.size = 11
subtasks = make_mock_taskset(10)
failed = mock_task("ts11", states.FAILURE, KeyError("Baz"))
save_result(failed)
failed_res = AsyncResult(failed["id"])
self.ts = TaskSetResult(gen_unique_id(), subtasks + [failed_res])
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i in xrange(self.size - 1):
t = it.next()
self.assertEqual(t.get(), i)
self.assertRaises(KeyError, it.next().get)
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), self.ts.total - 1)
def test___iter__(self):
it = iter(self.ts)
def consume():
return list(it)
self.assertRaises(KeyError, consume)
def test_join(self):
self.assertRaises(KeyError, self.ts.join)
def test_successful(self):
self.assertFalse(self.ts.successful())
def test_failed(self):
self.assertTrue(self.ts.failed())
def status(request):
if request.user.is_authenticated():
profile = request.user.profile
"""
I need to execute 2 separate tasks:
1) Download and save (set) - this should already be started
2) Calculate and save pmi information (set)
"""
if profile.stage==0:
response_data = {
"stage":0,
"state":'not yet started',
}
elif profile.stage==1:
result = TaskSetResult.restore(profile.task_id)
response_data = {
"stage":1,
"completed": result.completed_count(),
"total": result.total,
}
elif profile.stage==2:
result = TaskSetResult.restore(profile.task_id)
response_data = {
"stage":2,
"completed": result.completed_count(),
"total": result.total,
}
else:
response_data = {
"stage":3,
"state": "completed",
}
else:
response_data = {
"error": "user must be logged in"
}
return HttpResponse(json.dumps(response_data), mimetype="application/json")
def startDownload(request):
"""
Ajax call to start the download
"""
if request.user.is_authenticated():
if request.method == 'POST':
profile = request.user.profile
if profile.stage > 0:
if profile.stage < 3:
result = TaskSetResult.restore(profile.task_id)
response_data = {
"error": "download already started",
"stage" : profile.stage,
"completed" : result.completed_count(),
"total" : result.total,
}
else:
reponse_data = {
"error": "download already finished",
"stage" : profile.stage,
"state" : "completed",
}
else:
graphapi = facebook.GraphAPI(profile.fblogin.access_token)
me = graphapi.get_object('me')
friends = [(f['id'],f['name']) for f in graphapi.get_connections('me','friends')['data']]
friends.append((me['id'],me['name']))
subtasks = [tasks.dlUser.subtask((profile.id,graphapi,fbid,name)) for (fbid,name) in friends]
result = TaskSet(tasks=subtasks).apply_async()
result.save()
profile.stage = 1
profile.task_id = result.taskset_id
profile.save()
r = tasks.checkTaskSet.delay(result,profile.id)
response_data = {
"stage":1,
"completed": result.completed_count(),
"total": result.total,
}
else:
response_data = {
"error": "must be a post request"
}
else:
response_data = {
"error": "user must be logged in"
}
return HttpResponse(json.dumps(response_data), mimetype="application/json")
def process(self, taskid, format=None):
"process a taskset"
result = TaskSetResult.restore(taskid, backend=dbbackend)
if (result is None or
'taskids' not in session or
taskid not in session['taskids']):
if format == 'json':
return ajax_code(404,
_('The task status requested '
'has expired or does not exist'))
flash(_('The task status requested has expired or does not exist'))
redirect(url(controller='messages', action='quarantine'))
percent = "0.0"
status = 'PROGRESS'
results = []
#print '=' * 20, result.completed_count(), result.total, result.ready()
if result.ready():
finished = True
results = result.join()
else:
session['bulkprocess-count'] += 1
if (session['bulkprocess-count'] >= 10 and
result.completed_count() == 0):
result.revoke()
del session['bulkprocess-count']
session.save()
if format == 'json':
return ajax_code(503,
_('An error occured in processing, try again later'))
flash_alert(_('An error occured in processing, try again later'))
redirect(url(controller='messages', action='quarantine'))
finished = False
percent = "%.1f" % ((1.0 * int(result.completed_count()) /
int(result.total)) * 100)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(finished=finished,
results=results,
status=status,
completed=percent)
return json.dumps(data)
c.finished = finished
c.results = results
c.status = status
c.completed = percent
return render('/messages/taskstatus.html')
def process(self, taskid, format=None):
"process a taskset"
result = TaskSetResult.restore(taskid, backend=dbbackend)
if (result is None or
'taskids' not in session or
taskid not in session['taskids']):
if format == 'json':
return ajax_code(404,
_('The task status requested '
'has expired or does not exist'))
flash(_('The task status requested has expired or does not exist'))
redirect(url(controller='messages', action='quarantine'))
percent = "0.0"
status = 'PROGRESS'
results = []
#print '=' * 20, result.completed_count(), result.total, result.ready()
if result.ready():
finished = True
results = result.join()
else:
session['bulkprocess-count'] += 1
if (session['bulkprocess-count'] >= 10 and
result.completed_count() == 0):
result.revoke()
del session['bulkprocess-count']
session.save()
if format == 'json':
return ajax_code(503,
_('An error occured in processing, try again later'))
flash_alert(_('An error occured in processing, try again later'))
redirect(url(controller='messages', action='quarantine'))
finished = False
percent = "%.1f" % ((1.0 * int(result.completed_count()) /
int(result.total)) * 100)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(finished=finished,
results=results,
status=status,
completed=percent)
return json.dumps(data)
c.finished = finished
c.results = results
c.status = status
c.completed = percent
return render('/messages/taskstatus.html')
def on_chord_part_return(self, task, propagate=False):
if not self.implements_incr:
return
from celery import subtask
from celery.result import TaskSetResult
setid = task.request.taskset
if not setid:
return
key = self.get_key_for_chord(setid)
deps = TaskSetResult.restore(setid, backend=task.backend)
val = self.incr(key)
if val >= deps.total:
subtask(task.request.chord).delay(deps.join(propagate=propagate))
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def tasks(self):
tsr = TaskSetResult.restore(self.taskset_id)
dict = {}
if tsr is not None and hasattr(tsr, "taskset_id"):
dict["taskset_id"] = tsr.taskset_id
results = []
if tsr.subtasks:
for subtask in tsr.subtasks:
results.append(
{
"name": subtask.task_name,
"result": subtask.result,
"success": subtask.successful(),
"state": subtask.state,
}
)
dict["completed_count"] = tsr.completed_count()
dict["failed"] = tsr.failed()
dict["total"] = tsr.total
if tsr.total != 0:
dict["percent"] = float(tsr.completed_count()) / float(tsr.total) * 100
else:
dict["percent"] = 0
dict["ready"] = tsr.ready()
dict["successful"] = tsr.successful()
dict["waiting"] = tsr.waiting()
else:
dict["completed_count"] = 0
dict["failed"] = 0
dict["total"] = 0
dict["percent"] = 0
dict["successful"] = False
dict["waiting"] = False
dict["result"] = results
return dict
else:
dict["taskset_id"] = ""
dict["completed_count"] = 0
dict["failed"] = 0
dict["percent"] = 0
dict["ready"] = True
dict["successful"] = False
dict["total"] = 0
dict["waiting"] = False
return dict
def apply_async(self,
connection=None,
connect_timeout=conf.BROKER_CONNECTION_TIMEOUT):
"""Run all tasks in the taskset.
Returns a :class:`celery.result.TaskSetResult` instance.
Example
>>> ts = TaskSet(tasks=(
... RefreshFeedTask.subtask(["http://foo.com/rss"]),
... RefreshFeedTask.subtask(["http://bar.com/rss"]),
... ))
>>> result = ts.apply_async()
>>> result.taskset_id
"d2c9b261-8eff-4bfb-8459-1e1b72063514"
>>> result.subtask_ids
["b4996460-d959-49c8-aeb9-39c530dcde25",
"598d2d18-ab86-45ca-8b4f-0779f5d6a3cb"]
>>> result.waiting()
True
>>> time.sleep(10)
>>> result.ready()
True
>>> result.successful()
True
>>> result.failed()
False
>>> result.join()
[True, True]
"""
if conf.ALWAYS_EAGER:
return self.apply()
taskset_id = gen_unique_id()
publisher = TaskPublisher(connection=connection)
try:
results = [
task.apply_async(taskset_id=taskset_id, publisher=publisher)
for task in self.tasks
]
finally:
publisher.close()
return TaskSetResult(taskset_id, results)
class test_TaskSetResult(AppCase):
def setup(self):
self.size = 10
self.ts = TaskSetResult(uuid(), make_mock_group(self.app, self.size))
def test_total(self):
self.assertEqual(self.ts.total, self.size)
def test_compat_properties(self):
self.assertEqual(self.ts.taskset_id, self.ts.id)
self.ts.taskset_id = "foo"
self.assertEqual(self.ts.taskset_id, "foo")
def test_compat_subtasks_kwarg(self):
x = TaskSetResult(uuid(), subtasks=[1, 2, 3])
self.assertEqual(x.results, [1, 2, 3])
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i, t in enumerate(it):
self.assertEqual(t.get(), i)
class test_TaskSetResult(AppCase):
def setup(self):
self.size = 10
self.ts = TaskSetResult(uuid(), make_mock_group(self.app, self.size))
def test_total(self):
self.assertEqual(self.ts.total, self.size)
def test_compat_properties(self):
self.assertEqual(self.ts.taskset_id, self.ts.id)
self.ts.taskset_id = 'foo'
self.assertEqual(self.ts.taskset_id, 'foo')
def test_compat_subtasks_kwarg(self):
x = TaskSetResult(uuid(), subtasks=[1, 2, 3])
self.assertEqual(x.results, [1, 2, 3])
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i, t in enumerate(it):
self.assertEqual(t.get(), i)
class TestTaskSetResult(unittest.TestCase):
def setUp(self):
self.size = 10
self.ts = TaskSetResult(gen_unique_id(), make_mock_taskset(self.size))
def test_total(self):
self.assertEquals(self.ts.total, self.size)
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i, t in enumerate(it):
self.assertEquals(t.get(), i)
def test___iter__(self):
it = iter(self.ts)
results = sorted(list(it))
self.assertEquals(results, list(xrange(self.size)))
def test_join(self):
joined = self.ts.join()
self.assertEquals(joined, list(xrange(self.size)))
def test_successful(self):
self.assertTrue(self.ts.successful())
def test_failed(self):
self.assertFalse(self.ts.failed())
def test_waiting(self):
self.assertFalse(self.ts.waiting())
def test_ready(self):
self.assertTrue(self.ts.ready())
def test_completed_count(self):
self.assertEquals(self.ts.completed_count(), self.ts.total)
def task_status(request, taskid):
"""
Return task status based on:
djcelery.views.task_status
"""
result = TaskSetResult.restore(taskid)
percent = "0.0"
status = 'PROGRESS'
results = []
if result.ready():
finished = True
results = result.join()
else:
finished = False
percent = "%.1f" % ((1.0 * int(result.completed_count()) / int(result.total)) * 100)
rdict = {'taskid': taskid, 'finished': finished, 'results': results,
'status': status, 'completed': percent}
if request.is_ajax():
response = anyjson.dumps(rdict)
return HttpResponse(response,
content_type='application/javascript; charset=utf-8')
return render_to_response('messages/task_status.html', rdict,
context_instance=RequestContext(request))
def get_task_group_state(task_group_id):
"""return a list containing states of all tasks given a task set ID"""
task_group_state = []
percent_done = 0
taskset = TaskSetResult.restore(task_group_id)
if not taskset:
logger.error("TaskSet with UUID '%s' doesn't exist", task_group_id)
return task_group_state
for task in taskset.results:
# AsyncResult.info does not contain task state after task has finished
if task.state == celery.states.SUCCESS:
percent_done = 100
elif task.info:
try:
percent_done = task.info.get('percent_done') or 0
except AttributeError:
logger.error("Task %s failed: %s", task, task.info)
task_group_state.append({
'state': task.state,
'percent_done': percent_done,
})
return task_group_state
class test_pending_TaskSet(AppCase):
def setup(self):
self.ts = TaskSetResult(
uuid(),
[AsyncResult(uuid()), AsyncResult(uuid())])
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), 0)
def test_ready(self):
self.assertFalse(self.ts.ready())
def test_waiting(self):
self.assertTrue(self.ts.waiting())
def x_join(self):
with self.assertRaises(TimeoutError):
self.ts.join(timeout=0.001)
@skip_if_quick
def x_join_longer(self):
with self.assertRaises(TimeoutError):
self.ts.join(timeout=1)
class TestFailedTaskSetResult(TestTaskSetResult):
def setup(self):
self.size = 11
subtasks = make_mock_taskset(10)
failed = mock_task("ts11", states.FAILURE, KeyError("Baz"))
save_result(failed)
failed_res = AsyncResult(failed["id"])
self.ts = TaskSetResult(uuid(), subtasks + [failed_res])
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i in xrange(self.size - 1):
t = it.next()
self.assertEqual(t.get(), i)
with self.assertRaises(KeyError):
t = it.next() # need to do this in two lines or 2to3 borks.
t.get()
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), len(self.ts) - 1)
def test___iter__(self):
it = iter(self.ts)
def consume():
return list(it)
with self.assertRaises(KeyError):
consume()
def test_join(self):
with self.assertRaises(KeyError):
self.ts.join()
def test_successful(self):
self.assertFalse(self.ts.successful())
def test_failed(self):
self.assertTrue(self.ts.failed())
def setup(self):
self.ts = TaskSetResult(uuid(), [
AsyncResult(uuid()),
AsyncResult(uuid())])
def setUp(self):
self.ts = TaskSetResult(
gen_unique_id(),
[AsyncResult(gen_unique_id()),
AsyncResult(gen_unique_id())])
def TaskSetResult(self, taskset_id, results, **kwargs):
"""Create :class:`celery.result.TaskSetResult` instance."""
from celery.result import TaskSetResult
return TaskSetResult(taskset_id, results, app=self)
def test_revoke_from_resultset(self):
from celery.result import TaskSetResult, AsyncResult
r = TaskSetResult(gen_unique_id(), map(AsyncResult, [gen_unique_id()
for i in range(10)]))
r.revoke()
self.assertIn("revoke", MockBroadcastPublisher.sent)
class TestTaskSetResult(AppCase):
def setup(self):
self.size = 10
self.ts = TaskSetResult(uuid(), make_mock_taskset(self.size))
def test_total(self):
self.assertEqual(self.ts.total, self.size)
def test_iterate_raises(self):
ar = MockAsyncResultFailure(uuid())
ts = TaskSetResult(uuid(), [ar])
it = iter(ts)
self.assertRaises(KeyError, it.next)
def test_forget(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = TaskSetResult(uuid(), subs)
ts.forget()
for sub in subs:
self.assertTrue(sub.forgotten)
def test_getitem(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = TaskSetResult(uuid(), subs)
self.assertIs(ts[0], subs[0])
def test_save_restore(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = TaskSetResult(uuid(), subs)
ts.save()
self.assertRaises(AttributeError, ts.save, backend=object())
self.assertEqual(TaskSetResult.restore(ts.taskset_id).subtasks,
ts.subtasks)
ts.delete()
self.assertIsNone(TaskSetResult.restore(ts.taskset_id))
self.assertRaises(AttributeError,
TaskSetResult.restore, ts.taskset_id,
backend=object())
def test_join_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend)
for i in range(10)]
ts = TaskSetResult(uuid(), subtasks)
backend.ids = [subtask.task_id for subtask in subtasks]
res = ts.join_native()
self.assertEqual(res, range(10))
def test_iter_native(self):
backend = SimpleBackend()
subtasks = [AsyncResult(uuid(), backend=backend)
for i in range(10)]
ts = TaskSetResult(uuid(), subtasks)
backend.ids = [subtask.task_id for subtask in subtasks]
self.assertEqual(len(list(ts.iter_native())), 10)
def test_iterate_yields(self):
ar = MockAsyncResultSuccess(uuid())
ar2 = MockAsyncResultSuccess(uuid())
ts = TaskSetResult(uuid(), [ar, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def test_iterate_eager(self):
ar1 = EagerResult(uuid(), 42, states.SUCCESS)
ar2 = EagerResult(uuid(), 42, states.SUCCESS)
ts = TaskSetResult(uuid(), [ar1, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def test_join_timeout(self):
ar = MockAsyncResultSuccess(uuid())
ar2 = MockAsyncResultSuccess(uuid())
ar3 = AsyncResult(uuid())
ts = TaskSetResult(uuid(), [ar, ar2, ar3])
self.assertRaises(TimeoutError, ts.join, timeout=0.0000001)
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i, t in enumerate(it):
self.assertEqual(t.get(), i)
def test___iter__(self):
it = iter(self.ts)
results = sorted(list(it))
self.assertListEqual(results, list(xrange(self.size)))
def test_join(self):
joined = self.ts.join()
self.assertListEqual(joined, list(xrange(self.size)))
def test_successful(self):
self.assertTrue(self.ts.successful())
def test_failed(self):
self.assertFalse(self.ts.failed())
def test_waiting(self):
self.assertFalse(self.ts.waiting())
def test_ready(self):
self.assertTrue(self.ts.ready())
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), self.ts.total)
def setup(self):
self.size = 10
self.ts = TaskSetResult(uuid(), make_mock_group(self.app, self.size))
def setUp(self):
self.size = 10
self.ts = TaskSetResult(gen_unique_id(), make_mock_taskset(self.size))
def setup(self):
self.size = 10
self.ts = TaskSetResult(uuid(), make_mock_group(self.size))
def get_taskset_result(task_group_id):
return TaskSetResult.restore(task_group_id)
def setUp(self):
self.ts = TaskSetResult(gen_unique_id(), [
AsyncResult(gen_unique_id()),
AsyncResult(gen_unique_id())])
def test_compat_subtasks_kwarg(self):
x = TaskSetResult(uuid(), subtasks=[1, 2, 3])
self.assertEqual(x.results, [1, 2, 3])
class TestTaskSetResult(unittest.TestCase):
def setUp(self):
self.size = 10
self.ts = TaskSetResult(gen_unique_id(), make_mock_taskset(self.size))
def test_total(self):
self.assertEqual(self.ts.total, self.size)
def test_iterate_raises(self):
ar = MockAsyncResultFailure(gen_unique_id())
ts = TaskSetResult(gen_unique_id(), [ar])
it = iter(ts)
self.assertRaises(KeyError, it.next)
def test_iterate_yields(self):
ar = MockAsyncResultSuccess(gen_unique_id())
ar2 = MockAsyncResultSuccess(gen_unique_id())
ts = TaskSetResult(gen_unique_id(), [ar, ar2])
it = iter(ts)
self.assertEqual(it.next(), 42)
self.assertEqual(it.next(), 42)
def test_join_timeout(self):
ar = MockAsyncResultSuccess(gen_unique_id())
ar2 = MockAsyncResultSuccess(gen_unique_id())
ar3 = AsyncResult(gen_unique_id())
ts = TaskSetResult(gen_unique_id(), [ar, ar2, ar3])
self.assertRaises(TimeoutError, ts.join, timeout=0.0000001)
def test_itersubtasks(self):
it = self.ts.itersubtasks()
for i, t in enumerate(it):
self.assertEqual(t.get(), i)
def test___iter__(self):
it = iter(self.ts)
results = sorted(list(it))
self.assertListEqual(results, list(xrange(self.size)))
def test_join(self):
joined = self.ts.join()
self.assertListEqual(joined, list(xrange(self.size)))
def test_successful(self):
self.assertTrue(self.ts.successful())
def test_failed(self):
self.assertFalse(self.ts.failed())
def test_waiting(self):
self.assertFalse(self.ts.waiting())
def test_ready(self):
self.assertTrue(self.ts.ready())
def test_completed_count(self):
self.assertEqual(self.ts.completed_count(), self.ts.total)
