def _generate_test_chain(n):
gt_result = n * (1 + n) / 2
first = add.s(0, 0)
body = [add.s(i + 1) for i in range(N)]
c = chain(first, *body)
return c, gt_result
def test_delay():
result = add.delay(2, 2).get()
log.info('add.delay 2, 2) = %s' % result)
result = add.s(2, 2).delay().get()
log.info('add.s(2, 2).delay().get() = %s' % result)
result = add.s(2).delay(12).get()
print(result)
def test_grouping():
result = group(add.s(i, i) for i in range(10, 20))().get()
print(result)
result = chain(add.s(2, 2) | mul.s((4)))().get()
print(result)
result = chord((mul.s(i, i) for i in range(10)), xsum.s())().get()
print(result)
result = (group(mul.s(i, i) for i in range(10)) | xsum.s())().get()
print(result)
def main():
while True:
time.sleep(5)
print("Adding work to queue")
chain = add.s(1, 2) | step2.s()
res = chain()
for result, value in res.collect(intermediate=True):
print(value)
def test_group_results():
job = group([
add.s(2, 4),
mul.s(4, 9),
xsum.s([2, 3, 5]),
])
result = job.apply_async()
print(result.ready())
print(result.successful())
print(result.join())
print(result.get())
def _generate_test_chord(n):
gt_result = n
task = add.si(1, 0)
def _generate_chord_inner(n):
if n <= 0:
return tsum.s()
else:
return chord(header=[tsum.s(), task.clone()],
body=_generate_chord_inner(n - 1))
return chord(header=add.s(0, 1),
body=_generate_chord_inner(n - 1)), gt_result
def arrange_tasks():
# normal call
# add(0, 10)
# send task message
add.delay(1, 10)
add.apply_async((2, 10))
# executes 10 seconds from now
add.apply_async((3, 10), countdown=5)
# executes 10 seconds from now - eta
add.apply_async((4, 10), eta=datetime.utcnow() + timedelta(seconds=6))
# linking(callbacks / errbacks)
add.apply_async((5, 10), link=add.s(7))
def do_job1(self, feature_hash):
'''
function that mocks doing a specific job, it will pick
up a worker based on some policy, here just hardcoding
the policy
this job is doing (2 + 2) * 3 - 6 == 6, in the first queue via WORKER1
@param {string} feature_hash - the feature hash that representing a series of tasks
'''
self._configure_routing(feature_hash, WORKERS["WORKER1"])
# print("job1: %s" % self._celery_app.control.inspect().active())
job = chain(
add.s(2, 2).set(queue=feature_hash, routing_key=feature_hash),
mult.s(3).set(queue=feature_hash, routing_key=feature_hash),
minus.s(6).set(queue=feature_hash, routing_key=feature_hash))
job.delay()
from tasks import add
from celery import group
# se lanzar 100 tareas
for i in range(1, 100):
result = add.apply_async((i, i), )
print(i, result.get())
# se lanzan 4 tareas dentro de un grupo
numbers = [(2, 2), (4, 4), (8, 8), (16, 16)]
res = group(add.s(i, j) for i, j in numbers).apply_async(queue='priority.high',
serializer="json")
# aquí tenemos el resultado de las tareas
print(res.get())
# raise exception still can run
# res = (
# group(
# (add.s(4, 4) | mul.s(2)), # 16
# (add.s(5, 4) | mul.s(3)), # 27
# (add.s(5, 5) | mul.s(4)) # 40
# )
# | xsum.s() # 83
# )
# res.apply_async()
# raise exception still can run, and call handle error function
res = (
group(
(add.s(4, 4) | mul.s(2)), # 16
(add.s(5, 4) | mul.s(3)), # 27
(add.s(5, 5) | mul.s(4)) # 40
)
| xsum.s() # 83
)
res.apply_async(link_error=report_error())
# raise exception then it's run forever
# res = (
# group(
# (add.s(4, 4) | mul.s(2)), # 16
# (add.s(5, 4) | mul.s(3)), # 27
# (add.s(5, 5) | mul.s(4)) # 40
# )
# | xsum.s() # 83
from tasks import add from celery import group r = group(add.s(i, i) for i in range(2))() print(r.get())
def grouped():
# this will run in concurrent according to celery concurrency config
group(add.s(i, i) for i in xrange(10))()
return "grouped"
from celery import chord from tasks import add, tsum print chord(add.s(i, i) for i in xrange(100))(tsum.s()).get()
from tasks import add, mul, build_insight from celery import chain res = (add.s(4, 4) | mul.s(4)).apply_async() # build_insight.delay()
from tasks import add from celery import group r = group(add.s(i, i) for i in range(1000000)).apply_async() print(r.get())
from tasks import add, xsum # Basic example res = add.delay(5, 5) print(res.get()) # Same res = add.apply_async((5, 5)) print(res.get()) # Subprocess signature = add.s(5, 5) res = signature.apply_async() # res = signature.delay() print(res.get()) # Callback Example res = add.apply_async((2, 2), link=add.s(8)) print(res.get()) # Chaining from celery import chain chaining = chain(add.s(2, 2), add.s(4), add.s(8)) # 2 + 2 + 4 + 8 res = chaining.delay() print(res.get()) chaining = add.s(2, 2) | add.s(4) | add.s(8) res = chaining.delay() print(res.get()) # Grouping (Parallel)
from celery import group, chord from tasks import add, reduce_ import time callback = reduce_.s() header = [add.s(i, i) for i in range(10)] res = chord(header)(callback) a = res.get(timeout=1) print(a)
from tasks import throw from celery import chord, group from tasks import add, throw c = chord([add.s(4, 4), throw.s(), add.s(8, 8)])
def test():
res = add.s(2, 2).delay()
time.sleep(1)
print(res.state)
print(res.result)
# run_chain.py
from tasks import add
from celery import chain
# task signature 생성
sig1 = add.s(3, 5)
sig2 = add.s(8)
sig3 = add.s(2)
# chain signature 생성
sig_chain = chain(sig1, sig2, sig3)
print(f"chain: {sig_chain}")
# 비동기 작업 실행
res = sig_chain()
print(" 작업 ID 조회 ".center(50, "="))
print(f" chain: {res.id}")
print(" 작업 완료여부 조회 ".center(50, "="))
print(f" chain: {res.ready()}")
print(" 결과 데이터 조회 (완료될때까지 Pause) ".center(50, "="))
print(f" chain: {res.get()}")
print(" 작업 완료여부 조회 ".center(50, "="))
print(f" chain: {res.ready()}")
from celery import chain, group, chord
from tasks import add, xsum, raise_error, on_chord_error
if __name__ == '__main__':
## chain primitive
res = chain(add.s(2, 2), add.s(4), add.s(8))()
print('Chain result: %s' % res.get())
# shortcut of above
res = (add.s(2, 2) | add.s(4) | add.s(8))()
print('Chain shortcut: %s' % res.get())
res = chain(add.si(2, 2), add.si(4, 5), add.si(8, 8))()
print('Chain with independent task: %s' % res.get()) # 8 + 8
print('Parent result: %s' % res.parent.get()) # 4 + 5
print('Parent of parent result: %s' % res.parent.parent.get()) # 2 + 2
## group primitive
res = group(add.s(i, i) for i in range(10))()
print('Group result: %s' % res.get())
## chord primitive
res = chord((add.s(i, i) for i in range(10)), xsum.s())()
# is equal to: group(add.s(i, i) for i in range(10)) | xsum.s()
print('Chord result: %s' % res.get())
res = chord([add.s(2, 2), raise_error.s(), add.s(4, 4)], xsum.s())()
from celery import signature
from tasks import add
if __name__ == '__main__':
# shorcut: add.s(2, 2)
# or: signature('tasks.add', args=(2, 2), countdown=10)
sig = add.signature((2, 2), countdown=3)
print(sig, json.dumps(sig))
# call local
ret = sig()
print('Call local: %s' % ret)
# call remote
# NOTE: ~sig is equal to sig.delay().get()
async_ret = sig.delay()
ret = async_ret.get()
print('Call remote: %s' % ret)
# paitial
partial = add.s(2)
async_ret = partial.delay(3)
ret = async_ret.get()
print('Partial: %s' % ret)
async_ret = add.apply_async((2, 2),
link=add.signature((3, 3), immutable=True))
ret = async_ret.get()
print('Use signature in task link: %s' % ret)
# run_group.py
from tasks import add
from celery import group
# task signature 생성
sigs = [add.s(i, i + 1) for i in range(5)]
# group signature 생성
job = group(sigs)
print(f"group: {job}")
res = job()
print(" 작업 조회 ".center(50, "="))
print(f" task id: {res.id}")
print(f" result type: {type(res)}")
print(" 상태 조회 ".center(50, "="))
print(f" task ready: {res.ready()}")
print(" 결과 반환 ".center(50, "="))
print(f" task completed: {res.completed_count()}") # 완료한 subtask 갯수 출력
print(f" task results: {res.get()}")
print(f" task results: {res.join()}") # 호출한 순서대로 리턴
print(f" task completed: {res.completed_count()}")
print(" 상태 조회 ".center(50, "="))
print(f" task ready: {res.ready()}")
print(f" task successful: {res.successful()}") # 모든 작업이 성공적으로 완료되었는 지
from tasks import add, append, xsum
from celery import chord, group
print("HERE")
res = chord((add.s(i, i) for i in range(10)), xsum.s())()
print(res.get())
res = group(add.s(i, i) for i in range(10))()
print(res.get())
res = chord(add.s(x, x) for x in range(7))(append.si('complete and '),
interval=1)
print(res.get())
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from tasks import add
import celery
ids = []
for i in range(1000):
ids.append(i)
rs = []
for i in ids:
r = add.s([i, 2], [i + 1, 3])
rs.append(r)
jobs = celery.group(rs)
results = jobs.apply_async()
print results.get()
def chained():
chain(add.s(1, 2) | add.s(1) | add.s(2))()
return "chained"
from tasks import add, sleep, xsum, error, dummy
from celery import chord
from celery import Celery
if __name__ == '__main__':
# x = chord((add.s(i, i) for i in xrange(10)), sleep.si())
# print type(x)
# x()
# x = add.delay(1, 2).get()
c = add.s(1, 2) | add.s(3) | error.s() | dummy.s()
print c()
# sleep.delay()
# chain()
# print add.name
# add.delay(4, 4)
# app = Celery('tasks', broker='redis://localhost:6379/0')
# print app.control.inspect().active_queues()
# app.control.add_consumer('add', reply=True)
import time from celery import group, chain, chord from tasks import sendmail, add, take, no_argument, xsum # 默认一条celery队列，但是在任务执行的过程中，因为只有一条队列，所以任务执行是同步的。 result = sendmail.delay(dict(to='www.com')) # result1 = sendmail.apply_async(args=(dict(to='*****@*****.**'), )) # result2 = add.apply_async(args=(2, 3)) # 多条队列同步进行，需要开启多个路由, 需要在后面传入队列参数queue result3 = take.apply_async(args=(10, 1), queue='web_tasks') # 关联任务， 将前一个任务的结果作为参数传入下一个任务 result4 = add.apply_async(args=(2, 2), link=add.s(3), queue='default') # # 关联任务， 将前一个任务的结果作为参数传入下一个任务 # result5 = add.apply_async(args=(2, 2), link=add.s(4), queue='default') # # 关联任务， 将前一个任务的结果作为参数传入下一个任务 result6 = add.apply_async(args=(2, 2), link=no_argument.si(), queue='default') # # 关联任务， 将前一个任务的结果作为参数传入下一个任务 result7 = add.apply_async(args=(2, 2), link=no_argument.signature(immutable=True), queue='default') # 过期时间 result8 = add.apply_async(args=(2, 3), expires=10, queue='default') # 并行调度， 结果返回列表 result9 = group(add.s(i, i) for i in range(10))(queue='default') # 串行调度， 结果16 result10 = chain(add.s(2, 2), add.s(4), add.s(8))() # chord - 带回调的 group #result11 = chord((add.s(i, i) for i in range(10)), xsum.s())(queue='default')
