def load_tasks():
tasks = []
for ctype, base in class_map.iteritems():
if ctype not in active:
continue
for task in registry[ctype]:
fullname = 'pipeline.tasks.{}_{}'.format(ctype, task)
wrap2 = app.task(base=base, name=fullname, **task_kwargs)
wrap1 = taskcall(task)
def stub(*args, **kwargs):
pass
tasks.append(wrap2(wrap1(stub)))
return tasks
from redis import Redis
from app import app, router
@router.node(('word', ), entry_point=True)
def emit_words(msg):
for word in msg.document.strip().split(' '):
yield word
@router.node(('word', 'count'),
subscribe_to='tasks.emit_words',
celery_task=app.task(rate_limit='5/s'))
def tally_word(msg):
redis = Redis()
return msg.word, redis.zincrby('celery_emit_example', msg.word, 1)
from redis import Redis
from app import app, router
@router.node(('word',), entry_point=True)
def emit_words(msg):
for word in msg.document.strip().split(' '):
yield word
@router.node(('word', 'count'), subscribe_to='tasks.emit_words', celery_task=app.task(rate_limit='5/s'))
def tally_word(msg):
redis = Redis()
return msg.word, redis.zincrby('celery_emit_example', msg.word, 1)