class HueyQueue(BaseQueue):
def __init__(self, options):
BaseQueue.__init__(self, options)
# TODO: We should rethink ``store_errors=False`` -- how do we notice errors?
self._huey = RedisHuey(result_store=False,
store_none=False,
store_errors=False,
**options)
self._jobs = {}
@property
def huey(self):
return self._huey
@property
def jobs(self):
return self._jobs
def register_job(self, name, func):
if name in self._jobs:
raise QueueError(u"Job function {!r} already exists".format(name))
self._jobs[name] = self._huey.task(name=name)(func)
def enqueue(self, name, args, kwargs):
if name not in self._jobs:
raise QueueError(u"Unknown job: {!r}".format(name))
log.info(u"Sending {!r} job to the queue ...".format(name))
# We do not care about results
self._jobs[name](*args, **kwargs)
def test_task_decorators(self):
huey = RedisHuey()
def test_fn():
pass
test_fn_task = huey.task()(test_fn)
test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn)
self.assertTrue(isinstance(test_fn_task, TaskWrapper))
self.assertTrue(test_fn_task.func is test_fn)
self.assertTrue(isinstance(test_fn_cron, TaskWrapper))
self.assertTrue(test_fn_cron.func is test_fn)
test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task)
self.assertTrue(isinstance(test_cron_task, TaskWrapper))
self.assertTrue(test_cron_task.func is test_fn)
def test_task_decorators(self):
huey = RedisHuey()
def test_fn():
pass
test_fn_task = huey.task()(test_fn)
test_fn_cron = huey.periodic_task(crontab(minute='0'))(test_fn)
self.assertTrue(isinstance(test_fn_task, TaskWrapper))
self.assertTrue(test_fn_task.func is test_fn)
self.assertTrue(isinstance(test_fn_cron, TaskWrapper))
self.assertTrue(test_fn_cron.func is test_fn)
test_cron_task = huey.periodic_task(crontab(minute='0'))(test_fn_task)
self.assertTrue(isinstance(test_cron_task, TaskWrapper))
self.assertTrue(test_cron_task.func is test_fn)
state['last_task_class'] = type(task).__name__
class PutTask(QueueTask):
def execute(self):
k, v = self.data
state[k] = v
registry.register(PutTask)
class TestException(Exception):
pass
def _throw_error_task(message=None):
raise TestException(message or 'bampf')
throw_error_task = huey.task()(_throw_error_task)
throw_error_task_res = huey_results.task()(_throw_error_task)
@huey_results.task()
def add_values(a, b):
return a + b
@huey_results.task()
def add_values2(a, b):
return a + b
@huey_results.periodic_task(crontab(minute='0'))
def hourly_task2():
state['periodic'] = 2
@huey_results.task()
state['last_task_class'] = type(task).__name__
class PutTask(QueueTask):
def execute(self):
k, v = self.data
state[k] = v
registry.register(PutTask)
class TestException(Exception):
pass
def _throw_error_task(message=None):
raise TestException(message or 'bampf')
throw_error_task = huey.task()(_throw_error_task)
throw_error_task_res = huey_results.task(name='error_res')(_throw_error_task)
@huey_results.task()
def add_values(a, b):
return a + b
@huey_results.task()
def add_values2(a, b):
return a + b
@huey_results.periodic_task(crontab(minute='0'))
def hourly_task2():
state['periodic'] = 2
@huey_results.task()
bad = 1/0
state['last_task_class'] = type(task).__name__
class PutTask(QueueTask):
def execute(self):
k, v = self.data
state[k] = v
class TestException(Exception):
pass
@huey.task()
def _throw_error_task(message=None):
raise TestException(message or 'bampf')
throw_error_task = huey.task()(_throw_error_task)
throw_error_task_res = huey_results.task()(_throw_error_task)
@huey_results.task()
def add_values(a, b):
return a + b
@huey_results.periodic_task(crontab(minute='0'))
def hourly_task2():
state['periodic'] = 2
@huey_results.task()
def returns_none():
return None
@huey_store_none.task()
if config.REDIS_SSL:
redis_kwargs = dict(connection_class=SSLConnection, ssl_cert_reqs=None)
else:
redis_kwargs = dict()
connection_pool = ConnectionPool(
host=config.REDIS_HOST,
port=config.REDIS_PORT,
password=config.REDIS_PASSWORD,
**redis_kwargs,
)
huey = RedisHuey(connection_pool=connection_pool)
# Normal task, no retries
nonretriable_task = huey.task()
# These tasks retry every 10 minutes for a day.
retriable_task = huey.task(retries=(6 * 24), retry_delay=(60 * 10))
@huey.pre_execute(name="Set Correlation ID")
def register_correlation_id(task):
args, kwargs = task.data
correlation_id = kwargs.pop("correlation_id", "Rogue Task")
cf_logging.FRAMEWORK.context.set_correlation_id(correlation_id)
@huey.signal()
def log_task_transition(signal, task, exc=None):
args, kwargs = task.data
elif follower.id not in answerers:
for _, act in zip(r, follower.activities):
if act.type == FOLLOW_QUESTION and str(act.content.id) == qid:
new_followers.appendleft({
'uid': follower.id,
'time': act.time
})
new_follower_uids.add(follower.id)
fetch_followers_followees(follower.id, now)
break
elif act.type == ANSWER_QUESTION and str(act.content.question.id) == qid:
# 有可能这个时候 answerer 还没有入库
break
else:
logger.warning("Can't find follow question activity")
logger.warning("question: %s, follower: %s" % (qid, follower.id))
# 没有具体时间，就不记录。因为follower有序，时间可之后推定。
new_followers.appendleft({
'uid': follower.id,
'time': None
})
new_follower_uids.add(follower.id)
QuestionManager.add_question_follower(tid, qid, new_followers)
fetch_followers = huey.task(retries=3, retry_delay=2)(_fetch_followers)
fetch_followees = huey.task(retries=3, retry_delay=2)(_fetch_followees)
fetch_followers_followees = huey.task(retries=3, retry_delay=2)(_fetch_followers_followees)
fetch_question_follower = huey.task(retries=3, retry_delay=2)(_fetch_question_follower)
replace_database = huey.task()(_replace_database)
