def test_retries_with_success(self):
# this will fail once, then succeed
retry_task('blampf', False)
self.assertFalse('blampf' in state)
task = test_huey.dequeue()
with CaptureLogs() as capture:
self.worker(task)
self.assertLogs(capture, [
'Executing',
'Unhandled',
'Re-enqueueing'])
task = test_huey.dequeue()
self.assertEqual(task.retries, 2)
self.worker(task)
self.assertEqual(state['blampf'], 'fixed')
self.assertEqual(len(test_huey.queue), 0)
self.assertTaskEvents(
('started', task),
('error', task),
('retrying', task),
('enqueued', task),
('started', task),
('finished', task))
def test_metadata(self):
def run_task(fn, a=()):
fn(*a)
self.worker(test_huey.dequeue())
return test_huey.metadata()
metadata = run_task(modify_state, ('k1', 'v1'))
self.assertEqual(int(metadata[b('queuecmd_modify_state_executed')]), 1)
self.assertEqual(int(metadata[b('tasks_executed')]), 1)
self.assertEqual(metadata[b('queuecmd_modify_state_duration')],
metadata[b('tasks_duration')])
metadata = run_task(modify_state, ('k1', 'v2'))
self.assertEqual(int(metadata[b('queuecmd_modify_state_executed')]), 2)
self.assertEqual(int(metadata[b('tasks_executed')]), 2)
metadata = run_task(blow_up)
self.assertEqual(int(metadata[b('queuecmd_blow_up_errors')]), 1)
self.assertFalse(b('queuecmd_blow_up_executed') in metadata)
self.assertEqual(int(metadata[b('tasks_executed')]), 2)
self.assertEqual(int(metadata[b('tasks_errors')]), 1)
metadata = run_task(retry_task, ('test', False))
self.assertEqual(int(metadata[b('queuecmd_retry_task_errors')]), 1)
self.assertFalse(b('queuecmd_retry_task_executed') in metadata)
self.assertEqual(int(metadata[b('tasks_executed')]), 2)
self.assertEqual(int(metadata[b('tasks_errors')]), 2)
# Duration is recorded for errors.
duration = metadata[b('queuecmd_retry_task_duration')]
self.worker(test_huey.dequeue())
metadata = test_huey.metadata()
self.assertEqual(int(metadata[b('queuecmd_retry_task_errors')]), 1)
self.assertEqual(int(metadata[b('queuecmd_retry_task_executed')]), 1)
self.assertEqual(int(metadata[b('tasks_executed')]), 3)
self.assertEqual(int(metadata[b('tasks_errors')]), 2)
self.assertNotEqual(metadata[b('queuecmd_retry_task_duration')], duration)
# Scheduled, ready to run when dequeued -- runs like normal.
modify_state.schedule(args=('k1', 'v3'), eta=datetime.date(2015, 1, 1))
self.worker(test_huey.dequeue())
metadata = test_huey.metadata()
self.assertEqual(int(metadata[b('queuecmd_modify_state_executed')]), 3)
self.assertEqual(int(metadata[b('tasks_executed')]), 4)
# When task is put on schedule and not executed immediately, then
# the `scheduled` metadata count is incremented.
modify_state.schedule(args=('k1', 'v3'), eta=datetime.date(2030, 1, 1))
self.worker(test_huey.dequeue())
metadata = test_huey.metadata()
self.assertEqual(int(metadata[b('queuecmd_modify_state_executed')]), 3)
self.assertEqual(int(metadata[b('queuecmd_modify_state_scheduled')]), 1)
self.assertEqual(int(metadata[b('tasks_executed')]), 4)
def test_task_locking(self):
ret = locked_task(1, 2)
task = test_huey.dequeue()
self.worker(task)
self.assertEqual(ret.get(), 3)
ret = locked_task(2, 3)
task = test_huey.dequeue()
with test_huey.lock_task('test-lock'):
self.worker(task)
self.assertRaises(TaskException, ret.get)
def test_retry_with_task(self):
retry_with_task(1, -2)
task = test_huey.dequeue()
with CaptureLogs() as capture:
self.worker(task)
task = test_huey.dequeue()
self.worker(task)
self.assertEqual(len(test_huey), 0)
ret = retry_with_task(1, 1)
self.worker(test_huey.dequeue())
self.assertEqual(ret.get(), 2)
self.assertEqual(len(test_huey), 0)
def test_retries_and_logging(self):
# This will continually fail.
retry_task('blampf')
for i in reversed(range(4)):
task = test_huey.dequeue()
self.assertEqual(task.retries, i)
with CaptureLogs() as capture:
self.worker(task)
if i > 0:
self.assertLogs(capture, [
'Executing',
'Unhandled',
'Re-enqueueing'])
self.assertTaskEvents(
('started', task),
('error', task),
('retrying', task),
('enqueued', task))
else:
self.assertLogs(capture, [
'Executing',
'Unhandled'])
self.assertTaskEvents(
('started', task),
('error', task))
self.assertEqual(len(test_huey.queue), 0)
def test_revoking_schedule(self):
global state
dt = datetime.datetime(2011, 1, 1)
dt2 = datetime.datetime(2037, 1, 1)
r1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt, convert_utc=False)
r3 = modify_state.schedule(args=('k3', 'v3'), eta=dt2, convert_utc=False)
r4 = modify_state.schedule(args=('k4', 'v4'), eta=dt2, convert_utc=False)
# revoke r1 and r3
r1.revoke()
r3.revoke()
self.assertTrue(test_huey.is_revoked(r1.task))
self.assertFalse(test_huey.is_revoked(r2.task))
self.assertTrue(test_huey.is_revoked(r3.task))
self.assertFalse(test_huey.is_revoked(r4.task))
expected = [
#state, schedule
({}, 0),
({'k2': 'v2'}, 0),
({'k2': 'v2'}, 1),
({'k2': 'v2'}, 2),
]
for i in range(4):
curr_state, curr_sched = expected[i]
# dequeue a *single* message
task = test_huey.dequeue()
self.worker(task)
self.assertEqual(state, curr_state)
self.assertEqual(len(test_huey.schedule), curr_sched)
# lets pretend its 2037
future = dt2 + datetime.timedelta(seconds=1)
self.scheduler(future)
self.assertEqual(len(test_huey.schedule), 0)
# There are two tasks in the queue now (r3 and r4) -- process both.
for i in range(2):
task = test_huey.dequeue()
self.worker(task, future)
self.assertEqual(state, {'k2': 'v2', 'k4': 'v4'})
def test_periodic_with_retry(self):
dt = datetime.datetime(2011, 1, 1, 0, 3)
sched = self.scheduler(dt, True)
self.assertEqual(sched._counter, 1)
self.assertEqual(sched._q, 6)
self.assertEqual(state, {})
self.assertEqual(len(self.huey), 1)
task = test_huey.dequeue()
self.assertEqual(task.retries, 3)
self.worker(task, dt)
# Exception occurred, so now we retry.
self.assertEqual(len(self.huey), 1)
task = test_huey.dequeue()
self.assertEqual(task.retries, 2)
self.worker(task, dt)
self.assertEqual(state, {'p2': 2})
def test_revoking_normal(self):
# enqueue 2 normal commands
r1 = modify_state('k', 'v')
r2 = modify_state('k2', 'v2')
# revoke the first *before it has been checked*
r1.revoke()
self.assertTrue(test_huey.is_revoked(r1.task))
self.assertFalse(test_huey.is_revoked(r2.task))
# dequeue a *single* message (r1)
task = test_huey.dequeue()
self.worker(task)
self.assertTaskEvents(('revoked', r1.task))
# no changes and the task was not added to the schedule
self.assertFalse('k' in state)
# dequeue a *single* message
task = test_huey.dequeue()
self.worker(task)
self.assertTrue('k2' in state)
def test_explicit_retry(self):
explicit_retry('foo')
self.assertFalse('foo' in state)
task = test_huey.dequeue()
with CaptureLogs() as capture:
self.worker(task)
self.assertLogs(capture, ['Executing', 'Re-enqueueing'])
task = test_huey.dequeue()
self.assertEqual(task.retries, 1)
self.worker(task)
self.assertEqual(state['foo'], 'fixed')
self.assertEqual(len(test_huey), 0)
self.assertTaskEvents(
('started', task),
('retrying', task),
('started', task),
('finished', task))
explicit_retry('bar')
task = test_huey.dequeue()
self.worker(task)
del state['bar']
task = test_huey.dequeue()
self.worker(task)
del state['bar']
task = test_huey.dequeue()
with CaptureLogs() as capture:
self.worker(task)
self.assertLogs(capture, ['Executing', 'Cannot retry task'])
self.assertEqual(len(test_huey), 0)
def test_task_exception(self):
ret = blow_up()
task = test_huey.dequeue()
self.worker(task)
# Calling ".get()" on a task result will raise an exception if the
# task failed.
self.assertRaises(TaskException, ret.get)
try:
ret.get()
except Exception as exc:
self.assertTrue('blowed up' in exc.metadata['error'])
else:
assert False, 'Should not reach this point.'
def test_worker_exception(self):
with CaptureLogs() as capture:
blow_up()
task = test_huey.dequeue()
# Nothing happens because the task is not executed.
self.assertLogs(capture, [])
with CaptureLogs() as capture:
self.worker(task)
self.assertLogs(capture, [
'Executing',
'Unhandled exception in worker'])
self.assertTaskEvents(
('started', task),
('error', task))
def test_periodic_scheduler(self):
dt = datetime.datetime(2011, 1, 3, 3, 7)
sched = self.scheduler(dt, False)
self.assertEqual(sched._counter, 1)
self.assertEqual(sched._q, 5)
self.assertEqual(len(self.huey), 0)
dt = datetime.datetime(2011, 1, 1, 0, 2)
sched = self.scheduler(dt, True)
self.assertEqual(sched._counter, 0)
self.assertEqual(sched._q, 5)
self.assertEqual(state, {})
for i in range(len(self.huey.queue)):
task = test_huey.dequeue()
self.worker(task, dt)
self.assertEqual(state, {'p': 'y'})
def test_worker(self):
modify_state('k', 'w')
task = test_huey.dequeue()
self.worker(task)
self.assertEqual(state, {'k': 'w'})
def loop_periodic(ts):
self.scheduler(ts, True)
for i in range(len(self.huey)):
task = test_huey.dequeue()
self.worker(task, ts)
def run_task(fn, a=()):
fn(*a)
self.worker(test_huey.dequeue())
return test_huey.metadata()
def loop_periodic(ts):
self.scheduler(ts, True)
for i in range(len(self.huey.queue)):
task = test_huey.dequeue()
self.worker(task, ts)
def test_worker(self):
modify_state('k', 'w')
task = test_huey.dequeue()
self.worker(task)
self.assertEqual(state, {'k': 'w'})
