def test_pop_ready(self):
k = ready_key('test')
req1 = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10],
'request': {'foo': 23}
}
req2 = {
'owner_id': '1234',
'timestamp': 10,
'attempts': 0,
'intervals': [10],
'request': {'bar': 42}
}
yield add_ready(self.redis, 'test', [req1, req2])
self.assertEqual((yield lvalues(self.redis, k)), [req1, req2])
result = yield pop_ready(self.redis, 'test')
self.assertEqual(result, req1)
self.assertEqual((yield lvalues(self.redis, k)), [req2])
result = yield pop_ready(self.redis, 'test')
self.assertEqual(result, req2)
self.assertEqual((yield lvalues(self.redis, k)), [])
result = yield pop_ready(self.redis, 'test')
self.assertEqual(result, None)
self.assertEqual((yield lvalues(self.redis, k)), [])
def test_maintain(self):
k_p = pending_key('test')
k_r = ready_key('test')
worker = yield self.mk_worker({
'redis_prefix': 'test',
'frequency': 20,
})
worker.stop()
for t in range(5, 25, 5):
yield add_pending(worker.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(worker.redis, pending_key('test'))
pending_reqs = [v for t, v in pending]
worker.clock.advance(10)
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
worker.clock.advance(5)
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
worker.clock.advance(10)
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
worker.clock.advance(5)
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(worker.redis, k_p)), [])
def test_pop_pending_add_ready(self):
k_p = pending_key('test')
k_r = ready_key('test')
for t in range(5, 40, 5):
yield add_pending(self.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending_reqs = [r for t, r in (yield zitems(self.redis, k_p))]
yield pop_pending_add_ready(self.redis, 'test', 0, 50)
self.assertEqual((yield lvalues(self.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(self.redis, k_p)), [])
def test_add_ready(self):
k = ready_key('test')
req1 = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10],
'request': {'foo': 23}
}
req2 = {
'owner_id': '1234',
'timestamp': 10,
'attempts': 0,
'intervals': [10],
'request': {'bar': 42}
}
req3 = {
'owner_id': '1234',
'timestamp': 15,
'attempts': 0,
'intervals': [10],
'request': {'baz': 21}
}
self.assertEqual((yield lvalues(self.redis, k)), [])
yield add_ready(self.redis, 'test', [])
self.assertEqual((yield lvalues(self.redis, k)), [])
yield add_ready(self.redis, 'test', [req1])
self.assertEqual((yield lvalues(self.redis, k)), [req1])
yield add_ready(self.redis, 'test', [req2, req3])
self.assertEqual((yield lvalues(self.redis, k)), [req1, req2, req3])
def test_add_ready_no_serialize(self):
k = ready_key('test')
req1 = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10],
'request': {'foo': 23}
}
req2 = {
'owner_id': '1234',
'timestamp': 10,
'attempts': 0,
'intervals': [10],
'request': {'bar': 42}
}
yield add_ready(
self.redis, 'test', [json.dumps(req1), json.dumps(req2)],
serialize=False)
self.assertEqual((yield lvalues(self.redis, k)), [req1, req2])
def test_loop(self):
k = ready_key('test')
msgs = self.patch_log()
retries = self.patch_retry()
worker = yield self.mk_worker({'frequency': 5})
reqs = [{
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
} for t in range(5, 30, 5)]
yield add_ready(worker.redis, 'test', reqs)
self.assertEqual(pop_all(msgs), [
('Polling for requests to retry',),
('Retrieving next request from ready set',),
('Ready set is empty, rechecking on next poll',),
])
worker.clock.advance(5)
req = yield retries.get()
self.assertEqual(req, reqs[0])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[1:])
self.assertEqual(pop_all(msgs), [
('Polling for requests to retry',),
('Retrieving next request from ready set',),
('Scheduling request for retrying', reqs[0]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[1])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[2:])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[1]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[2])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[3:])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[2]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[3])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[4:])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[3]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[4])
self.assertEqual((yield lvalues(worker.redis, k)), [])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[4]),
('Retrieving next request from ready set',),
])
worker.clock.advance(10)
reqs = [{
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
} for t in range(5, 15, 5)]
yield add_ready(worker.redis, 'test', reqs)
self.assertEqual(pop_all(msgs), [
('Ready set is empty, rechecking on next poll',),
])
worker.clock.advance(5)
req = yield retries.get()
self.assertEqual(req, reqs[0])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[1:])
self.assertEqual(pop_all(msgs), [
('Polling for requests to retry',),
('Retrieving next request from ready set',),
('Scheduling request for retrying', reqs[0]),
('Retrieving next request from ready set',),
])
worker.clock.advance(5)
req = yield retries.get()
self.assertEqual(req, reqs[1])
self.assertEqual((yield lvalues(worker.redis, k)), [])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[1]),
('Retrieving next request from ready set',),
])
def test_loop(self):
k_p = pending_key('test')
k_r = ready_key('test')
msgs = self.patch_log()
worker = yield self.mk_worker({
'redis_prefix': 'test',
'frequency': 5,
})
for t in range(5, 25, 5):
yield add_pending(worker.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(worker.redis, pending_key('test'))
pending_reqs = [v for t, v in pending]
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.maintains.pop()
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:2])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[2:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(worker.redis, k_p)), [])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
self.assertEqual(worker.maintains, [])
worker.stop()
worker.clock.advance(5)
self.assertEqual(worker.maintains, [])
self.assertEqual(pop_all(msgs), [])