def test_pop_pending_limit(self):
k = pending_key('test')
for t in range(5, 40, 5):
yield add_pending(self.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(self.redis, k)
pending_reqs = [r for t, r in pending]
result = yield pop_pending(self.redis, 'test', 0, 50, limit=2)
self.assertEqual(result, pending_reqs[:2])
self.assertEqual((yield zitems(self.redis, k)), pending[2:])
result = yield pop_pending(self.redis, 'test', 0, 50, limit=3)
self.assertEqual(result, pending_reqs[2:5])
self.assertEqual((yield zitems(self.redis, k)), pending[5:])
result = yield pop_pending(self.redis, 'test', 0, 50, limit=3)
self.assertEqual(result, pending_reqs[5:])
self.assertEqual((yield zitems(self.redis, k)), [])
result = yield pop_pending(self.redis, 'test', 0, 50, limit=3)
self.assertEqual(result, [])
self.assertEqual((yield zitems(self.redis, k)), [])
def test_retry_timeout_reschedule(self):
k = pending_key('test')
msgs = self.patch_log()
worker = yield self.mk_worker({'timeout': 3})
srv = yield ToyServer.from_test(self)
self.patch_reactor_call_later(worker.clock)
yield worker.stop()
@srv.app.route('/foo')
def route(req):
return Deferred()
req = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10, 20],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
pop_all(msgs)
d = worker.retry(req)
worker.clock.advance(2)
self.assertEqual((yield zitems(worker.redis, k)), [])
worker.clock.advance(4)
yield d
self.assertEqual(pop_all(msgs), [
('Retrying request', req),
('Retry timed out', req),
('Rescheduling retry', req),
])
self.assertEqual((yield zitems(worker.redis, k)), [
(5 + 20, {
'owner_id': '1234',
'timestamp': 5,
'attempts': 1,
'intervals': [10, 20],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}),
])
def test_retry(self):
msgs = self.patch_log()
worker = yield self.mk_worker()
yield worker.stop()
srv = yield ToyServer.from_test(self)
reqs = []
@srv.app.route('/foo')
def route(req):
reqs.append(req)
req = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
pop_all(msgs)
yield worker.retry(req)
self.assertEqual(pop_all(msgs), [
('Retrying request', req),
('Retry successful (200)', req),
])
[req] = reqs
self.assertEqual(req.method, 'POST')
self.assertEqual((yield zitems(worker.redis, pending_key('test'))), [])
def test_add_pending_next_retry(self):
k = pending_key('test')
self.assertEqual((yield zitems(self.redis, k)), [])
yield add_pending(self.redis, 'test', {
'owner_id': '1234',
'timestamp': 10,
'attempts': 1,
'intervals': [50, 60],
'request': {'foo': 23}
})
self.assertEqual((yield zitems(self.redis, k)), [
(10 + 60, {
'owner_id': '1234',
'timestamp': 10,
'attempts': 1,
'intervals': [50, 60],
'request': {'foo': 23},
}),
])
yield add_pending(self.redis, 'test', {
'owner_id': '1234',
'timestamp': 5,
'attempts': 2,
'intervals': [20, 90, 100],
'request': {'bar': 42}
})
self.assertEqual((yield zitems(self.redis, k)), [
(10 + 60, {
'owner_id': '1234',
'timestamp': 10,
'attempts': 1,
'intervals': [50, 60],
'request': {'foo': 23},
}),
(5 + 100, {
'owner_id': '1234',
'timestamp': 5,
'attempts': 2,
'intervals': [20, 90, 100],
'request': {'bar': 42},
}),
])
def test_maintain(self):
k_p = pending_key('test')
k_r = ready_key('test')
worker = yield self.mk_worker({
'redis_prefix': 'test',
'frequency': 20,
})
worker.stop()
for t in range(5, 25, 5):
yield add_pending(worker.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(worker.redis, pending_key('test'))
pending_reqs = [v for t, v in pending]
worker.clock.advance(10)
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
worker.clock.advance(5)
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
worker.clock.advance(10)
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
worker.clock.advance(5)
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
yield worker.maintain()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(worker.redis, k_p)), [])
def test_pop_pending_add_ready(self):
k_p = pending_key('test')
k_r = ready_key('test')
for t in range(5, 40, 5):
yield add_pending(self.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending_reqs = [r for t, r in (yield zitems(self.redis, k_p))]
yield pop_pending_add_ready(self.redis, 'test', 0, 50)
self.assertEqual((yield lvalues(self.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(self.redis, k_p)), [])
def test_retry_end(self):
msgs = self.patch_log()
worker = yield self.mk_worker()
srv = yield ToyServer.from_test(self)
yield worker.stop()
@srv.app.route('/foo')
def route(req):
req.setResponseCode(500)
req1 = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 1,
'intervals': [10, 20],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
req2 = {
'owner_id': '1234',
'timestamp': 10,
'attempts': 2,
'intervals': [10, 30, 40],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
pop_all(msgs)
yield worker.retry(req1)
self.assertEqual(pop_all(msgs), [
('Retrying request', req1),
('Retry failed (500)', req1),
('No remaining retry intervals, discarding request', req1),
])
yield worker.retry(req2)
self.assertEqual(pop_all(msgs), [
('Retrying request', req2),
('Retry failed (500)', req2),
('No remaining retry intervals, discarding request', req2),
])
self.assertEqual((yield zitems(worker.redis, pending_key('test'))), [])
def test_requests(self):
msgs = self.patch_log()
k = pending_key('test')
self.assertEqual(
(yield get_req_count(self.app.redis, 'test', '1234')), 0)
pop_all(msgs)
resp = yield self.post('/requests/', {
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
}
}, headers={'X-Owner-ID': '1234'})
self.assertEqual(resp.code, http.OK)
self.assertEqual((yield resp.content()), json.dumps({}))
self.assertEqual(pop_all(msgs), [
("Adding request to pending set", {
'owner_id': '1234',
'timestamp': 10,
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
},
})
])
self.assertEqual(
(yield get_req_count(self.app.redis, 'test', '1234')), 1)
self.assertEqual((yield zitems(self.app.redis, k)), [
(self.time + 30, {
'attempts': 0,
'timestamp': 10,
'owner_id': '1234',
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
}
}),
])
def test_pop_pending_no_deserialize(self):
k = pending_key('test')
for t in range(5, 35, 5):
yield add_pending(self.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(self.redis, k)
pending_reqs = [r for t, r in pending]
result = yield pop_pending(
self.redis, 'test', 0, 10 + 13, deserialize=False)
self.assertEqual([json.loads(r) for r in result], pending_reqs[:2])
def test_loop(self):
k_p = pending_key('test')
k_r = ready_key('test')
msgs = self.patch_log()
worker = yield self.mk_worker({
'redis_prefix': 'test',
'frequency': 5,
})
for t in range(5, 25, 5):
yield add_pending(worker.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(worker.redis, pending_key('test'))
pending_reqs = [v for t, v in pending]
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.maintains.pop()
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:2])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[2:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(worker.redis, k_p)), [])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
self.assertEqual(worker.maintains, [])
worker.stop()
worker.clock.advance(5)
self.assertEqual(worker.maintains, [])
self.assertEqual(pop_all(msgs), [])