def test_retry(self):
msgs = self.patch_log()
worker = yield self.mk_worker()
yield worker.stop()
srv = yield ToyServer.from_test(self)
reqs = []
@srv.app.route('/foo')
def route(req):
reqs.append(req)
req = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
pop_all(msgs)
yield worker.retry(req)
self.assertEqual(pop_all(msgs), [
('Retrying request', req),
('Retry successful (200)', req),
])
[req] = reqs
self.assertEqual(req.method, 'POST')
self.assertEqual((yield zitems(worker.redis, pending_key('test'))), [])
def test_requests_limit_reached(self):
msgs = self.patch_log()
yield set_req_count(self.app.redis, 'test', '1234', 500)
pop_all(msgs)
resp = yield self.post('/requests/', {
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
}
}, headers={'X-Owner-ID': '1234'})
self.assertEqual(pop_all(msgs), [
("Request limit reached for 1234 at a request count of 500",)
])
self.assertEqual(resp.code, 429)
self.assertEqual(json.loads((yield resp.content())), {
'errors': [{
'type': 'too_many_requests',
'message': "Only 500 unfinished requests are "
"allowed per owner"
}]
})
def test_retry_end(self):
msgs = self.patch_log()
worker = yield self.mk_worker()
srv = yield ToyServer.from_test(self)
yield worker.stop()
@srv.app.route('/foo')
def route(req):
req.setResponseCode(500)
req1 = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 1,
'intervals': [10, 20],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
req2 = {
'owner_id': '1234',
'timestamp': 10,
'attempts': 2,
'intervals': [10, 30, 40],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
pop_all(msgs)
yield worker.retry(req1)
self.assertEqual(pop_all(msgs), [
('Retrying request', req1),
('Retry failed (500)', req1),
('No remaining retry intervals, discarding request', req1),
])
yield worker.retry(req2)
self.assertEqual(pop_all(msgs), [
('Retrying request', req2),
('Retry failed (500)', req2),
('No remaining retry intervals, discarding request', req2),
])
self.assertEqual((yield zitems(worker.redis, pending_key('test'))), [])
def test_retry_timeout_reschedule(self):
k = pending_key('test')
msgs = self.patch_log()
worker = yield self.mk_worker({'timeout': 3})
srv = yield ToyServer.from_test(self)
self.patch_reactor_call_later(worker.clock)
yield worker.stop()
@srv.app.route('/foo')
def route(req):
return Deferred()
req = {
'owner_id': '1234',
'timestamp': 5,
'attempts': 0,
'intervals': [10, 20],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}
pop_all(msgs)
d = worker.retry(req)
worker.clock.advance(2)
self.assertEqual((yield zitems(worker.redis, k)), [])
worker.clock.advance(4)
yield d
self.assertEqual(pop_all(msgs), [
('Retrying request', req),
('Retry timed out', req),
('Rescheduling retry', req),
])
self.assertEqual((yield zitems(worker.redis, k)), [
(5 + 20, {
'owner_id': '1234',
'timestamp': 5,
'attempts': 1,
'intervals': [10, 20],
'request': {
'url': "%s/foo" % (srv.url,),
'method': 'POST'
}
}),
])
def test_requests(self):
msgs = self.patch_log()
k = pending_key('test')
self.assertEqual(
(yield get_req_count(self.app.redis, 'test', '1234')), 0)
pop_all(msgs)
resp = yield self.post('/requests/', {
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
}
}, headers={'X-Owner-ID': '1234'})
self.assertEqual(resp.code, http.OK)
self.assertEqual((yield resp.content()), json.dumps({}))
self.assertEqual(pop_all(msgs), [
("Adding request to pending set", {
'owner_id': '1234',
'timestamp': 10,
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
},
})
])
self.assertEqual(
(yield get_req_count(self.app.redis, 'test', '1234')), 1)
self.assertEqual((yield zitems(self.app.redis, k)), [
(self.time + 30, {
'attempts': 0,
'timestamp': 10,
'owner_id': '1234',
'intervals': [30, 90],
'request': {
'url': 'http://www.example.org',
'method': 'GET',
}
}),
])
def test_loop(self):
k = ready_key('test')
msgs = self.patch_log()
retries = self.patch_retry()
worker = yield self.mk_worker({'frequency': 5})
reqs = [{
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
} for t in range(5, 30, 5)]
yield add_ready(worker.redis, 'test', reqs)
self.assertEqual(pop_all(msgs), [
('Polling for requests to retry',),
('Retrieving next request from ready set',),
('Ready set is empty, rechecking on next poll',),
])
worker.clock.advance(5)
req = yield retries.get()
self.assertEqual(req, reqs[0])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[1:])
self.assertEqual(pop_all(msgs), [
('Polling for requests to retry',),
('Retrieving next request from ready set',),
('Scheduling request for retrying', reqs[0]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[1])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[2:])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[1]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[2])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[3:])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[2]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[3])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[4:])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[3]),
('Retrieving next request from ready set',),
])
req = yield retries.get()
self.assertEqual(req, reqs[4])
self.assertEqual((yield lvalues(worker.redis, k)), [])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[4]),
('Retrieving next request from ready set',),
])
worker.clock.advance(10)
reqs = [{
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
} for t in range(5, 15, 5)]
yield add_ready(worker.redis, 'test', reqs)
self.assertEqual(pop_all(msgs), [
('Ready set is empty, rechecking on next poll',),
])
worker.clock.advance(5)
req = yield retries.get()
self.assertEqual(req, reqs[0])
self.assertEqual((yield lvalues(worker.redis, k)), reqs[1:])
self.assertEqual(pop_all(msgs), [
('Polling for requests to retry',),
('Retrieving next request from ready set',),
('Scheduling request for retrying', reqs[0]),
('Retrieving next request from ready set',),
])
worker.clock.advance(5)
req = yield retries.get()
self.assertEqual(req, reqs[1])
self.assertEqual((yield lvalues(worker.redis, k)), [])
self.assertEqual(pop_all(msgs), [
('Scheduling request for retrying', reqs[1]),
('Retrieving next request from ready set',),
])
def test_pop_all(self):
values = [1, 2, 3]
self.assertEqual(pop_all(values), [1, 2, 3])
self.assertEqual(values, [])
def test_loop(self):
k_p = pending_key('test')
k_r = ready_key('test')
msgs = self.patch_log()
worker = yield self.mk_worker({
'redis_prefix': 'test',
'frequency': 5,
})
for t in range(5, 25, 5):
yield add_pending(worker.redis, 'test', {
'owner_id': '1234',
'timestamp': t,
'attempts': 0,
'intervals': [10],
'request': {'foo': t}
})
pending = yield zitems(worker.redis, pending_key('test'))
pending_reqs = [v for t, v in pending]
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.maintains.pop()
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), [])
self.assertEqual((yield zitems(worker.redis, k_p)), pending)
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:1])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[1:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:2])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[2:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs[:3])
self.assertEqual((yield zitems(worker.redis, k_p)), pending[3:])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
worker.clock.advance(5)
yield worker.maintains.pop()
self.assertEqual((yield lvalues(worker.redis, k_r)), pending_reqs)
self.assertEqual((yield zitems(worker.redis, k_p)), [])
self.assertEqual(pop_all(msgs), [
'Checking for requests to move from pending to ready',
'1 request(s) moved from pending to ready',
])
self.assertEqual(worker.maintains, [])
worker.stop()
worker.clock.advance(5)
self.assertEqual(worker.maintains, [])
self.assertEqual(pop_all(msgs), [])