def test_err(self):
r = ManualReadable([23])
e = Exception()
errs = []
d = r.read()
d.addErrback(lambda f: errs.append(f.value))
yield r.err(e)
self.assertEqual(errs, [e])
def test_loop_retry_err(self):
e1 = Exception()
e3 = Exception()
errors = self.patch_on_error()
r = ManualReadable([1, 2, 3])
w = ManualWritable()
self.patch(RetrySenderWorker, 'next_req', staticmethod(r.read))
self.patch(RetrySenderWorker, 'retry', staticmethod(w.write))
yield self.mk_worker({
'frequency': 5,
'concurrency_limit': 2,
})
# We've read all three requests from redis and are busy retrying the
# first two
yield r.next()
yield r.next()
yield r.next()
self.assertEqual(errors, [])
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [1, 2])
self.assertEqual(w.written, [])
# Retry 1 throws an error, we catch it. We now have space for
# request 3.
yield w.err(e1)
self.assertEqual(errors, [e1])
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [2, 3])
self.assertEqual(w.written, [])
# Retry 2 succeeds.
yield w.next()
self.assertEqual(errors, [e1])
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [3])
self.assertEqual(w.written, [2])
# Retry 3 throws an error, we catch it.
yield w.err(e3)
self.assertEqual(errors, [e1, e3])
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [])
self.assertEqual(w.written, [2])
def test_reading(self):
r = ManualReadable([1, 2, 3, 4])
self.assertEqual(r.unread, [1, 2, 3, 4])
self.assertEqual(r.reading, [])
d1 = r.read()
self.assertEqual(r.unread, [2, 3, 4])
self.assertEqual(r.reading, [1])
d2 = r.read()
self.assertEqual(r.unread, [3, 4])
self.assertEqual(r.reading, [1, 2])
self.assertEqual(d1, r.next())
self.assertEqual((yield d1), 1)
self.assertEqual(r.unread, [3, 4])
self.assertEqual(r.reading, [2])
self.assertEqual(d2, r.next())
self.assertEqual((yield d2), 2)
self.assertEqual(r.unread, [3, 4])
self.assertEqual(r.reading, [])
d = r.read()
self.assertEqual(r.unread, [4])
self.assertEqual(r.reading, [3])
self.assertEqual(d, r.next())
self.assertEqual((yield d), 3)
d = r.read()
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [4])
self.assertEqual(d, r.next())
self.assertEqual((yield d), 4)
self.assertEqual((yield r.read()), None)
self.assertEqual((yield r.read()), None)
def test_loop_concurrency_limit(self):
r = ManualReadable([1, 2, 3, 4, 5])
w = ManualWritable()
self.patch(RetrySenderWorker, 'next_req', staticmethod(r.read))
self.patch(RetrySenderWorker, 'retry', staticmethod(w.write))
yield self.mk_worker({
'frequency': 5,
'concurrency_limit': 2,
})
# We haven't yet started any retries
self.assertEqual(r.unread, [2, 3, 4, 5])
self.assertEqual(r.reading, [1])
self.assertEqual(w.writing, [])
self.assertEqual(w.written, [])
# We've started retrying request 1 and still have space
yield r.next()
self.assertEqual(r.unread, [3, 4, 5])
self.assertEqual(r.reading, [2])
self.assertEqual(w.writing, [1])
self.assertEqual(w.written, [])
# We've started retrying request 2 and are at capacity
yield r.next()
self.assertEqual(r.unread, [4, 5])
self.assertEqual(r.reading, [3])
self.assertEqual(w.writing, [1, 2])
self.assertEqual(w.written, [])
# We've read request 3 from redis but haven't retried it yet, since we
# are waiting for request 1 and 2 to complete
yield r.next()
self.assertEqual(r.unread, [4, 5])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [1, 2])
self.assertEqual(w.written, [])
# Request 1 has completed, so we have space to start retrying
# request 3 and ask redis for request 4.
yield w.next()
self.assertEqual(r.unread, [5])
self.assertEqual(r.reading, [4])
self.assertEqual(w.writing, [2, 3])
self.assertEqual(w.written, [1])
# We've read request 4 from redis but haven't retried it yet, since we
# are waiting for request 2 and 3 to complete
yield r.next()
self.assertEqual(r.unread, [5])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [2, 3])
self.assertEqual(w.written, [1])
# Request 2 has completed, so we have space to start retrying
# request 3 and ask redis for request 5.
yield w.next()
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [5])
self.assertEqual(w.writing, [3, 4])
self.assertEqual(w.written, [1, 2])
# Request 3 and 4 complete while we are waiting for request 5
# from redis
yield w.next()
yield w.next()
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [5])
self.assertEqual(w.writing, [])
self.assertEqual(w.written, [1, 2, 3, 4])
# We've read request 5 from redis and started retrying it
yield r.next()
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [5])
self.assertEqual(w.written, [1, 2, 3, 4])
# We've retried request 5. Redis says we have nothing more to read, so
# we are done.
yield w.next()
self.assertEqual(r.unread, [])
self.assertEqual(r.reading, [])
self.assertEqual(w.writing, [])
self.assertEqual(w.written, [1, 2, 3, 4, 5])
