def setUp(self):
self.handler = MockDownloaderHandler(Settings())
self.clock = Clock()
self.slot = Slot(self.handler,
self.default_concurrency,
self.default_delay,
self.default_randomize_delay,
clock=self.clock)
def setUp(self):
self.handler = MockDownloaderHandler(Settings())
self.clock = Clock()
self.slot = Slot(self.handler, self.default_concurrency,
self.default_delay, self.default_randomize_delay,
clock=self.clock)
class DownloaderSlotTest(unittest.TestCase):
default_concurrency = 2
default_delay = 0
default_randomize_delay = False
def setUp(self):
self.handler = MockDownloaderHandler(Settings())
self.clock = Clock()
self.slot = Slot(self.handler, self.default_concurrency,
self.default_delay, self.default_randomize_delay,
clock=self.clock)
def test_basic(self):
received = []
def downloaded(result):
received.append(result)
# enqueue 3 requests
r1, dfd1 = get_request('1', func=downloaded)
self.slot.enqueue(r1, dfd1)
self.assertEqual(len(self.slot.in_progress), 1)
self.assertEqual(len(self.slot.transferring), 1)
r2, dfd2 = get_request('2', func=downloaded)
r3, dfd3 = get_request('3', func=downloaded)
self.slot.enqueue(r2, dfd2)
self.slot.enqueue(r3, dfd3)
self.assertEqual(len(self.slot.in_progress), 3)
self.assertEqual(len(self.slot.transferring), 2)
self.assertEqual(self.slot.free_slots, 0)
# download r2
self.handler.call(r2, Response(''))
self.assertIs(received[-1].request, r2)
self.assertEqual(len(self.slot.transferring), 2)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(self.slot.free_slots, 0)
# download r1 and r3
self.handler.call(r3, Response(''))
self.handler.call(r1, Response(''))
self.assertIs(received[-2].request, r3)
self.assertIs(received[-1].request, r1)
self.assertEqual(self.slot.free_slots, 2)
# nothing happens now
self.clock.advance(5)
self.assertEqual(len(self.slot.in_progress), 0)
self.assertEqual(self.slot.free_slots, 2)
def test_delay(self):
self.slot.concurrency = 1
self.slot.delay = 5
self.clock.advance(10) # so we don't start on time 0
# enqueue 3 requests
r1, dfd1 = get_request('1')
self.slot.enqueue(r1, dfd1)
r2, dfd2 = get_request('2')
self.slot.enqueue(r2, dfd2)
r3, dfd3 = get_request('3')
self.slot.enqueue(r3, dfd3)
self.assertEqual(len(self.slot.in_progress), 3)
self.assertEqual(len(self.slot.transferring), 1)
self.assertEqual(self.slot.last_download_time, 10)
self.assertEqual(self.slot.delayed_processing.get_time(), 15)
# download the 1st request
self.handler.call(r1, Response(''))
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 0)
self.assertEqual(self.slot.free_slots, 1)
# we should still wait
self.clock.advance(3)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 0)
self.assertEqual(self.slot.free_slots, 1)
# make the 2nd request downloading
self.clock.advance(3)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 1)
self.assertEqual(self.slot.free_slots, 0)
self.assertEqual(self.slot.last_download_time, 16)
self.assertEqual(self.slot.delayed_processing.get_time(), 21)
# wait and nothing happens
self.clock.advance(10)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 1)
self.assertFalse(self.slot.delayed_processing.is_scheduled())
def test_random_delay(self):
self.slot.delay = 5
self.slot.randomize_delay = False
delays = [self.slot.get_download_delay() for x in xrange(10)]
self.assertTrue(all(x == 5 for x in delays))
self.slot.randomize_delay = True
lower = 0.5 * 5
upper = 1.5 * 5
delays = [self.slot.get_download_delay() for x in xrange(100)]
self.assertTrue(all(lower <= x <= upper for x in delays))
third1 = (2 * lower + upper) / 3
third2 = (lower + 2 * upper) / 3
self.assertTrue(any(x <= third1 for x in delays))
self.assertTrue(any(third1 <= x <= third2 for x in delays))
self.assertTrue(any(third2 <= x for x in delays))
def test_fail(self):
received = []
def downloaded(result):
received.append(result)
# enqueue 3 requests
r1, dfd1 = get_request('1', func=downloaded)
self.slot.enqueue(r1, dfd1)
r2, dfd2 = get_request('2', func=downloaded)
self.slot.enqueue(r2, dfd2)
r3, dfd3 = get_request('3', func=downloaded)
self.slot.enqueue(r3, dfd3)
# fail the first request
err = ValueError('my bad')
self.handler.fail(r1, err)
self.assertEqual(received[-1].value, err)
# other requests should be ok
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 2)
self.handler.call(r2, Response(''))
self.assertEqual(received[-1].request, r2)
self.handler.call(r3, Response(''))
self.assertEqual(received[-1].request, r3)
self.assertEqual(len(self.slot.in_progress), 0)
self.assertEqual(len(self.slot.transferring), 0)
def test_exception(self):
self.slot.download_handler = ExceptionDownloaderHandler(Settings())
r1, dfd1 = get_request('1')
self.slot.enqueue(r1, dfd1)
return self.assertFailure(dfd1, Exception)
def test_failure(self):
self.slot.download_handler = FailureDownloaderHandler(Settings())
download_values = []
def downloaded(value):
download_values.append(value)
for i in xrange(2):
r, dfd = get_request(str(i))
dfd.addBoth(downloaded)
self.slot.enqueue(r, dfd)
self.assertEqual(len(download_values), 2)
self.assertIsInstance(download_values[0], Failure)
self.assertIsInstance(download_values[1], Failure)
self.assertIsNot(download_values[0], download_values[1])
self.assertIsInstance(download_values[0].value, ValueError)
class DownloaderSlotTest(unittest.TestCase):
default_concurrency = 2
default_delay = 0
default_randomize_delay = False
def setUp(self):
self.handler = MockDownloaderHandler(Settings())
self.clock = Clock()
self.slot = Slot(self.handler,
self.default_concurrency,
self.default_delay,
self.default_randomize_delay,
clock=self.clock)
def test_basic(self):
received = []
def downloaded(result):
received.append(result)
# enqueue 3 requests
r1, dfd1 = get_request('1', func=downloaded)
self.slot.enqueue(r1, dfd1)
self.assertEqual(len(self.slot.in_progress), 1)
self.assertEqual(len(self.slot.transferring), 1)
r2, dfd2 = get_request('2', func=downloaded)
r3, dfd3 = get_request('3', func=downloaded)
self.slot.enqueue(r2, dfd2)
self.slot.enqueue(r3, dfd3)
self.assertEqual(len(self.slot.in_progress), 3)
self.assertEqual(len(self.slot.transferring), 2)
self.assertEqual(self.slot.free_slots, 0)
# download r2
self.handler.call(r2, Response(''))
self.assertIs(received[-1].request, r2)
self.assertEqual(len(self.slot.transferring), 2)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(self.slot.free_slots, 0)
# download r1 and r3
self.handler.call(r3, Response(''))
self.handler.call(r1, Response(''))
self.assertIs(received[-2].request, r3)
self.assertIs(received[-1].request, r1)
self.assertEqual(self.slot.free_slots, 2)
# nothing happens now
self.clock.advance(5)
self.assertEqual(len(self.slot.in_progress), 0)
self.assertEqual(self.slot.free_slots, 2)
def test_delay(self):
self.slot.concurrency = 1
self.slot.delay = 5
self.clock.advance(10) # so we don't start on time 0
# enqueue 3 requests
r1, dfd1 = get_request('1')
self.slot.enqueue(r1, dfd1)
r2, dfd2 = get_request('2')
self.slot.enqueue(r2, dfd2)
r3, dfd3 = get_request('3')
self.slot.enqueue(r3, dfd3)
self.assertEqual(len(self.slot.in_progress), 3)
self.assertEqual(len(self.slot.transferring), 1)
self.assertEqual(self.slot.last_download_time, 10)
self.assertEqual(self.slot.delayed_processing.get_time(), 15)
# download the 1st request
self.handler.call(r1, Response(''))
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 0)
self.assertEqual(self.slot.free_slots, 1)
# we should still wait
self.clock.advance(3)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 0)
self.assertEqual(self.slot.free_slots, 1)
# make the 2nd request downloading
self.clock.advance(3)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 1)
self.assertEqual(self.slot.free_slots, 0)
self.assertEqual(self.slot.last_download_time, 16)
self.assertEqual(self.slot.delayed_processing.get_time(), 21)
# wait and nothing happens
self.clock.advance(10)
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 1)
self.assertFalse(self.slot.delayed_processing.is_scheduled())
def test_random_delay(self):
self.slot.delay = 5
self.slot.randomize_delay = False
delays = [self.slot.get_download_delay() for x in xrange(10)]
self.assertTrue(all(x == 5 for x in delays))
self.slot.randomize_delay = True
lower = 0.5 * 5
upper = 1.5 * 5
delays = [self.slot.get_download_delay() for x in xrange(100)]
self.assertTrue(all(lower <= x <= upper for x in delays))
third1 = (2 * lower + upper) / 3
third2 = (lower + 2 * upper) / 3
self.assertTrue(any(x <= third1 for x in delays))
self.assertTrue(any(third1 <= x <= third2 for x in delays))
self.assertTrue(any(third2 <= x for x in delays))
def test_fail(self):
received = []
def downloaded(result):
received.append(result)
# enqueue 3 requests
r1, dfd1 = get_request('1', func=downloaded)
self.slot.enqueue(r1, dfd1)
r2, dfd2 = get_request('2', func=downloaded)
self.slot.enqueue(r2, dfd2)
r3, dfd3 = get_request('3', func=downloaded)
self.slot.enqueue(r3, dfd3)
# fail the first request
err = ValueError('my bad')
self.handler.fail(r1, err)
self.assertEqual(received[-1].value, err)
# other requests should be ok
self.assertEqual(len(self.slot.in_progress), 2)
self.assertEqual(len(self.slot.transferring), 2)
self.handler.call(r2, Response(''))
self.assertEqual(received[-1].request, r2)
self.handler.call(r3, Response(''))
self.assertEqual(received[-1].request, r3)
self.assertEqual(len(self.slot.in_progress), 0)
self.assertEqual(len(self.slot.transferring), 0)
def test_exception(self):
self.slot.download_handler = ExceptionDownloaderHandler(Settings())
r1, dfd1 = get_request('1')
self.slot.enqueue(r1, dfd1)
return self.assertFailure(dfd1, Exception)
def test_failure(self):
self.slot.download_handler = FailureDownloaderHandler(Settings())
download_values = []
def downloaded(value):
download_values.append(value)
for i in xrange(2):
r, dfd = get_request(str(i))
dfd.addBoth(downloaded)
self.slot.enqueue(r, dfd)
self.assertEqual(len(download_values), 2)
self.assertIsInstance(download_values[0], Failure)
self.assertIsInstance(download_values[1], Failure)
self.assertIsNot(download_values[0], download_values[1])
self.assertIsInstance(download_values[0].value, ValueError)