def setUp(self): self.handler = MockDownloaderHandler(Settings()) self.clock = Clock() self.slot = Slot(self.handler, self.default_concurrency, self.default_delay, self.default_randomize_delay, clock=self.clock)
class DownloaderSlotTest(unittest.TestCase): default_concurrency = 2 default_delay = 0 default_randomize_delay = False def setUp(self): self.handler = MockDownloaderHandler(Settings()) self.clock = Clock() self.slot = Slot(self.handler, self.default_concurrency, self.default_delay, self.default_randomize_delay, clock=self.clock) def test_basic(self): received = [] def downloaded(result): received.append(result) # enqueue 3 requests r1, dfd1 = get_request('1', func=downloaded) self.slot.enqueue(r1, dfd1) self.assertEqual(len(self.slot.in_progress), 1) self.assertEqual(len(self.slot.transferring), 1) r2, dfd2 = get_request('2', func=downloaded) r3, dfd3 = get_request('3', func=downloaded) self.slot.enqueue(r2, dfd2) self.slot.enqueue(r3, dfd3) self.assertEqual(len(self.slot.in_progress), 3) self.assertEqual(len(self.slot.transferring), 2) self.assertEqual(self.slot.free_slots, 0) # download r2 self.handler.call(r2, Response('')) self.assertIs(received[-1].request, r2) self.assertEqual(len(self.slot.transferring), 2) self.assertEqual(len(self.slot.in_progress), 2) self.assertEqual(self.slot.free_slots, 0) # download r1 and r3 self.handler.call(r3, Response('')) self.handler.call(r1, Response('')) self.assertIs(received[-2].request, r3) self.assertIs(received[-1].request, r1) self.assertEqual(self.slot.free_slots, 2) # nothing happens now self.clock.advance(5) self.assertEqual(len(self.slot.in_progress), 0) self.assertEqual(self.slot.free_slots, 2) def test_delay(self): self.slot.concurrency = 1 self.slot.delay = 5 self.clock.advance(10) # so we don't start on time 0 # enqueue 3 requests r1, dfd1 = get_request('1') self.slot.enqueue(r1, dfd1) r2, dfd2 = get_request('2') self.slot.enqueue(r2, dfd2) r3, dfd3 = get_request('3') self.slot.enqueue(r3, dfd3) self.assertEqual(len(self.slot.in_progress), 3) self.assertEqual(len(self.slot.transferring), 1) self.assertEqual(self.slot.last_download_time, 10) self.assertEqual(self.slot.delayed_processing.get_time(), 15) # download the 1st request self.handler.call(r1, Response('')) self.assertEqual(len(self.slot.in_progress), 2) self.assertEqual(len(self.slot.transferring), 0) self.assertEqual(self.slot.free_slots, 1) # we should still wait self.clock.advance(3) self.assertEqual(len(self.slot.in_progress), 2) self.assertEqual(len(self.slot.transferring), 0) self.assertEqual(self.slot.free_slots, 1) # make the 2nd request downloading self.clock.advance(3) self.assertEqual(len(self.slot.in_progress), 2) self.assertEqual(len(self.slot.transferring), 1) self.assertEqual(self.slot.free_slots, 0) self.assertEqual(self.slot.last_download_time, 16) self.assertEqual(self.slot.delayed_processing.get_time(), 21) # wait and nothing happens self.clock.advance(10) self.assertEqual(len(self.slot.in_progress), 2) self.assertEqual(len(self.slot.transferring), 1) self.assertFalse(self.slot.delayed_processing.is_scheduled()) def test_random_delay(self): self.slot.delay = 5 self.slot.randomize_delay = False delays = [self.slot.get_download_delay() for x in xrange(10)] self.assertTrue(all(x == 5 for x in delays)) self.slot.randomize_delay = True lower = 0.5 * 5 upper = 1.5 * 5 delays = [self.slot.get_download_delay() for x in xrange(100)] self.assertTrue(all(lower <= x <= upper for x in delays)) third1 = (2 * lower + upper) / 3 third2 = (lower + 2 * upper) / 3 self.assertTrue(any(x <= third1 for x in delays)) self.assertTrue(any(third1 <= x <= third2 for x in delays)) self.assertTrue(any(third2 <= x for x in delays)) def test_fail(self): received = [] def downloaded(result): received.append(result) # enqueue 3 requests r1, dfd1 = get_request('1', func=downloaded) self.slot.enqueue(r1, dfd1) r2, dfd2 = get_request('2', func=downloaded) self.slot.enqueue(r2, dfd2) r3, dfd3 = get_request('3', func=downloaded) self.slot.enqueue(r3, dfd3) # fail the first request err = ValueError('my bad') self.handler.fail(r1, err) self.assertEqual(received[-1].value, err) # other requests should be ok self.assertEqual(len(self.slot.in_progress), 2) self.assertEqual(len(self.slot.transferring), 2) self.handler.call(r2, Response('')) self.assertEqual(received[-1].request, r2) self.handler.call(r3, Response('')) self.assertEqual(received[-1].request, r3) self.assertEqual(len(self.slot.in_progress), 0) self.assertEqual(len(self.slot.transferring), 0) def test_exception(self): self.slot.download_handler = ExceptionDownloaderHandler(Settings()) r1, dfd1 = get_request('1') self.slot.enqueue(r1, dfd1) return self.assertFailure(dfd1, Exception) def test_failure(self): self.slot.download_handler = FailureDownloaderHandler(Settings()) download_values = [] def downloaded(value): download_values.append(value) for i in xrange(2): r, dfd = get_request(str(i)) dfd.addBoth(downloaded) self.slot.enqueue(r, dfd) self.assertEqual(len(download_values), 2) self.assertIsInstance(download_values[0], Failure) self.assertIsInstance(download_values[1], Failure) self.assertIsNot(download_values[0], download_values[1]) self.assertIsInstance(download_values[0].value, ValueError)