def setUp(self):
self.clock = Clock()
self.request_queue = MemoryQueue()
self.response_queue = ResponseQueue()
self.dwn = Downloader(Settings(self.default_settings),
self.request_queue,
self.response_queue,
download_handler=MockDownloaderHandler(
Settings()),
clock=self.clock)
self.handler = self.dwn.download_handler
def __init__(self, download_handler, concurrency, delay, randomize_delay,
clock=None):
self.download_handler = download_handler
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
self.in_progress = set() # request waiting to be downloaded
self.transferring = set() # requests being downloaded (subset of `in_progress`)
self.last_download_time = 0
self.queue = MemoryQueue() # queue of (request, deferred)
# clock is used in unittests
self.clock = clock or reactor
self.delayed_processing = ScheduledCall(self._process, clock=self.clock)
def setUp(self):
self.clock = Clock()
self.request_queue = MemoryQueue()
self.response_queue = ResponseQueue()
self.dwn = Downloader(Settings(self.default_settings), self.request_queue,
self.response_queue,
download_handler=MockDownloaderHandler(Settings()),
clock=self.clock)
self.handler = self.dwn.download_handler
def __init__(self, download_handler, concurrency, delay, randomize_delay,
clock=None):
self.download_handler = download_handler
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
self.in_progress = set() # request waiting to be downloaded
self.transferring = set() # requests being downloaded (subset of `in_progress`)
self.last_download_time = 0
self.queue = MemoryQueue() # queue of (request, deferred)
# clock is used in unittests
self.clock = clock or reactor
self.delayed_processing = ScheduledCall(self._process, clock=self.clock)
def setup(self):
assert self.spider is not None, 'Spider is not set in Engine.'
# IMPORTANT: order of the following initializations is very important
# so please, think twice about any changes to it
# initialize logging
if self.settings.get_bool('LOG_ENABLED'):
log.start(self.settings['LOG_FILE'], self.settings['LOG_LEVEL'],
self.settings['LOG_STDOUT'],
self.settings['LOG_ENCODING'])
# initialize signals
self.signals = SignalManager(self)
#initialize stats
stats_cls = load_object(self.settings.get('STATS_CLASS'))
self.stats = stats_cls(self)
# initialize downloader
self.request_queue = PriorityQueue(lambda _: MemoryQueue())
self.response_queue = ResponseQueue(
self.settings.get_int('RESPONSE_ACTIVE_SIZE_LIMIT'))
self.downloader = Downloader(self.settings,
self.request_queue,
self.response_queue,
clock=self.clock)
# initialize extensions
self.extensions = ExtensionManager(self)
# initialize downloader pipeline
self.pipeline = PipelineManager(self)
self.initialized = True
# now that everything is ready, set the spider's engine
self.spider.set_engine(self)
class Slot(object):
'''Slot represents a queue of requests for one particular domain.
It respects both DOWNLOAD_DELAY and CONCURRENT_REQUESTS_PER_DOMAIN.
'''
def __init__(self, download_handler, concurrency, delay, randomize_delay,
clock=None):
self.download_handler = download_handler
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
self.in_progress = set() # request waiting to be downloaded
self.transferring = set() # requests being downloaded (subset of `in_progress`)
self.last_download_time = 0
self.queue = MemoryQueue() # queue of (request, deferred)
# clock is used in unittests
self.clock = clock or reactor
self.delayed_processing = ScheduledCall(self._process, clock=self.clock)
def enqueue(self, request, dfd):
'''Main entry point.
Put the new request to the queue and if possible, start downloading it.
'''
def remove_in_progress(response):
self.in_progress.remove(request)
return response
self.in_progress.add(request)
dfd.addBoth(remove_in_progress)
self.queue.push((request, dfd))
self._process()
@property
def free_slots(self):
return self.concurrency - len(self.transferring)
def is_idle(self):
return len(self.in_progress) == 0
def _process(self):
'''Process the requests in the queue, while respecting the delay and
concurrency.
'''
if self.delayed_processing.is_scheduled() or self._schedule_delay():
return
while self.queue and self.free_slots > 0:
self.last_download_time = self.clock.seconds()
request, downloaded_dfd = self.queue.pop()
dfd = self._download(request)
dfd.chainDeferred(downloaded_dfd)
if self._schedule_delay():
return
def _schedule_delay(self):
if self.delay:
penalty = (self.last_download_time + self.get_download_delay() -
self.clock.seconds())
if penalty > 0:
# following schedule should always be successfull, because
# `_schedule_delay()` is only called from within `_process()`
self.delayed_processing.schedule(penalty)
return True
return False
def _download(self, request):
dfd = defer.succeed(request)
# download the response
dfd.addCallback(self.download_handler.download_request)
# it is VERY important to wrap the failure into a new object!
# For errors like ConnectionLost, the same Failure object is returned
# everytime and we cannot use 'failure.request' field.
def wrap_failure(failure):
return Failure(failure.value)
dfd.addErrback(wrap_failure)
# put the request into the set of `transferring` to block other requests
# after the response is downloaded, remove it from `transferring`
def remove_transferring(response):
self.transferring.remove(request)
self._process() # process unblocked requests
return response
self.transferring.add(request)
dfd.addBoth(remove_transferring)
return dfd
def get_download_delay(self):
if self.randomize_delay:
return random.uniform(0.5 * self.delay, 1.5 * self.delay)
return self.delay
class DownloaderTest(unittest.TestCase):
default_settings = {
'CONCURRENT_REQUESTS': 2,
'CONCURRENT_REQUESTS_PER_DOMAIN': 1,
'DOWNLOAD_DELAY': 0,
'RANDOMIZE_DOWNLOAD_DELAY': False}
def setUp(self):
self.clock = Clock()
self.request_queue = MemoryQueue()
self.response_queue = ResponseQueue()
self.dwn = Downloader(Settings(self.default_settings), self.request_queue,
self.response_queue,
download_handler=MockDownloaderHandler(Settings()),
clock=self.clock)
self.handler = self.dwn.download_handler
def _update_dwn(self, **kwargs):
'''Update downloader with the new settings.
'''
new_settings = self.default_settings.copy()
new_settings.update(**kwargs)
self.dwn.processing.cancel()
self.dwn = Downloader(Settings(new_settings), self.request_queue, self.response_queue,
download_handler=MockDownloaderHandler(Settings()),
clock=self.clock)
self.handler = self.dwn.download_handler
def test_concurrency(self):
# standard situation
self._update_dwn()
self.assertEqual(self.dwn.total_concurrency, 2)
self.assertEqual(self.dwn.domain_concurrency, 1)
self.assertTrue(self.dwn.use_domain_specific)
# delay set
self._update_dwn(CONCURRENT_REQUESTS=10, CONCURRENT_REQUESTS_PER_DOMAIN=5,
DOWNLOAD_DELAY=5)
self.assertEqual(self.dwn.total_concurrency, 1)
self.assertEqual(self.dwn.domain_concurrency, 1)
self.assertFalse(self.dwn.use_domain_specific)
# domain concurrency is 0
self._update_dwn(CONCURRENT_REQUESTS=10, CONCURRENT_REQUESTS_PER_DOMAIN=0)
self.assertEqual(self.dwn.total_concurrency, 10)
self.assertEqual(self.dwn.domain_concurrency, 10)
self.assertFalse(self.dwn.use_domain_specific)
# domain concurrency is too big
self._update_dwn(CONCURRENT_REQUESTS=5, CONCURRENT_REQUESTS_PER_DOMAIN=10)
self.assertEqual(self.dwn.total_concurrency, 5)
self.assertEqual(self.dwn.domain_concurrency, 5)
self.assertFalse(self.dwn.use_domain_specific)
self._update_dwn(CONCURRENT_REQUESTS=5, CONCURRENT_REQUESTS_PER_DOMAIN=5)
self.assertFalse(self.dwn.use_domain_specific)
def test_get_slot(self):
key, slot = self.dwn._get_slot(Request('http://www.github.com/'))
self.assertEqual(key, 'www.github.com')
key2, slot2 = self.dwn._get_slot(Request('http://www.github.com/hello/world#bla'))
self.assertEqual(key2, 'www.github.com')
self.assertIs(slot2, slot)
key3, slot3 = self.dwn._get_slot(Request('http://sites.github.com/'))
self.assertEqual(key3, 'sites.github.com')
self.assertIsNot(slot3, slot)
self.assertEqual(len(self.dwn.slots), 2)
# don't use domain specific slots
self.dwn.use_domain_specific = False
key, slot = self.dwn._get_slot(Request('http://www.github.com/'))
self.assertEqual(key, '')
key2, slot2 = self.dwn._get_slot(Request('http://sites.github.com/'))
self.assertIs(slot2, slot)
def test_basic(self):
# create 5 requests with slot ids: a, b, a, a, c
requests = [get_request(id)[0] for id in 'abaac']
map(lambda r: self.request_queue.push(r), requests)
self.assertEqual(self.dwn.free_slots, 2)
self.assertTrue(self.dwn.is_idle())
# start downloading first two requests
self.clock.advance(0)
self.assertEqual(self.dwn.free_slots, 0)
self.assertFalse(self.dwn.is_idle())
# no more requests are scheduled, until download is finished
self.clock.advance(20)
self.assertEqual(len(self.request_queue), 3)
# download the first request
self.handler.call(requests[0], Response('hello'))
self.assertEqual(self.dwn.free_slots, 1) # slot is immediately available
# result is also available
result = self.response_queue.peek()
self.assertIs(result.request, requests[0])
self.assertEqual(result.url, 'hello')
# enqueue third request
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.assertEqual(self.dwn.free_slots, 0)
# download second request
self.handler.call(requests[1], Response(''))
# enqueue fourth request
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.assertEqual(self.dwn.free_slots, 0)
# fourth request should not begin download, until 3rd request is done
self.assertRaises(KeyError, self.handler.call, requests[3], Response(''))
# finish
self.handler.call(requests[2], Response(''))
self.handler.call(requests[3], Response(''))
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.handler.call(requests[4], Response(''))
# final checks
self.clock.pump([1] * 10)
self.assertEqual(len(self.response_queue), 5)
self.assertTrue(self.dwn.is_idle())
def test_close(self):
req1 = get_request('a')[0]
req2 = get_request('b')[0]
self.request_queue.push(req1)
self.clock.advance(20)
self.request_queue.push(req2)
# test basic attributes, before and after closing
self.assertTrue(self.dwn.running)
self.assertTrue(self.dwn.processing.is_scheduled())
self.dwn.close()
self.assertFalse(self.dwn.running)
self.assertFalse(self.dwn.processing.is_scheduled())
self.clock.advance(20)
self.assertEqual(len(self.request_queue), 1) # request 2 remains unqueued
# downloader behavior after closing
self.assertEqual(len(self.response_queue), 0)
self.handler.call(req1, Response(''))
self.assertEqual(len(self.response_queue), 0)
def test_fail(self):
self._update_dwn(CONCURRENT_REQUESTS=3, CONCURRENT_REQUESTS_PER_DOMAIN=2)
requests = [get_request(id)[0] for id in 'aab']
map(lambda r: self.request_queue.push(r), requests)
# enqueue requests
self.clock.advance(0)
# fail 1st request
err = ValueError('my bad')
self.handler.fail(requests[0], err)
self.assertEqual(self.dwn.free_slots, 1)
fail = self.response_queue.pop()
self.assertIs(fail.request, requests[0])
self.assertIs(fail.value, err)
# fail 3rd request
self.handler.fail(requests[2], err)
fail = self.response_queue.pop()
self.assertIs(fail.request, requests[2])
self.assertIs(fail.value, err)
# succeed 2nd request
self.handler.call(requests[1], Response('nice!', request=requests[1]))
resp = self.response_queue.pop()
self.assertIs(resp.request, requests[1])
self.assertEqual(resp.url, 'nice!')
def test_clear_slots(self):
requests = [get_request(id)[0] for id in xrange(30)]
for r in requests:
self.request_queue.push(r)
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.handler.call(r, Response(''))
self.assertLessEqual(len(self.dwn.slots), 2 * self.dwn.total_concurrency)
class DownloaderTest(unittest.TestCase):
default_settings = {
'CONCURRENT_REQUESTS': 2,
'CONCURRENT_REQUESTS_PER_DOMAIN': 1,
'DOWNLOAD_DELAY': 0,
'RANDOMIZE_DOWNLOAD_DELAY': False
}
def setUp(self):
self.clock = Clock()
self.request_queue = MemoryQueue()
self.response_queue = ResponseQueue()
self.dwn = Downloader(Settings(self.default_settings),
self.request_queue,
self.response_queue,
download_handler=MockDownloaderHandler(
Settings()),
clock=self.clock)
self.handler = self.dwn.download_handler
def _update_dwn(self, **kwargs):
'''Update downloader with the new settings.
'''
new_settings = self.default_settings.copy()
new_settings.update(**kwargs)
self.dwn.processing.cancel()
self.dwn = Downloader(Settings(new_settings),
self.request_queue,
self.response_queue,
download_handler=MockDownloaderHandler(
Settings()),
clock=self.clock)
self.handler = self.dwn.download_handler
def test_concurrency(self):
# standard situation
self._update_dwn()
self.assertEqual(self.dwn.total_concurrency, 2)
self.assertEqual(self.dwn.domain_concurrency, 1)
self.assertTrue(self.dwn.use_domain_specific)
# delay set
self._update_dwn(CONCURRENT_REQUESTS=10,
CONCURRENT_REQUESTS_PER_DOMAIN=5,
DOWNLOAD_DELAY=3.14)
self.assertEqual(self.dwn.download_delay, 3.14)
self.assertEqual(self.dwn.total_concurrency, 1)
self.assertEqual(self.dwn.domain_concurrency, 1)
self.assertFalse(self.dwn.use_domain_specific)
# domain concurrency is 0
self._update_dwn(CONCURRENT_REQUESTS=10,
CONCURRENT_REQUESTS_PER_DOMAIN=0)
self.assertEqual(self.dwn.total_concurrency, 10)
self.assertEqual(self.dwn.domain_concurrency, 10)
self.assertFalse(self.dwn.use_domain_specific)
# domain concurrency is too big
self._update_dwn(CONCURRENT_REQUESTS=5,
CONCURRENT_REQUESTS_PER_DOMAIN=10)
self.assertEqual(self.dwn.total_concurrency, 5)
self.assertEqual(self.dwn.domain_concurrency, 5)
self.assertFalse(self.dwn.use_domain_specific)
self._update_dwn(CONCURRENT_REQUESTS=5,
CONCURRENT_REQUESTS_PER_DOMAIN=5)
self.assertFalse(self.dwn.use_domain_specific)
def test_get_slot(self):
key, slot = self.dwn._get_slot(Request('http://www.github.com/'))
self.assertEqual(key, 'www.github.com')
key2, slot2 = self.dwn._get_slot(
Request('http://www.github.com/hello/world#bla'))
self.assertEqual(key2, 'www.github.com')
self.assertIs(slot2, slot)
key3, slot3 = self.dwn._get_slot(Request('http://sites.github.com/'))
self.assertEqual(key3, 'sites.github.com')
self.assertIsNot(slot3, slot)
self.assertEqual(len(self.dwn.slots), 2)
# don't use domain specific slots
self.dwn.use_domain_specific = False
key, slot = self.dwn._get_slot(Request('http://www.github.com/'))
self.assertEqual(key, '')
key2, slot2 = self.dwn._get_slot(Request('http://sites.github.com/'))
self.assertIs(slot2, slot)
def test_basic(self):
# create 5 requests with slot ids: a, b, a, a, c
requests = [get_request(id)[0] for id in 'abaac']
map(lambda r: self.request_queue.push(r), requests)
self.assertEqual(self.dwn.free_slots, 2)
self.assertTrue(self.dwn.is_idle())
# start downloading first two requests
self.clock.advance(0)
self.assertEqual(self.dwn.free_slots, 0)
self.assertFalse(self.dwn.is_idle())
# no more requests are scheduled, until download is finished
self.clock.advance(20)
self.assertEqual(len(self.request_queue), 3)
# download the first request
self.handler.call(requests[0], Response('hello'))
self.assertEqual(self.dwn.free_slots,
1) # slot is immediately available
# result is also available
result = self.response_queue.peek()
self.assertIs(result.request, requests[0])
self.assertEqual(result.url, 'hello')
# enqueue third request
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.assertEqual(self.dwn.free_slots, 0)
# download second request
self.handler.call(requests[1], Response(''))
# enqueue fourth request
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.assertEqual(self.dwn.free_slots, 0)
# fourth request should not begin download, until 3rd request is done
self.assertRaises(KeyError, self.handler.call, requests[3],
Response(''))
# finish
self.handler.call(requests[2], Response(''))
self.handler.call(requests[3], Response(''))
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.handler.call(requests[4], Response(''))
# final checks
self.clock.pump([1] * 10)
self.assertEqual(len(self.response_queue), 5)
self.assertTrue(self.dwn.is_idle())
def test_close(self):
req1 = get_request('a')[0]
req2 = get_request('b')[0]
self.request_queue.push(req1)
self.clock.advance(20)
self.request_queue.push(req2)
# test basic attributes, before and after closing
self.assertTrue(self.dwn.running)
self.assertTrue(self.dwn.processing.is_scheduled())
self.dwn.close()
self.assertFalse(self.dwn.running)
self.assertFalse(self.dwn.processing.is_scheduled())
self.clock.advance(20)
self.assertEqual(len(self.request_queue),
1) # request 2 remains unqueued
# downloader behavior after closing
self.assertEqual(len(self.response_queue), 0)
self.handler.call(req1, Response(''))
self.assertEqual(len(self.response_queue), 0)
def test_fail(self):
self._update_dwn(CONCURRENT_REQUESTS=3,
CONCURRENT_REQUESTS_PER_DOMAIN=2)
requests = [get_request(id)[0] for id in 'aab']
map(lambda r: self.request_queue.push(r), requests)
# enqueue requests
self.clock.advance(0)
# fail 1st request
err = ValueError('my bad')
self.handler.fail(requests[0], err)
self.assertEqual(self.dwn.free_slots, 1)
fail = self.response_queue.pop()
self.assertIs(fail.request, requests[0])
self.assertIs(fail.value, err)
# fail 3rd request
self.handler.fail(requests[2], err)
fail = self.response_queue.pop()
self.assertIs(fail.request, requests[2])
self.assertIs(fail.value, err)
# succeed 2nd request
self.handler.call(requests[1], Response('nice!', request=requests[1]))
resp = self.response_queue.pop()
self.assertIs(resp.request, requests[1])
self.assertEqual(resp.url, 'nice!')
def test_clear_slots(self):
requests = [get_request(id)[0] for id in xrange(30)]
for r in requests:
self.request_queue.push(r)
self.clock.advance(Downloader.QUEUE_CHECK_FREQUENCY)
self.handler.call(r, Response(''))
self.assertLessEqual(len(self.dwn.slots),
2 * self.dwn.total_concurrency)
def qfactory(priority):
return MemoryQueue()
class Slot(object):
'''Slot represents a queue of requests for one particular domain.
It respects both DOWNLOAD_DELAY and CONCURRENT_REQUESTS_PER_DOMAIN.
'''
def __init__(self, download_handler, concurrency, delay, randomize_delay,
clock=None):
self.download_handler = download_handler
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
self.in_progress = set() # request waiting to be downloaded
self.transferring = set() # requests being downloaded (subset of `in_progress`)
self.last_download_time = 0
self.queue = MemoryQueue() # queue of (request, deferred)
# clock is used in unittests
self.clock = clock or reactor
self.delayed_processing = ScheduledCall(self._process, clock=self.clock)
def enqueue(self, request, dfd):
'''Main entry point.
Put the new request to the queue and if possible, start downloading it.
'''
def remove_in_progress(response):
self.in_progress.remove(request)
return response
self.in_progress.add(request)
dfd.addBoth(remove_in_progress)
self.queue.push((request, dfd))
self._process()
@property
def free_slots(self):
return self.concurrency - len(self.transferring)
def is_idle(self):
return len(self.in_progress) == 0
def _process(self):
'''Process the requests in the queue, while respecting the delay and
concurrency.
'''
if self.delayed_processing.is_scheduled() or self._schedule_delay():
return
while self.queue and self.free_slots > 0:
self.last_download_time = self.clock.seconds()
request, downloaded_dfd = self.queue.pop()
dfd = self._download(request)
dfd.chainDeferred(downloaded_dfd)
if self._schedule_delay():
return
def _schedule_delay(self):
if self.delay:
penalty = (self.last_download_time + self.get_download_delay() -
self.clock.seconds())
if penalty > 0:
# following schedule should always be successfull, because
# `_schedule_delay()` is only called from within `_process()`
self.delayed_processing.schedule(penalty)
return True
return False
def _download(self, request):
dfd = defer.succeed(request)
# download the response
dfd.addCallback(self.download_handler.download_request)
# it is VERY important to wrap the failure into a new object!
# For errors like ConnectionLost, the same Failure object is returned
# everytime and we cannot use 'failure.request' field.
def wrap_failure(failure):
return Failure(failure.value)
dfd.addErrback(wrap_failure)
# put the request into the set of `transferring` to block other requests
# after the response is downloaded, remove it from `transferring`
def remove_transferring(response):
self.transferring.remove(request)
self._process() # process unblocked requests
return response
self.transferring.add(request)
dfd.addBoth(remove_transferring)
return dfd
def get_download_delay(self):
if self.randomize_delay:
return random.uniform(0.5 * self.delay, 1.5 * self.delay)
return self.delay