Beispiel #1
0
 def __init__(self, download_handler, concurrency, delay, randomize_delay,
              clock=None):
     self.download_handler = download_handler
     self.concurrency = concurrency
     self.delay = delay
     self.randomize_delay = randomize_delay
     self.in_progress = set()  # request waiting to be downloaded
     self.transferring = set()  # requests being downloaded (subset of `in_progress`)
     self.last_download_time = 0
     self.queue = MemoryQueue()  # queue of (request, deferred)
     # clock is used in unittests
     self.clock = clock or reactor
     self.delayed_processing = ScheduledCall(self._process, clock=self.clock)
Beispiel #2
0
    def __init__(self, settings, project, command_invoked='', clock=None):
        '''Constructor of Engine should be very lightweight, so that things
        can be easily unittested. For any more complicated initialization
        use `setup()`.
        '''
        self.settings = settings
        self.project = project
        self.spiders = SpiderManager(settings)

        self.stop_if_idle = True
        self.initialized = False  # True, when `setup()` has been called
        # name of the command invoking the engine. E.g. `crawl`, `shell`, etc.
        self.command_invoked = command_invoked

        self.spider = None
        self.pending_requests = 0
        self.running = False
        self.paused = False
        # clock is used in unittests
        self.clock = clock or reactor
        self.processing = ScheduledCall(self._process_queue, clock=self.clock)
Beispiel #3
0
 def test_init(self):
     # test initializing ScheduledCall without overriding its clock
     sc = ScheduledCall(self.obj.func, *self.default_args,
                        **self.default_kwargs)
     sc.schedule()
     sc.cancel()
Beispiel #4
0
 def setUp(self):
     self.clock = Clock()
     self.obj = ModifiedObject()
     self.sc = ScheduledCall(self.obj.func, clock=self.clock,
                             *self.default_args,
                             **self.default_kwargs)
Beispiel #5
0
class ScheduledCallTest(unittest.TestCase):
    default_args = (10, 'hello')
    default_kwargs = {'a': 47, 'b': 'c'}

    def setUp(self):
        self.clock = Clock()
        self.obj = ModifiedObject()
        self.sc = ScheduledCall(self.obj.func, clock=self.clock,
                                *self.default_args,
                                **self.default_kwargs)

    def _check(self, args, kwargs):
        if args is None:
            self.assertIsNone(self.obj.args)
        else:
            self.assertTupleEqual(self.obj.args, args)

        if kwargs is None:
            self.assertIsNone(self.obj.kwargs)
        else:
            self.assertEqual(self.obj.kwargs, kwargs)

    def test_init(self):
        # test initializing ScheduledCall without overriding its clock
        sc = ScheduledCall(self.obj.func, *self.default_args,
                           **self.default_kwargs)
        sc.schedule()
        sc.cancel()

    def test_get_time_and_is_scheduled(self):
        self.clock.advance(10)

        self.assertFalse(self.sc.is_scheduled())
        self.assertEqual(self.sc.get_time(), 0)
        self.sc.schedule(5)
        self.assertTrue(self.sc.is_scheduled())
        self.assertEqual(self.sc.get_time(), 15)
        self.clock.advance(5)
        self.assertFalse(self.sc.is_scheduled())
        self.assertEqual(self.sc.get_time(), 0)

    def test_no_delay(self):
        self.sc.schedule()
        self._check(None, None)
        self.clock.advance(0)
        self._check(self.default_args, self.default_kwargs)

    def test_default(self):
        self.assertTrue(self.sc.schedule(5))
        self._check(None, None)
        self.clock.advance(1)
        self.assertFalse(self.sc.schedule(1))
        self.clock.advance(2)
        self._check(None, None)
        self.clock.advance(3)
        self._check(self.default_args, self.default_kwargs)

    def test_cancel(self):
        self.sc.schedule(5)
        self.clock.advance(3)
        self.sc.cancel()
        self.clock.advance(3)
        self._check(None, None)
        self.assertTrue(self.sc.schedule(1))
        self.clock.advance(1)
        self._check(self.default_args, self.default_kwargs)

    def test_overwrite(self):
        over_args = ('crawlmi',)
        over_kwargs = {'a': 50, 'd': 'e'}
        self.sc.schedule(5, *over_args, **over_kwargs)
        self.clock.advance(5)
        self._check(over_args, over_kwargs)

    def test_partial_overwrite(self):
        over_args = ('crawlmi',)
        self.sc.schedule(5, *over_args)
        self.clock.advance(5)
        self._check(over_args, {})

    def test_nested_schedule(self):
        def func(*args, **kwargs):
            self.obj.func(*args, **kwargs)
            self.sc.schedule()
        self.sc.func = func
        self.sc.schedule()
        self.assertEqual(self.obj.num_calls, 0)
        self.clock.advance(0)
        self.assertEqual(self.obj.num_calls, 1)
        self.clock.advance(0)
        self.assertEqual(self.obj.num_calls, 2)

    def test_nested_schedule_delay(self):
        args1 = ('a',)
        kwargs1 = {'a': 'b'}
        args2 = ('b',)
        kwargs2 = {'b': 'c'}

        def func(*args, **kwargs):
            self.obj.func(*args, **kwargs)
            self.sc.schedule(4, *args2, **kwargs2)
        self.sc.func = func
        self.sc.schedule(3, *args1, **kwargs1)
        self.clock.advance(3)
        self.assertIsNotNone(self.sc._call)
        self._check(args1, kwargs1)
        self.clock.advance(3)
        self._check(args1, kwargs1)
        self.clock.advance(1)
        self._check(args2, kwargs2)
Beispiel #6
0
class Slot(object):
    '''Slot represents a queue of requests for one particular domain.
    It respects both DOWNLOAD_DELAY and CONCURRENT_REQUESTS_PER_DOMAIN.
    '''

    def __init__(self, download_handler, concurrency, delay, randomize_delay,
                 clock=None):
        self.download_handler = download_handler
        self.concurrency = concurrency
        self.delay = delay
        self.randomize_delay = randomize_delay
        self.in_progress = set()  # request waiting to be downloaded
        self.transferring = set()  # requests being downloaded (subset of `in_progress`)
        self.last_download_time = 0
        self.queue = MemoryQueue()  # queue of (request, deferred)
        # clock is used in unittests
        self.clock = clock or reactor
        self.delayed_processing = ScheduledCall(self._process, clock=self.clock)

    def enqueue(self, request, dfd):
        '''Main entry point.
        Put the new request to the queue and if possible, start downloading it.
        '''
        def remove_in_progress(response):
            self.in_progress.remove(request)
            return response
        self.in_progress.add(request)
        dfd.addBoth(remove_in_progress)
        self.queue.push((request, dfd))
        self._process()

    @property
    def free_slots(self):
        return self.concurrency - len(self.transferring)

    def is_idle(self):
        return len(self.in_progress) == 0

    def _process(self):
        '''Process the requests in the queue, while respecting the delay and
        concurrency.
        '''
        if self.delayed_processing.is_scheduled() or self._schedule_delay():
            return

        while self.queue and self.free_slots > 0:
            self.last_download_time = self.clock.seconds()
            request, downloaded_dfd = self.queue.pop()
            dfd = self._download(request)
            dfd.chainDeferred(downloaded_dfd)
            if self._schedule_delay():
                return

    def _schedule_delay(self):
        if self.delay:
            penalty = (self.last_download_time + self.get_download_delay() -
                       self.clock.seconds())
            if penalty > 0:
                # following schedule should always be successfull, because
                # `_schedule_delay()` is only called from within `_process()`
                self.delayed_processing.schedule(penalty)
                return True
        return False

    def _download(self, request):
        dfd = defer.succeed(request)
        # download the response
        dfd.addCallback(self.download_handler.download_request)

        # it is VERY important to wrap the failure into a new object!
        # For errors like ConnectionLost, the same Failure object is returned
        # everytime and we cannot use 'failure.request' field.
        def wrap_failure(failure):
            return Failure(failure.value)
        dfd.addErrback(wrap_failure)

        # put the request into the set of `transferring` to block other requests
        # after the response is downloaded, remove it from `transferring`
        def remove_transferring(response):
            self.transferring.remove(request)
            self._process()  # process unblocked requests
            return response
        self.transferring.add(request)
        dfd.addBoth(remove_transferring)
        return dfd

    def get_download_delay(self):
        if self.randomize_delay:
            return random.uniform(0.5 * self.delay, 1.5 * self.delay)
        return self.delay
Beispiel #7
0
class Engine(object):
    '''
    WARNING: don't stop() and start() engine. Use pause() and unpause(),
    instead.
    '''

    # how many seconds to wait between the checks of response_queue
    QUEUE_CHECK_FREQUENCY = 0.1
    # how often to check is still paused
    PAUSED_CHECK_FREQUENCY = 5
    # how often to check if being idle
    IDLE_CHECK_FREQUENCY = 5

    def __init__(self, settings, project, command_invoked='', clock=None):
        '''Constructor of Engine should be very lightweight, so that things
        can be easily unittested. For any more complicated initialization
        use `setup()`.
        '''
        self.settings = settings
        self.project = project
        self.spiders = SpiderManager(settings)

        self.stop_if_idle = True
        self.initialized = False  # True, when `setup()` has been called
        # name of the command invoking the engine. E.g. `crawl`, `shell`, etc.
        self.command_invoked = command_invoked

        self.spider = None
        self.pending_requests = 0
        self.running = False
        self.paused = False
        # clock is used in unittests
        self.clock = clock or reactor
        self.processing = ScheduledCall(self._process_queue, clock=self.clock)

    def set_spider(self, spider):
        self.spider = spider
        self.settings.spider_settings = spider.spider_settings()

    def setup(self):
        assert self.spider is not None, 'Spider is not set in Engine.'

        # IMPORTANT: order of the following initializations is very important
        # so please, think twice about any changes to it

        # initialize logging
        if self.settings.get_bool('LOG_ENABLED'):
            log.start(
                self.settings['LOG_FILE'],
                self.settings['LOG_LEVEL'],
                self.settings['LOG_STDOUT'],
                self.settings['LOG_ENCODING'])

        # initialize signals
        self.signals = SignalManager(self)

        #initialize stats
        stats_cls = load_object(self.settings.get('STATS_CLASS'))
        self.stats = stats_cls(self)

        # initialize downloader
        self.request_queue = PriorityQueue(lambda _: MemoryQueue())
        self.response_queue = ResponseQueue(
            self.settings.get_int('RESPONSE_ACTIVE_SIZE_LIMIT'))
        self.downloader = Downloader(self.settings, self.request_queue,
                                     self.response_queue, clock=self.clock)

        # initialize extensions
        self.extensions = ExtensionManager(self)
        # initialize downloader pipeline
        self.pipeline = PipelineManager(self)

        self.initialized = True

        # now that everything is ready, set the spider's engine
        self.spider.set_engine(self)

    def crawl_start_requests(self):
        # process start requests from spider
        try:
            requests = self.spider.start_requests()
            for req in arg_to_iter(requests):
                self.download(req)
        except:
            log.err(Failure(), 'Error when processing start requests.')

    def start(self):
        assert self.initialized, 'Engine is not initialized. Call `setup()` to initialize it.'

        self.start_time = time()
        self.running = True
        self.signals.send(signal=signals.engine_started)
        self.processing.schedule(self.QUEUE_CHECK_FREQUENCY)

    def stop(self, reason=''):
        assert self.running, 'Engine is not running.'
        self.running = False

        def _stop(_):
            self.processing.cancel()
            self.downloader.close()
            self.request_queue.close()
            self.response_queue.close()
            log.msg(format='Engine stopped (%(reason)s)', reason=reason)
            self.signals.send(signal=signals.engine_stopped, reason=reason)
            self.stats.dump_stats()

        dfd = defer_succeed(reason, clock=self.clock)
        dfd.addBoth(_stop)
        return dfd

    def pause(self):
        self.paused = True

    def unpause(self):
        self.paused = False

    def download(self, request):
        '''"Download" the given request. First pass it through the downloader
        pipeline.
            - if the request is received, push it to `request_queue`
            - if the response is received , push it to `response_queue`
        '''
        def _success(request_or_response):
            if isinstance(request_or_response, Request):
                self.signals.send(signal=signals.request_received,
                                  request=request_or_response)
                if self.running:
                    self.request_queue.push(request_or_response.priority,
                                            request_or_response)
            elif isinstance(request_or_response, Response):
                request_or_response.request = request
                if self.running:
                    self.response_queue.push(request_or_response)

        def _failure(failure):
            failure.request = request
            dfd = defer_fail(failure, clock=self.clock)
            dfd.addBoth(self._handle_pipeline_result)
            dfd.addBoth(self._finalize_download)
            return dfd

        self.pending_requests += 1
        d = defer_succeed(request, clock=self.clock)
        d.addCallback(self.pipeline.process_request)
        d.addCallbacks(_success, _failure)
        return d

    def is_idle(self):
        return self.pending_requests == 0 and len(self.response_queue) == 0

    def _process_queue(self):
        if not self.running:
            return
        elif self.paused:
            self.processing.schedule(self.PAUSED_CHECK_FREQUENCY)
        elif self.response_queue:
            response = self.response_queue.pop()
            if isinstance(response, Response):
                self.signals.send(signal=signals.response_downloaded,
                                  response=response)
            dfd = defer_result(response, clock=self.clock)
            dfd.addBoth(self.pipeline.process_response)
            dfd.addBoth(self._handle_pipeline_result)
            dfd.addBoth(self._finalize_download)
            dfd.addBoth(lambda _: self.processing.schedule(0))
        elif self.is_idle():
            # send `spider_idle` signal
            res = self.signals.send(signal=signals.spider_idle,
                                    dont_log=DontStopEngine)
            dont_stop = any(isinstance(x, Failure) and
                            isinstance(x.value, DontStopEngine)
                            for _, x in res)
            # more requests have been scheduled
            if not self.is_idle():
                self.processing.schedule(0)
            # slow down a little, but still run
            elif dont_stop or not self.stop_if_idle:
                self.processing.schedule(self.IDLE_CHECK_FREQUENCY)
            else:
                self.stop('finished')
        else:
            self.processing.schedule(self.QUEUE_CHECK_FREQUENCY)

    def _finalize_download(self, _):
        self.pending_requests -= 1

    def _handle_pipeline_result(self, result):
        if result is None:
            pass
        elif isinstance(result, Request):
            self.download(result)
        else:
            assert isinstance(result, (Response, Failure))
            request = result.request
            if isinstance(result, Response):
                flags = ' %s' % result.flags if result.flags else ''
                log.msg(format='Crawled %(url)s [%(status)s]%(flags)s',
                        level=log.DEBUG, url=result.url, status=result.status,
                        flags=flags)
                self.signals.send(signal=signals.response_received,
                                  response=result)
            else:
                self.signals.send(signal=signals.failure_received,
                                  failure=result)
            dfd = defer_result(result, clock=self.clock)
            dfd.addCallbacks(request.callback or self.spider.parse,
                             request.errback)
            dfd.addCallbacks(
                self._handle_spider_output,
                self._handle_spider_error,
                callbackKeywords={'request': request},
                errbackKeywords={'request': request})
            return dfd

    def _handle_spider_output(self, result, request):
        result = arg_to_iter(result)
        for request in result:
            assert isinstance(request, Request), \
                'spider must return None, request or iterable of requests'
            self.download(request)

    def _handle_spider_error(self, failure, request):
        error = failure.value
        if isinstance(error, StopEngine):
            self.stop(error.reason)
            return
        # set `request` in a case the error was raised inside the spider
        failure.request = request
        self.signals.send(signal=signals.spider_error, failure=failure)
        if not getattr(failure.value, 'quiet', False):
            log.err(failure, 'Error when downloading %s' % request)

    def __str__(self):
        return '<%s at 0x%0x>' % (type(self).__name__, id(self))
    __repr__ = __str__