def download(self, request):
'''"Download" the given request. First pass it through the downloader
pipeline.
- if the request is received, push it to `request_queue`
- if the response is received , push it to `response_queue`
'''
def _success(request_or_response):
if isinstance(request_or_response, Request):
self.signals.send(signal=signals.request_received,
request=request_or_response)
if self.running:
self.request_queue.push(request_or_response.priority,
request_or_response)
elif isinstance(request_or_response, Response):
request_or_response.request = request
if self.running:
self.response_queue.push(request_or_response)
def _failure(failure):
failure.request = request
dfd = defer_fail(failure, clock=self.clock)
dfd.addBoth(self._handle_pipeline_result)
dfd.addBoth(self._finalize_download)
return dfd
self.pending_requests += 1
d = defer_succeed(request, clock=self.clock)
d.addCallback(self.pipeline.process_request)
d.addCallbacks(_success, _failure)
return d
def download(self, request):
'''"Download" the given request. First pass it through the downloader
pipeline.
- if the request is received, push it to `request_queue`
- if the response is received , push it to `response_queue`
'''
def _success(request_or_response):
if isinstance(request_or_response, Request):
self.signals.send(signal=signals.request_received,
request=request_or_response)
if self.running:
self.request_queue.push(request_or_response.priority,
request_or_response)
elif isinstance(request_or_response, Response):
request_or_response.request = request
if self.running:
self.response_queue.push(request_or_response)
def _failure(failure):
failure.request = request
dfd = defer_fail(failure, clock=self.clock)
dfd.addBoth(self._handle_pipeline_result)
dfd.addBoth(self._finalize_download)
return dfd
self.pending_requests += 1
d = defer_succeed(request, clock=self.clock)
d.addCallback(self.pipeline.process_request)
d.addCallbacks(_success, _failure)
return d
def stop(self, reason=''):
assert self.running, 'Engine is not running.'
self.running = False
def _stop(_):
self.processing.cancel()
self.downloader.close()
self.request_queue.close()
self.response_queue.close()
log.msg(format='Engine stopped (%(reason)s)', reason=reason)
self.signals.send(signal=signals.engine_stopped, reason=reason)
self.stats.dump_stats()
dfd = defer_succeed(reason, clock=self.clock)
dfd.addBoth(_stop)
return dfd
def stop(self, reason=''):
assert self.running, 'Engine is not running.'
self.running = False
def _stop(_):
self.processing.cancel()
self.downloader.close()
self.request_queue.close()
self.response_queue.close()
log.msg(format='Engine stopped (%(reason)s)', reason=reason)
self.signals.send(signal=signals.engine_stopped, reason=reason)
self.stats.dump_stats()
dfd = defer_succeed(reason, clock=self.clock)
dfd.addBoth(_stop)
return dfd
