def _print_setting(self, opts):
if opts.get:
print settings_.get(opts.get)
elif opts.getbool:
print settings_.getbool(opts.getbool)
elif opts.getint:
print settings_.getint(opts.getint)
elif opts.getfloat:
print settings_.getfloat(opts.getfloat)
elif opts.getlist:
print settings_.getlist(opts.getlist)
def __init__(self):
self.interval = settings.getfloat('LOGSTATS_INTERVAL')
if not self.interval:
raise NotConfigured
self.slots = {}
self.multiplier = 60.0 / self.interval
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.response_received, signal=signals.response_received)
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
dispatcher.connect(self.engine_started, signal=signals.engine_started)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
def __init__(self, download_delay=None, max_concurrent_requests=None):
if download_delay is None:
self._download_delay = settings.getfloat("DOWNLOAD_DELAY")
else:
self._download_delay = float(download_delay)
if self._download_delay:
self.max_concurrent_requests = 1
elif max_concurrent_requests is None:
self.max_concurrent_requests = settings.getint("CONCURRENT_REQUESTS_PER_SPIDER")
else:
self.max_concurrent_requests = max_concurrent_requests
if self._download_delay and settings.getbool("RANDOMIZE_DOWNLOAD_DELAY"):
# same policy as wget --random-wait
self.random_delay_interval = (0.5 * self._download_delay, 1.5 * self._download_delay)
else:
self.random_delay_interval = None
self.active = set()
self.queue = []
self.transferring = set()
self.closing = False
self.lastseen = 0
self.next_request_calls = set()