def __init__(self, crawler): self._commander = Commander( crawler.settings.get('API_SCRAPOXY'), crawler.settings.get('API_SCRAPOXY_PASSWORD')) self._WAIT_FOR_SCALE = crawler.settings.get('WAIT_FOR_SCALE') or 120 crawler.signals.connect(self.spider_opened, signals.spider_opened) crawler.signals.connect(self.spider_closed, signals.spider_closed)
def __init__(self, crawler): """Access the settings of the crawler to connect to Scrapoxy. """ self._http_status_codes = crawler.settings.get( 'BLACKLIST_HTTP_STATUS_CODES', [503]) self._sleep_min = crawler.settings.get('SCRAPOXY_SLEEP_MIN', 60) self._sleep_max = crawler.settings.get('SCRAPOXY_SLEEP_MAX', 180) self._commander = Commander( crawler.settings.get('API_SCRAPOXY'), crawler.settings.get('API_SCRAPOXY_PASSWORD'))
def __init__(self): self.settings = get_project_settings() self.commander = Commander(self.settings.get('API_SCRAPOXY'), self.settings.get('API_SCRAPOXY_PASSWORD')) configure_logging(settings=None, install_root_handler=False) logging.config.dictConfig(self.settings['LOGGING_SETTINGS'])