def process_items (self): if self.api: request = Request(self.base_url, callback=self.search_by_api, dont_filter=True, errback=self.error_callback) request.meta['api'] = self.api request.meta['cookiejar'] = 'FracFocusScraper:%s' % self.api self.log('** Scraping API %s' % self.api, log.INFO) yield request else: for item in NrcBot.process_items (self): yield item
def process_items(self): if self.api: request = Request(self.base_url, callback=self.search_by_api, dont_filter=True, errback=self.error_callback) request.meta['api'] = self.api request.meta['cookiejar'] = 'FracFocusScraper:%s' % self.api self.log('** Scraping API %s' % self.api, log.INFO) yield request else: for item in NrcBot.process_items(self): yield item
def process_items (self): self.schedule_tasks () for item in NrcBot.process_items (self): yield item
def __init__(self, **kwargs): # TODO: make class FusionTableClient. in connect(), authenticate usgin U/P from settings # self.ft = FusionTableClient() # self.ft.connect() NrcBot.__init__(self)
def __init__ (self, **kwargs): self.api = kwargs.get('api',None) if kwargs.has_key('state'): FracFocusScraper.job_item_limit = 1 NrcBot.__init__(self, **kwargs)
def __init__(self, **kwargs): NrcBot.__init__(self, **kwargs) locale.setlocale (locale.LC_ALL, 'en_US.UTF-8')
def process_items(self): self.schedule_tasks() for item in NrcBot.process_items(self): yield item
def __init__(self, **kwargs): NrcBot.__init__(self, **kwargs) logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
def __init__(self, **kwargs): self.api = kwargs.get('api', None) if kwargs.has_key('state'): FracFocusScraper.job_item_limit = 1 NrcBot.__init__(self, **kwargs)