def process_items (self):
if self.api:
request = Request(self.base_url, callback=self.search_by_api, dont_filter=True, errback=self.error_callback)
request.meta['api'] = self.api
request.meta['cookiejar'] = 'FracFocusScraper:%s' % self.api
self.log('** Scraping API %s' % self.api, log.INFO)
yield request
else:
for item in NrcBot.process_items (self):
yield item
def process_items(self):
if self.api:
request = Request(self.base_url,
callback=self.search_by_api,
dont_filter=True,
errback=self.error_callback)
request.meta['api'] = self.api
request.meta['cookiejar'] = 'FracFocusScraper:%s' % self.api
self.log('** Scraping API %s' % self.api, log.INFO)
yield request
else:
for item in NrcBot.process_items(self):
yield item
def process_items (self):
self.schedule_tasks ()
for item in NrcBot.process_items (self):
yield item
def __init__(self, **kwargs):
# TODO: make class FusionTableClient. in connect(), authenticate usgin U/P from settings
# self.ft = FusionTableClient()
# self.ft.connect()
NrcBot.__init__(self)
def __init__ (self, **kwargs):
self.api = kwargs.get('api',None)
if kwargs.has_key('state'):
FracFocusScraper.job_item_limit = 1
NrcBot.__init__(self, **kwargs)
def __init__(self, **kwargs):
NrcBot.__init__(self, **kwargs)
locale.setlocale (locale.LC_ALL, 'en_US.UTF-8')
def process_items(self):
self.schedule_tasks()
for item in NrcBot.process_items(self):
yield item
def __init__(self, **kwargs):
NrcBot.__init__(self, **kwargs)
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)
def __init__(self, **kwargs):
# TODO: make class FusionTableClient. in connect(), authenticate usgin U/P from settings
# self.ft = FusionTableClient()
# self.ft.connect()
NrcBot.__init__(self)
def __init__(self, **kwargs):
self.api = kwargs.get('api', None)
if kwargs.has_key('state'):
FracFocusScraper.job_item_limit = 1
NrcBot.__init__(self, **kwargs)
