def get(self):
try:
self._get()
except Exception as e:
logger.error(e,exc_info=True)
self.write('%s : %s'%(type(e),e))
self.set_status(500)
def _fetch(self, query, url, recurse=True):
logger.info("fetching: %s", url)
try:
response = yield cyclone_fetch(url, headers=self.headers)
self.handle_response(query, url, response, recurse=recurse)
except EnhanceYourCalmException:
logger.error("HTTP 429 Too Many Requests ; will retry call later")
reactor.callLater(5, self._fetch, *[query, url], **{"recurse": recurse})
except Exception as e:
logger.error("url fetch error: %s:%s", type(e), e, exc_info=True)
def safe_listen(self):
abnormal_exception = None
try:
self.listen_stream()
except tweetstream.ReconnectImmediatelyError as rie:
logger.error('got %s in stream: %s',
type(rie),
rie)
self.update_wait()
except tweetstream.ReconnectLinearlyError as rie:
logger.error('got %s in stream: %s',
type(rie),
rie)
self.update_wait(amount=1)
except tweetstream.ReconnectExponentiallyError as rie:
logger.error('got %s in stream: %s',
type(rie),
rie)
self.update_wait(amount=1, factor=2)
except Exception as e:
logger.error('got abnormal exception in stream: %s',e,exc_info=True)
abnormal_exception = e
finally:
if abnormal_exception:
return abnormal_exception
if self.last_wait and time.time() - self.last_wait > 15*60:
logger.info('resetting time wait (last wait: %s)',
self.last_wait)
self.update_wait(reset=True)
self.last_wait = time.time()
logger.info("will sleep %s secs",self.wait_secs)
time.sleep(self.wait_secs)