def follow_stream(self, conf, follow, track):
conn = Microblog("twitter", conf, streaming=True)
try:
for tweet in conn.search_stream(follow, track):
self.fact.update_timeout()
if self.fact.status == "closed":
break
if tweet:
if tweet.get("disconnect") or tweet.get("hangup"):
self.log("Disconnected %s" % ("(timeout)" if tweet.get("heartbeat_timeout") else tweet), error=True)
break
if tweet.get('timeout'):
continue # heartbeat
if tweet.get('text'):
self.pile.insert(0, tweet)
else:
try:
self.fact.db['tweets'].update(spec={'id': tweet['delete']['status']['id']}, document={'$set': {'deleted': True}}, multi=True)
if config.DEBUG:
self.log("Mark a tweet as deleted: %s" % tweet['delete']['status']['id'], hint=True)
except:
if config.DEBUG:
self.log(tweet, hint=True)
except socket.error as e:
self.log("Stream lost connection with %s: %s" % (type(e), e), error=True)
except Exception as e:
if str(e).strip():
self.log("Stream crashed with %s: %s" % (type(e), e), error=True)
else:
self._handle_error(failure.Failure(e), "following", "stream")
self.depiler.stop()
self.flush_tweets()
self.log("Feeder closed.", hint=True)
if self.fact.status != "closed":
self.fact.status = "stopped"
def follow_stream(self, conf, follow, track):
conn = Microblog("twitter", conf, streaming=True)
ct = 0
tweets = []
flush = time.time() + 29
try:
for tweet in conn.search_stream(follow, track):
if self.fact.status.startswith("clos"):
self._flush_tweets(tweets)
self.log("Feeder closed.", "stream", hint=True)
break
elif not tweet or not tweet.get('text'):
if tweet and not tweet.get('delete'):
self.log(tweet, "stream")
continue
elif tweet.get("disconnect"):
self._flush_tweets(tweets)
self.log("Disconnected %s" % tweet, "stream", error=True)
break
tweets.append(tweet)
ct += 1
if ct > 9 or time.time() > flush:
self._flush_tweets(tweets)
ct = 0
tweets = []
flush = time.time() + 29
except Exception as e:
self.log(e, "stream", error=True)
self._handle_error(e.traceback, "while followoing", "stream")
return
def follow_stream(self, conf, follow, track):
conn = Microblog("twitter", conf, streaming=True)
try:
for tweet in conn.search_stream(follow, track):
self.fact.update_timeout()
if self.fact.status == "closed":
break
if tweet:
if tweet.get("disconnect") or tweet.get("hangup"):
self.log("Disconnected %s" %
("(timeout)" if tweet.get("heartbeat_timeout")
else tweet),
error=True)
break
if tweet.get('timeout'):
continue # heartbeat
if tweet.get('id_str'):
tweet = reformat_extended_tweets(tweet)
self.pile.insert(0, tweet)
else:
try:
self.fact.db['tweets'].update(
spec={'id': tweet['delete']['status']['id']},
document={'$set': {
'deleted': True
}},
multi=True)
if config.DEBUG:
self.log("Mark a tweet as deleted: %s" %
tweet['delete']['status']['id'],
hint=True)
except:
if config.DEBUG:
self.log(tweet, hint=True)
except socket.error as e:
self.log("Stream lost connection with %s: %s" % (type(e), e),
error=True)
except Exception as e:
if str(e).strip():
self.log("Stream crashed with %s: %s" % (type(e), e),
error=True)
else:
self._handle_error(failure.Failure(e), "following", "stream")
self.depiler.stop()
self.flush_tweets()
self.log("Feeder closed.", hint=True)
if self.fact.status != "closed":
self.fact.status = "stopped"
