def follow_stream(self, conf, follow, track): conn = Microblog("twitter", conf, streaming=True) try: for tweet in conn.search_stream(follow, track): self.fact.update_timeout() if self.fact.status == "closed": break if tweet: if tweet.get("disconnect") or tweet.get("hangup"): self.log("Disconnected %s" % ("(timeout)" if tweet.get("heartbeat_timeout") else tweet), error=True) break if tweet.get('timeout'): continue # heartbeat if tweet.get('text'): self.pile.insert(0, tweet) else: try: self.fact.db['tweets'].update(spec={'id': tweet['delete']['status']['id']}, document={'$set': {'deleted': True}}, multi=True) if config.DEBUG: self.log("Mark a tweet as deleted: %s" % tweet['delete']['status']['id'], hint=True) except: if config.DEBUG: self.log(tweet, hint=True) except socket.error as e: self.log("Stream lost connection with %s: %s" % (type(e), e), error=True) except Exception as e: if str(e).strip(): self.log("Stream crashed with %s: %s" % (type(e), e), error=True) else: self._handle_error(failure.Failure(e), "following", "stream") self.depiler.stop() self.flush_tweets() self.log("Feeder closed.", hint=True) if self.fact.status != "closed": self.fact.status = "stopped"
def follow_stream(self, conf, follow, track): conn = Microblog("twitter", conf, streaming=True) ct = 0 tweets = [] flush = time.time() + 29 try: for tweet in conn.search_stream(follow, track): if self.fact.status.startswith("clos"): self._flush_tweets(tweets) self.log("Feeder closed.", "stream", hint=True) break elif not tweet or not tweet.get('text'): if tweet and not tweet.get('delete'): self.log(tweet, "stream") continue elif tweet.get("disconnect"): self._flush_tweets(tweets) self.log("Disconnected %s" % tweet, "stream", error=True) break tweets.append(tweet) ct += 1 if ct > 9 or time.time() > flush: self._flush_tweets(tweets) ct = 0 tweets = [] flush = time.time() + 29 except Exception as e: self.log(e, "stream", error=True) self._handle_error(e.traceback, "while followoing", "stream") return
def follow_stream(self, conf, follow, track): conn = Microblog("twitter", conf, streaming=True) try: for tweet in conn.search_stream(follow, track): self.fact.update_timeout() if self.fact.status == "closed": break if tweet: if tweet.get("disconnect") or tweet.get("hangup"): self.log("Disconnected %s" % ("(timeout)" if tweet.get("heartbeat_timeout") else tweet), error=True) break if tweet.get('timeout'): continue # heartbeat if tweet.get('id_str'): tweet = reformat_extended_tweets(tweet) self.pile.insert(0, tweet) else: try: self.fact.db['tweets'].update( spec={'id': tweet['delete']['status']['id']}, document={'$set': { 'deleted': True }}, multi=True) if config.DEBUG: self.log("Mark a tweet as deleted: %s" % tweet['delete']['status']['id'], hint=True) except: if config.DEBUG: self.log(tweet, hint=True) except socket.error as e: self.log("Stream lost connection with %s: %s" % (type(e), e), error=True) except Exception as e: if str(e).strip(): self.log("Stream crashed with %s: %s" % (type(e), e), error=True) else: self._handle_error(failure.Failure(e), "following", "stream") self.depiler.stop() self.flush_tweets() self.log("Feeder closed.", hint=True) if self.fact.status != "closed": self.fact.status = "stopped"