class Scraper(object): def __init__(self, trending): self.trending = trending self.tc = TwitterClient() self.start_twitter_stream() thread = Thread(target=self.start_stock_twits_stream) thread.start() """ Soft copy of trending @:param trending """ def set_trending(self, trending): for i in range(len(trending)): self.trending[i] = trending[i] def start_twitter_stream(self): self.tc.get_tweets_stream(self.trending) def close_twitter_stream(self): pass self.tc.close_stream() def start_stock_twits_stream(self): while (StockInfo.flag): messages = stc.get_stock_streams(self.trending)['messages'] for message in messages: # Check what stock the message is talking about for stock in self.trending: if stock not in message['body']: continue if stock not in StockInfo.stock_info: StockInfo.stock_info[stock] = PriorityQueue(maxsize=50) # Check if someone is spamming stream if self.isDup(StockInfo.stock_info[stock], message['body']): continue if StockInfo.stock_info[stock].full(): StockInfo.stock_info[stock].get() tweet_sentiment = self.tc.get_tweet_sentiment( message['body']) MyLogger.dwrite(message['body'], str(tweet_sentiment)) StockInfo.stock_info[stock].put( (message['created_at'], tweet_sentiment, message['body'])) # Rate limit is 400 requests per hour = 1/9 requests per second t.sleep(10) def isDup(self, q: PriorityQueue, text): queue_arr = list(q.queue) for data in queue_arr: if data == text: return True return False