def on_data(self, tweet):
try:
tweet = json.loads(tweet)
raw_tweet= {"id":tweet['id'],
"user_id":tweet['user']['id'],
"screen_name":tweet['user']['screen_name'],
"created_at":tweet['created_at'],
"text":tweet['text']}
if str(raw_tweet['user_id']) in load_follow_list():
self.log(raw_tweet['screen_name'])
self.log(self.counter)
self.collection.insert(raw_tweet, continue_on_error=True)
self.counter +=1
else:
self.log(raw_tweet['screen_name'])
except:
self.log("database error but streamer will continue")
return True
def on_data(self, tweet):
try:
tweet = json.loads(tweet)
raw_tweet = {
"id": tweet['id'],
"user_id": tweet['user']['id'],
"screen_name": tweet['user']['screen_name'],
"created_at": tweet['created_at'],
"text": tweet['text']
}
if str(raw_tweet['user_id']) in load_follow_list():
self.log(raw_tweet['screen_name'])
self.log(self.counter)
self.collection.insert(raw_tweet, continue_on_error=True)
self.counter += 1
else:
self.log(raw_tweet['screen_name'])
except:
self.log("database error but streamer will continue")
return True
print >> sys.stderr, 'Encountered error with status code:', status_code
return True # Don't kill the stream
def on_timeout(self):
print >> sys.stderr, 'Timeout...'
return True # Don't kill the stream
if __name__ == '__main__':
credentials = load_credentials()
auth = tweepy_auth(credentials, user=True)
api = tweepy_api(auth)
print 'Running streamer'
with open('matches.csv', 'a') as matchCsv:
matchCsv.write(
"Tweet,DealProduct,DealPrice,MarketProduct,MarketPrice,HoursLeft,BidCount,PositiveFeedbackPct\n"
)
with open('trades.csv', 'a') as tradeCsv:
tradeCsv.write(
"Tweet,DealProduct,DealPrice,MarketProduct,MarketPrice,HoursLeft,BidCount,PositiveFeedbackPct\n"
)
sapi = streaming.Stream(auth, CustomStreamListener(api))
try:
sapi.filter(follow=load_follow_list())
except KeyboardInterrupt:
print "Twitter streaming interrupted"
except UnicodeError:
print "Unicode error"
sapi.filter(follow=load_follow_list())
print "TRADE: " + tweetText + "," + dealProduct + "," + dealPrice + "," + marketProduct + "," + marketPrice + "," + str(hoursLeft) + "," + str(bidCount) + "," + str(positiveFeedbackPercent) + "\n"
tradeCsv.write(tweetText + "," + dealProduct + "," + str(dealPrice) + "," + marketProduct + "," + str(marketPrice) + "," + str(hoursLeft) + "," + str(bidCount) + "," + str(positiveFeedbackPercent) + "\n")
return True
def on_error(self, status_code):
print >> sys.stderr, 'Encountered error with status code:', status_code
return True # Don't kill the stream
def on_timeout(self):
print >> sys.stderr, 'Timeout...'
return True # Don't kill the stream
if __name__ == '__main__':
credentials = load_credentials()
auth = tweepy_auth(credentials, user=True)
api = tweepy_api(auth)
print 'Running streamer'
with open('matches.csv','a') as matchCsv:
matchCsv.write("Tweet,DealProduct,DealPrice,MarketProduct,MarketPrice,HoursLeft,BidCount,PositiveFeedbackPct\n")
with open('trades.csv','a') as tradeCsv:
tradeCsv.write("Tweet,DealProduct,DealPrice,MarketProduct,MarketPrice,HoursLeft,BidCount,PositiveFeedbackPct\n")
sapi = streaming.Stream(auth, CustomStreamListener(api))
try:
sapi.filter(follow=load_follow_list())
except KeyboardInterrupt:
print "Twitter streaming interrupted"
except UnicodeError:
print "Unicode error"
sapi.filter(follow=load_follow_list())
