def work():
# Opening a CSV file to save the gathered tweets
with open(filename+".csv", 'w', encoding="UTF-8") as file:
global writer
writer = csv.writer(file)
# Add a header row to the CSV
writer.writerow(["Tweet", "Matched Keywords", "Date", "User",
"Source", "Tweet ID", "Tweet URL"])
# Initializing the twitter streap Stream
try:
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=keywords)
# Stop temporarily when hitting Twitter rate Limit
except tweepy.RateLimitError:
print("RateLimitError...waiting ~15 minutes to continue")
time.sleep(1001)
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=[keywords])
# Stop temporarily when getting a timeout or connection error
except (Timeout, ssl.SSLError, ReadTimeoutError,
ConnectionError) as exc:
print("Timeout/connection error...waiting ~15 minutes to continue")
time.sleep(1001)
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=[keywords])
# Stop temporarily when getting other errors
except tweepy.TweepError as e:
if 'Failed to send request:' in e.reason:
print("Time out error caught.")
time.sleep(1001)
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=[keywords])
else:
print("Other error with this user...passing")
pass
rawConn.close()
class StreamListener(StreamListener):
mentionCounter = 0
def on_status(self, status): #function acts as a generator/iterator
if status.retweeted:
return
bodyText = (str(status.text)).lower()
i = 0
for i in range (0,len(coinList)):
if ((tally(bodyText, coinList[i], coinInitialList[i]) != 0) and not("https://" in bodyText) and not("rt" in bodyText)): #checks to see if comment is relevant before writing
rawConn = sqlite3.connect('twitterRawData.db')
rawCursor = rawConn.cursor()
currentTime = datetime.datetime.now()
rawCursor.execute("INSERT INTO twitterRawData VALUES (?,?,0)", (bodyText,currentTime))
rawCursor.close()
rawConn.commit()
rawConn.close()
print(bodyText)
def error(self, statusCode): #handles excessive request error
if statusCode == 420:
return False
mentionCounter = 0
stream_Listener = StreamListener()
stream = Stream(auth=api.auth, listener = stream_Listener)
stream.filter(track=trackingList)
user_id = str(status.user.id)
sentiment = sentiment_score(text)
positivity = round(sentiment['pos'], 4)
negativity = round(sentiment['neg'], 4)
compound = round(sentiment['compound'], 4)
polarity = round((TextBlob(text)).sentiment.polarity, 4)
city_id = get_city_id(centroid_lat, centroid_long)
get_or_create_tweet(user_id, loc, id_str, created, centroid_lat, centroid_long, text, city_id)
def on_exception(self, exception):
print(exception)
return
def on_error(self, status_code):
if status_code == 420:
return False
flu = ['flu', 'influenza', 'cough', 'fever', 'sore throat', 'headache',
'phlegm', 'runny nose', 'stuffy nose', 'Robitussin',
'dayquil', 'nyquil', 'tamiflu', 'vomit', 'body ache', 'mucinex',
'pneumonia', 'vomit', 'bodyache', 'medicine']
stream_listener = StreamListener()
stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
while True:
try:
stream.filter(track=flu)
except (ProtocolError, AttributeError, ReadTimeoutError):
continue
test = stream.filter(track=flu)
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
# Create a wrapper for the API provided by Twitter
api = tweepy.API(auth,wait_on_rate_limit=True)
# Setting up the keywords, hashtag or mentions we want to listen
keywords = ["#music", "music", "music review", "song", "album"]
# Set the name for CSV file where the tweets will be saved
filename = "musicreview"
#get myStream to sync up with Streamlistener and prepare a filter
myStream = tweepy.Stream(auth = api.auth, listener=StreamListener())
# We need to implement StreamListener to use Tweepy to listen to Twitter
class StreamListener(tweepy.StreamListener):
tweet_counter = 0 # Static variable and counter of how many tweet downloaded
def on_status(self, status):
try:
# saves the tweet object
tweet_object = status
# Checks if its a extended tweet (>140 characters)
def on_status(self, status):
if hasattr(status, 'retweeted_status'):
return
print type(status)
print(status.text)
def on_error(self, status_code):
if status_code == 420:
return False
#Since it wasn't explicitly mentioned, keeping the API keys as it is
#API keys
ckey = '*****'
csec = '*****'
atok = '*****'
asec = '*****'
#Auth
auth = OAuthHandler(ckey, csec)
auth.set_access_token(atok, asec)
api = API(auth)
#Query to search for tweets using emoticons values
#For example, \U0001F602 refers to 'happy'
#1F602 = happy, 1F62D = sad, 1F621= angry, 2764 = love, 1F61C = playful, 1F631 = confused
query = [u'\U0001F602']
#stream prints out the tweets matching the query to the terminal
stream = Stream(auth=api.auth, listener=StreamListener())
stream.filter(track=query, languages=["en"], stall_warnings=True, async=True)
def run(self):
"""
Initializing a Stream & Use Search Engine with User's Hashtags.
Args:
None.
Parameters:
self.app.listener (StreamListener): Initializing & stores StreamListener object.
self.app.stream (Stream): Initializing & Store Stream object with auth & listener args.
tweets_list[] (list): Temporary list in order to store specific hashtag's tweets
self.tweet_matrix (list): list of lists - Storing lists of tweets for each hashtag.
self.pb (QProgressBar): Progess bar which present the current status of the search.
self.tag_list (dict): Hashtags list itself for searching tweets.
temp_value (float): Temporary value for the continuation of the progress bar.
start (float): current time in seconds.
clock (str): current time.
self.signal (pyqtBoundSignal):Emiting the tweets list of lists back to data_and_analasys_to_excel method.
Returns:
None
"""
# Handles Twitter authetification and the connection to Twitter Streaming API
self.app.listener = StreamListener()
self.app.stream = Stream(self.app.auth, self.app.listener)
# Capture the Tweets in a Temporary List
tweets_list = []
try:
# Creating the Progress Bar and present it.
self.pb = self.__create_progress_bar(self)
self.pb.show()
# Temporary value for the continuation of the progress bar
temp_value = 0
for hashtag, search_item in zip(self.tag_list, range(1, len(self.tag_list) + 1)):
QApplication.processEvents()
# Starting time
start = time.time()
clock = datetime.now().strftime("%H:%M:%S")
cursor = Cursor(self.twitter_client.search, q=hashtag, result_type='mixed', tweet_mode='extended',
include_entities=True, lang="en").items(self.num_of_tweets)
# Use Cursor to search for hashtag and copy it into a tweets_list
for tweet in cursor:
QApplication.processEvents()
# Capture a list of tweets
tweets_list.append(tweet)
# Progress Bar Continuation's Configuration
time.sleep(0.05)
self.pb.setValue(temp_value + 100/len(self.tag_list))
temp_value += 100/len(self.tag_list)
print("Search #{} took {} from {}".format(search_item, time.time() - start, clock))
# Copy to the List of lists which stores all the tweets for each hashtags & clean the temp list
self.tweet_matrix.append(tweets_list.copy())
tweets_list.clear()
# Hiding the Progress Bar
self.pb.close()
# Disconnect from the stream
self.app.stream.disconnect()
# Terminate and delete the search thread.
self.app.search_thread.terminate()
del self.app.search_thread
# Emiting the tweets list of lists back to data_and_analasys_to_excel method.
self.signal.emit(self.tweet_matrix)
except RateLimitError as limit:
print('RateLimit Error: {0}'.format(limit))
except TweepError as error:
print('Tweepy Error: {0}'.format(error))
except Exception as e:
print('Search Error: {0}'.format(e))
# access_token = keys.access_token
# access_token_secret = keys.access_token_secret
# auth1 = tweepy.OAuthHandler(consumer_key, consumer_secret)
# auth1.set_access_token(access_token, access_token_secret)
#auth
# LOCATIONS are the longitude, latitude coordinate corners for a box that restricts the
# geographic area from which you will stream tweets. The first two define the southwest
# corner of the box and the second two define the northeast corner of the box.
LOCATIONS = [
-124.7771694,
24.520833,
-66.947028,
49.384472, # Contiguous US
-164.639405,
58.806859,
-144.152365,
71.76871, # Alaska
-160.161542,
18.776344,
-154.641396,
22.878623
] # Hawaii
stream_listener = StreamListener(api=tweepy.API(wait_on_rate_limit=True))
stream = tweepy.Stream(auth=auth,
listener=stream_listener,
tweet_mode='extended')
stream.filter(track=[
'Rabobank', '#Rabo', '#interestrate', 'negative interest', 'banking'
])
