def __init__(self, check_speed):
StreamListener.__init__(self)
thread = threading.Thread(target=worker)
thread.setDaemon(True)
thread.start()
self.queue = SofTalk_Queue
self.check_speed = check_speed
def __init__(self, output_path):
StreamListener.__init__(self)
# Initialise counter and storage variables
self._counter = 1
self._output_path = output_path
self._data = []
def __init__(self, check_speed):
StreamListener.__init__(self)
thread = threading.Thread(target=worker)
thread.setDaemon(True)
thread.start()
self.queue = SofTalk_Queue
self.check_speed = check_speed
def __init__(self, name: str, connection: str, queue_name: str):
"""
Initializes StreamListener and AMQP Producer.
:param connection: AMQP connection
"""
Publisher.__init__(self, connection, queue_name)
StreamListener.__init__(self)
self.name = name
def __init__(self, output_path, no_per_file, compression="gzip"):
StreamListener.__init__(self)
# Initialise counter and storage variables
self._counter = 1
self._output_path = output_path
self._no_per_file = no_per_file
self._data = []
def __init__(self, queue, terms, communications_channel):
# Call the constructor for the superclass.
StreamListener.__init__(self)
# Set up the pointer to the queue of tweets.
self.queue = queue
self.terms = terms
self.communications_channel = communications_channel
def __init__(self, queue, terms, communications_channel):
# Call the constructor for the superclass.
StreamListener.__init__(self)
# Set up the pointer to the queue of tweets.
self.queue = queue
self.terms = terms
self.communications_channel = communications_channel
def __init__(self, url="http://localhost:8080/workspace0", on_gui = True):
StreamListener.__init__(self);
self.url = url;
self.g = GephiJsonClient(url = self.url);
self.g.cleanAll();
self.degree_dict = {};
self.retweeted_dict = {};
self.displayed_users = deque(maxlen=1000);
self.displayed_dict = {};
self.current_tm = "";
def __init__(self, api, config, sample=15):
StreamListener.__init__(self, api)
self.client: MongoClient = MongoClient(config["host"], config["port"])
self.db: Database = self.client[config["db"]]
self.collection: Collection = self.db[config["collection"]]
self.sample: int = sample
self.index_RT: int = 1
self.index_info: int = 0
self.queue = queue.Queue()
t = Thread(target=self._storing, daemon=True).start()
def __init__(self, duik, old_duik, auth):
api = tweepy.API(auth)
StreamListener.__init__(self, api=api)
self.api = api or API()
self.old_duik = old_duik
self.duik = duik
self.matrix = None
frameinfo = getframeinfo(currentframe())
logging.debug('file %s, line %s auth %s duik, %s', frameinfo.filename, frameinfo.lineno+1, auth, duik)
def __init__(self):
StreamListener.__init__(self)
self.scores = {}
# Configure the Pi to use the BCM (Broadcom) pin names, rather than the pin positions
GPIO.setmode(GPIO.BCM)
self.red_pin = 27
self.blue_pin = 18
self.green_pin = 17
GPIO.setup(self.red_pin, GPIO.OUT)
GPIO.setup(self.green_pin, GPIO.OUT)
GPIO.setup(self.blue_pin, GPIO.OUT)
def __init__(self, people, keywords, data_dir, save_dir):
StreamListener.__init__(self)
self.TwitterClassifier = Classifier(
'./classifier/HugeTwitter-classifier.h5',
'./classifier/HugeTwitter-vocabulary.pickle')
self.myFilter = Filter(keywords=keywords, people=people)
self.data_dir = data_dir
self.save_dir = save_dir
self.tweets_in_group = {group: 0 for group in people.keys()}
def __init__(self, duik, old_duik, auth):
api = tweepy.API(auth)
StreamListener.__init__(self, api=api)
self.api = api or API()
self.old_duik = old_duik
self.duik = duik
self.matrix = None
frameinfo = getframeinfo(currentframe())
logging.info('file %s, line %s auth %s duik, %s', frameinfo.filename, frameinfo.lineno+1, auth, duik)
def __init__(self, trainer, queue):
"""
:param trainer: Instance used to create features and train a new
classifier
:param queue: Queue for retrieving classifiers
"""
StreamListener.__init__(self)
self.trainer = trainer
self.classifier = None
self.queue = queue
self.timer = Timer(REMODEL, self.retrain)
self.timer.start()
def __init__(self, queue, api=None, num_tweets=0, language=None):
StreamListener.__init__(self, api=api)
self.api = api or API()
self.tweet_counter = 0
self.language = language # should be a ISO 639-1 code
if num_tweets == 0:
self.num_tweets = DEFAULT_NUM_TWEETS
else:
self.num_tweets = num_tweets
self.start_time = time.gmtime()
self.prefix = str(int(time.time()))
self.queue = queue
def __init__(self):
register_default_functions()
StreamListener.__init__(self)
try:
self.username = settings.TWITTER_USERNAME
self.password = settings.TWITTER_PASSWORD
except AttributeError:
print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in settings.py"
self.username = raw_input('Twitter username: '******'Twitter password: ')
self.status_lock = RLock()
self.statuses = []
self.query_builder = gen_query_builder()
self.stream = None
def __init__(self):
register_default_functions()
StreamListener.__init__(self)
try:
self.username = settings.TWITTER_USERNAME
self.password = settings.TWITTER_PASSWORD
except AttributeError:
print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in settings.py"
self.username = raw_input('Twitter username: '******'Twitter password: ')
self.status_lock = RLock()
self.statuses = []
self.query_builder = gen_query_builder()
self.stream = None
def __init__(self):
register_default_functions()
StreamListener.__init__(self)
try:
self.username = settings.TWITTER_USERNAME
self.password = settings.TWITTER_PASSWORD
except AttributeError:
print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in settings.py"
self.username = raw_input('Twitter username: '******'Twitter password: ')
self.status_lock = RLock()
self.statuses = []
self.query_builder = gen_query_builder()
self.stream = None
# initializing database engine - could have done this anywhere. do it here for now
StatusSource.set_engine()
def __init__(self):
register_default_functions()
StreamListener.__init__(self)
try:
self.consumer_key = settings.CONSUMER_KEY
self.consumer_secret = settings.CONSUMER_SECRET
self.access_token = settings.ACCESS_TOKEN
self.access_token_secret = settings.ACCESS_TOKEN_SECRET
except AttributeError:
print "Check if CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, and ACCESS_TOKEN_SECRET are defined in settings.py"
self.consumer_key = raw_input('Consumer key: ')
self.consumer_secret = getpass('Consumer secret: ')
self.access_token = raw_input('Access token: ')
self.access_token_secret = getpass('Access token secret: ')
self.status_lock = RLock()
self.statuses = []
self.query_builder = gen_query_builder()
self.stream = None
def __init__(self):
register_default_functions()
StreamListener.__init__(self)
try:
self.consumer_key = settings.CONSUMER_KEY
self.consumer_secret = settings.CONSUMER_SECRET
self.access_token = settings.ACCESS_TOKEN
self.access_token_secret = settings.ACCESS_TOKEN_SECRET
except AttributeError:
print "Check if CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, and ACCESS_TOKEN_SECRET are defined in settings.py"
self.consumer_key = raw_input('Consumer key: ')
self.consumer_secret = getpass('Consumer secret: ')
self.access_token = raw_input('Access token: ')
self.access_token_secret = getpass('Access token secret: ')
self.status_lock = RLock()
self.statuses = []
self.query_builder = gen_query_builder()
self.stream = None
def work():
# Opening a CSV file to save the gathered tweets
with open(filename+".csv", 'w', encoding="UTF-8") as file:
global writer
writer = csv.writer(file)
# Add a header row to the CSV
writer.writerow(["Tweet", "Matched Keywords", "Date", "User",
"Source", "Tweet ID", "Tweet URL"])
# Initializing the twitter streap Stream
try:
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=keywords)
# Stop temporarily when hitting Twitter rate Limit
except tweepy.RateLimitError:
print("RateLimitError...waiting ~15 minutes to continue")
time.sleep(1001)
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=[keywords])
# Stop temporarily when getting a timeout or connection error
except (Timeout, ssl.SSLError, ReadTimeoutError,
ConnectionError) as exc:
print("Timeout/connection error...waiting ~15 minutes to continue")
time.sleep(1001)
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=[keywords])
# Stop temporarily when getting other errors
except tweepy.TweepError as e:
if 'Failed to send request:' in e.reason:
print("Time out error caught.")
time.sleep(1001)
streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
streamingAPI.filter(track=[keywords])
else:
print("Other error with this user...passing")
pass
def __init__(self):
StreamListener.__init__(self)
try:
username = settings.TWITTER_USERNAME
password = settings.TWITTER_PASSWORD
except AttributeError:
print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in private_settings.py"
username = raw_input("Twitter username: "******"Twitter password: ")
self.status_lock = RLock()
self.statuses = []
self.query_builder = gen_query_builder()
self.stream = Stream(
username,
password,
self, # this object implements StreamListener
timeout=600, # reconnect if no messages in 600s
retry_count=20, # try reconnecting 20 times
retry_time=10.0, # wait 10s if no HTTP 200
snooze_time=1.0,
) # wait 1s if timeout in 600s
def __init__(self, drop_publisher, predicates, firehose=None,
predicate_type=None):
StreamListener.__init__(self)
self.drop_publisher = drop_publisher
self.firehose = firehose
self.predicate_type = predicate_type
self.predicate_dict = dict(predicates)
# Whether to match follow predicates
self.follow_match = self.predicate_type == 'follow'
# Flatten the fiter predicates
self.predicate_list = utils.flatten_filter_predicates(predicates)
self.drop_queue = Queue()
# Spawn a predicate match worker
FilterPredicateMatcher(self.drop_publisher,
self.predicate_list,
self.drop_queue,
self.follow_match)
def __init__(self, drop_publisher, predicates, firehose=None,
predicate_type=None):
StreamListener.__init__(self)
self.drop_publisher = drop_publisher
self.firehose = firehose
self.predicate_type = predicate_type
self.predicate_dict = dict(predicates)
# Whether to match follow predicates
self.follow_match = self.predicate_type == 'follow'
# Flatten the fiter predicates
self.predicate_list = utils.flatten_filter_predicates(predicates)
self.drop_queue = Queue()
# Spawn a predicate match worker
FilterPredicateMatcher(self.drop_publisher,
self.predicate_list,
self.drop_queue,
self.follow_match)
rawConn.close()
class StreamListener(StreamListener):
mentionCounter = 0
def on_status(self, status): #function acts as a generator/iterator
if status.retweeted:
return
bodyText = (str(status.text)).lower()
i = 0
for i in range (0,len(coinList)):
if ((tally(bodyText, coinList[i], coinInitialList[i]) != 0) and not("https://" in bodyText) and not("rt" in bodyText)): #checks to see if comment is relevant before writing
rawConn = sqlite3.connect('twitterRawData.db')
rawCursor = rawConn.cursor()
currentTime = datetime.datetime.now()
rawCursor.execute("INSERT INTO twitterRawData VALUES (?,?,0)", (bodyText,currentTime))
rawCursor.close()
rawConn.commit()
rawConn.close()
print(bodyText)
def error(self, statusCode): #handles excessive request error
if statusCode == 420:
return False
mentionCounter = 0
stream_Listener = StreamListener()
stream = Stream(auth=api.auth, listener = stream_Listener)
stream.filter(track=trackingList)
def __init__(self, api: API, me: User):
StreamListener.__init__(self, api)
self.me = me
def __init__(self, required_size):
StreamListener.__init__(self)
self.required_size = required_size
self.sample = []
# access_token = keys.access_token
# access_token_secret = keys.access_token_secret
# auth1 = tweepy.OAuthHandler(consumer_key, consumer_secret)
# auth1.set_access_token(access_token, access_token_secret)
#auth
# LOCATIONS are the longitude, latitude coordinate corners for a box that restricts the
# geographic area from which you will stream tweets. The first two define the southwest
# corner of the box and the second two define the northeast corner of the box.
LOCATIONS = [
-124.7771694,
24.520833,
-66.947028,
49.384472, # Contiguous US
-164.639405,
58.806859,
-144.152365,
71.76871, # Alaska
-160.161542,
18.776344,
-154.641396,
22.878623
] # Hawaii
stream_listener = StreamListener(api=tweepy.API(wait_on_rate_limit=True))
stream = tweepy.Stream(auth=auth,
listener=stream_listener,
tweet_mode='extended')
stream.filter(track=[
'Rabobank', '#Rabo', '#interestrate', 'negative interest', 'banking'
])
def __init__(self, query):
"""Initialize twitter stream"""
StreamListener.__init__(self)
self._mongo = pymongo.MongoClient().minerva[query]
self._filters = []
def __init__(self, api, sample=15, test=False):
StreamListener.__init__(self, api)
self.index_RT: int = 1
self.sample: int = sample
self.test: bool = test
def __init__(self, mongo: Mongo, collection):
StreamListener.__init__(self)
self.counter = 0
self.mongo = mongo
self.collection = collection
def on_status(self, status):
if hasattr(status, 'retweeted_status'):
return
print type(status)
print(status.text)
def on_error(self, status_code):
if status_code == 420:
return False
#Since it wasn't explicitly mentioned, keeping the API keys as it is
#API keys
ckey = '*****'
csec = '*****'
atok = '*****'
asec = '*****'
#Auth
auth = OAuthHandler(ckey, csec)
auth.set_access_token(atok, asec)
api = API(auth)
#Query to search for tweets using emoticons values
#For example, \U0001F602 refers to 'happy'
#1F602 = happy, 1F62D = sad, 1F621= angry, 2764 = love, 1F61C = playful, 1F631 = confused
query = [u'\U0001F602']
#stream prints out the tweets matching the query to the terminal
stream = Stream(auth=api.auth, listener=StreamListener())
stream.filter(track=query, languages=["en"], stall_warnings=True, async=True)
user_id = str(status.user.id)
sentiment = sentiment_score(text)
positivity = round(sentiment['pos'], 4)
negativity = round(sentiment['neg'], 4)
compound = round(sentiment['compound'], 4)
polarity = round((TextBlob(text)).sentiment.polarity, 4)
city_id = get_city_id(centroid_lat, centroid_long)
get_or_create_tweet(user_id, loc, id_str, created, centroid_lat, centroid_long, text, city_id)
def on_exception(self, exception):
print(exception)
return
def on_error(self, status_code):
if status_code == 420:
return False
flu = ['flu', 'influenza', 'cough', 'fever', 'sore throat', 'headache',
'phlegm', 'runny nose', 'stuffy nose', 'Robitussin',
'dayquil', 'nyquil', 'tamiflu', 'vomit', 'body ache', 'mucinex',
'pneumonia', 'vomit', 'bodyache', 'medicine']
stream_listener = StreamListener()
stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
while True:
try:
stream.filter(track=flu)
except (ProtocolError, AttributeError, ReadTimeoutError):
continue
test = stream.filter(track=flu)
def __init__(self, time_limit):
StreamListener.__init__(self)
self.start_time = time.time()
self.limit = time_limit
def __init__(self, loop_number, tweet_collection):
""" """
StreamListener.__init__(self)
self.loop_number = loop_number
self.tweet_collection = tweet_collection
self.processedTweet = ProcessingTweet(self.tweet_collection)
def __init__(self, loop_number, tweet_collection):
""" """
StreamListener.__init__(self)
self.loop_number = loop_number
self.tweet_collection = tweet_collection
self.processedTweet = ProcessingTweet(self.tweet_collection)
def __init__(self, required_size):
StreamListener.__init__(self)
self.required_size = required_size
self.sample = []
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)
# Create a wrapper for the API provided by Twitter
api = tweepy.API(auth,wait_on_rate_limit=True)
# Setting up the keywords, hashtag or mentions we want to listen
keywords = ["#music", "music", "music review", "song", "album"]
# Set the name for CSV file where the tweets will be saved
filename = "musicreview"
#get myStream to sync up with Streamlistener and prepare a filter
myStream = tweepy.Stream(auth = api.auth, listener=StreamListener())
# We need to implement StreamListener to use Tweepy to listen to Twitter
class StreamListener(tweepy.StreamListener):
tweet_counter = 0 # Static variable and counter of how many tweet downloaded
def on_status(self, status):
try:
# saves the tweet object
tweet_object = status
# Checks if its a extended tweet (>140 characters)
def on_data(self, raw_data):
self.raw_data = raw_data
return StreamListener.on_data(self, raw_data)
def run(self):
"""
Initializing a Stream & Use Search Engine with User's Hashtags.
Args:
None.
Parameters:
self.app.listener (StreamListener): Initializing & stores StreamListener object.
self.app.stream (Stream): Initializing & Store Stream object with auth & listener args.
tweets_list[] (list): Temporary list in order to store specific hashtag's tweets
self.tweet_matrix (list): list of lists - Storing lists of tweets for each hashtag.
self.pb (QProgressBar): Progess bar which present the current status of the search.
self.tag_list (dict): Hashtags list itself for searching tweets.
temp_value (float): Temporary value for the continuation of the progress bar.
start (float): current time in seconds.
clock (str): current time.
self.signal (pyqtBoundSignal):Emiting the tweets list of lists back to data_and_analasys_to_excel method.
Returns:
None
"""
# Handles Twitter authetification and the connection to Twitter Streaming API
self.app.listener = StreamListener()
self.app.stream = Stream(self.app.auth, self.app.listener)
# Capture the Tweets in a Temporary List
tweets_list = []
try:
# Creating the Progress Bar and present it.
self.pb = self.__create_progress_bar(self)
self.pb.show()
# Temporary value for the continuation of the progress bar
temp_value = 0
for hashtag, search_item in zip(self.tag_list, range(1, len(self.tag_list) + 1)):
QApplication.processEvents()
# Starting time
start = time.time()
clock = datetime.now().strftime("%H:%M:%S")
cursor = Cursor(self.twitter_client.search, q=hashtag, result_type='mixed', tweet_mode='extended',
include_entities=True, lang="en").items(self.num_of_tweets)
# Use Cursor to search for hashtag and copy it into a tweets_list
for tweet in cursor:
QApplication.processEvents()
# Capture a list of tweets
tweets_list.append(tweet)
# Progress Bar Continuation's Configuration
time.sleep(0.05)
self.pb.setValue(temp_value + 100/len(self.tag_list))
temp_value += 100/len(self.tag_list)
print("Search #{} took {} from {}".format(search_item, time.time() - start, clock))
# Copy to the List of lists which stores all the tweets for each hashtags & clean the temp list
self.tweet_matrix.append(tweets_list.copy())
tweets_list.clear()
# Hiding the Progress Bar
self.pb.close()
# Disconnect from the stream
self.app.stream.disconnect()
# Terminate and delete the search thread.
self.app.search_thread.terminate()
del self.app.search_thread
# Emiting the tweets list of lists back to data_and_analasys_to_excel method.
self.signal.emit(self.tweet_matrix)
except RateLimitError as limit:
print('RateLimit Error: {0}'.format(limit))
except TweepError as error:
print('Tweepy Error: {0}'.format(error))
except Exception as e:
print('Search Error: {0}'.format(e))