Ejemplo n.º 1
0
 def __init__(self, check_speed):
     StreamListener.__init__(self)
     thread = threading.Thread(target=worker)
     thread.setDaemon(True)
     thread.start()
     self.queue = SofTalk_Queue
     self.check_speed = check_speed
    def __init__(self, output_path):
        StreamListener.__init__(self)

        # Initialise counter and storage variables
        self._counter = 1
        self._output_path = output_path
        self._data = []
Ejemplo n.º 3
0
 def __init__(self, check_speed):
     StreamListener.__init__(self)
     thread = threading.Thread(target=worker)
     thread.setDaemon(True)
     thread.start()
     self.queue = SofTalk_Queue
     self.check_speed = check_speed
Ejemplo n.º 4
0
 def __init__(self, name: str, connection: str, queue_name: str):
     """
     Initializes StreamListener and AMQP Producer.
     :param connection: AMQP connection
     """
     Publisher.__init__(self, connection, queue_name)
     StreamListener.__init__(self)
     self.name = name
Ejemplo n.º 5
0
    def __init__(self, output_path, no_per_file, compression="gzip"):
        StreamListener.__init__(self)

        # Initialise counter and storage variables
        self._counter = 1
        self._output_path = output_path
        self._no_per_file = no_per_file
        self._data = []
Ejemplo n.º 6
0
    def __init__(self, queue, terms, communications_channel):
        # Call the constructor for the superclass.
        StreamListener.__init__(self)

        # Set up the pointer to the queue of tweets.
        self.queue = queue
        self.terms = terms
        self.communications_channel = communications_channel
    def __init__(self, queue, terms, communications_channel):
        # Call the constructor for the superclass.
        StreamListener.__init__(self)

        # Set up the pointer to the queue of tweets.
        self.queue = queue
        self.terms = terms
        self.communications_channel = communications_channel
 def __init__(self, url="http://localhost:8080/workspace0", on_gui = True):
     StreamListener.__init__(self);
     self.url = url;
     self.g = GephiJsonClient(url = self.url);
     self.g.cleanAll();  
     self.degree_dict = {};
     self.retweeted_dict = {};
     self.displayed_users = deque(maxlen=1000);
     self.displayed_dict = {};
     self.current_tm = "";
Ejemplo n.º 9
0
 def __init__(self, api, config, sample=15):
     StreamListener.__init__(self, api)
     self.client: MongoClient = MongoClient(config["host"], config["port"])
     self.db: Database = self.client[config["db"]]
     self.collection: Collection = self.db[config["collection"]]
     self.sample: int = sample
     self.index_RT: int = 1
     self.index_info: int = 0
     self.queue = queue.Queue()
     t = Thread(target=self._storing, daemon=True).start()
    def __init__(self, duik, old_duik, auth):
        api = tweepy.API(auth)
        StreamListener.__init__(self, api=api)
        self.api = api or API()

        self.old_duik = old_duik
        self.duik = duik
        self.matrix = None

        frameinfo = getframeinfo(currentframe())
        logging.debug('file %s, line %s auth %s duik, %s', frameinfo.filename, frameinfo.lineno+1, auth, duik)
Ejemplo n.º 11
0
 def __init__(self):
     StreamListener.__init__(self)
     self.scores = {}
     # Configure the Pi to use the BCM (Broadcom) pin names, rather than the pin positions
     GPIO.setmode(GPIO.BCM)
     self.red_pin = 27
     self.blue_pin = 18
     self.green_pin = 17
     GPIO.setup(self.red_pin, GPIO.OUT)
     GPIO.setup(self.green_pin, GPIO.OUT)
     GPIO.setup(self.blue_pin, GPIO.OUT)
Ejemplo n.º 12
0
    def __init__(self, people, keywords, data_dir, save_dir):
        StreamListener.__init__(self)
        self.TwitterClassifier = Classifier(
            './classifier/HugeTwitter-classifier.h5',
            './classifier/HugeTwitter-vocabulary.pickle')
        self.myFilter = Filter(keywords=keywords, people=people)

        self.data_dir = data_dir
        self.save_dir = save_dir

        self.tweets_in_group = {group: 0 for group in people.keys()}
    def __init__(self, duik, old_duik, auth):
        api = tweepy.API(auth)
        StreamListener.__init__(self, api=api)
        self.api = api or API()

        self.old_duik = old_duik
        self.duik = duik
        self.matrix = None

        frameinfo = getframeinfo(currentframe())
        logging.info('file %s, line %s auth %s duik, %s', frameinfo.filename, frameinfo.lineno+1, auth, duik)
    def __init__(self, trainer, queue):
        """
        :param trainer: Instance used to create features and train a new
        classifier
        :param queue: Queue for retrieving classifiers
        """
        StreamListener.__init__(self)
        self.trainer = trainer
        self.classifier = None
        self.queue = queue

        self.timer = Timer(REMODEL, self.retrain)
        self.timer.start()
    def __init__(self, queue, api=None, num_tweets=0, language=None):
        StreamListener.__init__(self, api=api)
        self.api = api or API()
        self.tweet_counter = 0
        self.language = language # should be a ISO 639-1 code

        if num_tweets == 0:
            self.num_tweets = DEFAULT_NUM_TWEETS
        else:
            self.num_tweets = num_tweets

        self.start_time = time.gmtime()
        self.prefix = str(int(time.time()))
        self.queue = queue
Ejemplo n.º 16
0
 def __init__(self):
     register_default_functions()
     StreamListener.__init__(self)
     try:
         self.username = settings.TWITTER_USERNAME
         self.password = settings.TWITTER_PASSWORD
     except AttributeError:
         print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in settings.py"
         self.username = raw_input('Twitter username: '******'Twitter password: ')
     self.status_lock = RLock()
     self.statuses = []
     self.query_builder = gen_query_builder()
     self.stream = None
Ejemplo n.º 17
0
 def __init__(self):
     register_default_functions()
     StreamListener.__init__(self)
     try:
         self.username = settings.TWITTER_USERNAME
         self.password = settings.TWITTER_PASSWORD
     except AttributeError:
         print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in settings.py"
         self.username = raw_input('Twitter username: '******'Twitter password: ')
     self.status_lock = RLock()
     self.statuses = []
     self.query_builder = gen_query_builder()
     self.stream = None
Ejemplo n.º 18
0
 def __init__(self):
     register_default_functions()
     StreamListener.__init__(self)
     try:
         self.username = settings.TWITTER_USERNAME
         self.password = settings.TWITTER_PASSWORD
     except AttributeError:
         print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in settings.py"
         self.username = raw_input('Twitter username: '******'Twitter password: ')
     self.status_lock = RLock()
     self.statuses = []
     self.query_builder = gen_query_builder()
     self.stream = None
     # initializing database engine - could have done this anywhere. do it here for now
     StatusSource.set_engine()
Ejemplo n.º 19
0
 def __init__(self):
     register_default_functions()
     StreamListener.__init__(self)
     try:
         self.consumer_key = settings.CONSUMER_KEY
         self.consumer_secret = settings.CONSUMER_SECRET
         self.access_token = settings.ACCESS_TOKEN
         self.access_token_secret = settings.ACCESS_TOKEN_SECRET
     except AttributeError:
         print "Check if CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, and ACCESS_TOKEN_SECRET are defined in settings.py"
         self.consumer_key = raw_input('Consumer key: ')
         self.consumer_secret = getpass('Consumer secret: ')
         self.access_token = raw_input('Access token: ')
         self.access_token_secret = getpass('Access token secret: ')
     self.status_lock = RLock()
     self.statuses = []
     self.query_builder = gen_query_builder()
     self.stream = None
Ejemplo n.º 20
0
    def __init__(self):
        register_default_functions()
        StreamListener.__init__(self)
        try:
			self.consumer_key = settings.CONSUMER_KEY
			self.consumer_secret = settings.CONSUMER_SECRET
			self.access_token = settings.ACCESS_TOKEN
			self.access_token_secret = settings.ACCESS_TOKEN_SECRET
        except AttributeError:
            print "Check if CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, and ACCESS_TOKEN_SECRET are defined in settings.py"
            self.consumer_key = raw_input('Consumer key: ')
            self.consumer_secret = getpass('Consumer secret: ')
            self.access_token = raw_input('Access token: ')
            self.access_token_secret = getpass('Access token secret: ')
        self.status_lock = RLock()
        self.statuses = []
        self.query_builder = gen_query_builder()
        self.stream = None
Ejemplo n.º 21
0
def work():

    # Opening a CSV file to save the gathered tweets
    with open(filename+".csv", 'w', encoding="UTF-8") as file:
        global writer
        writer = csv.writer(file)

        # Add a header row to the CSV
        writer.writerow(["Tweet", "Matched Keywords", "Date", "User",
                        "Source", "Tweet ID", "Tweet URL"])

        # Initializing the twitter streap Stream
        try:
            streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
            streamingAPI.filter(track=keywords)

        # Stop temporarily when hitting Twitter rate Limit
        except tweepy.RateLimitError:
            print("RateLimitError...waiting ~15 minutes to continue")
            time.sleep(1001)
            streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
            streamingAPI.filter(track=[keywords])

        # Stop temporarily when getting a timeout or connection error
        except (Timeout, ssl.SSLError, ReadTimeoutError,
                ConnectionError) as exc:
            print("Timeout/connection error...waiting ~15 minutes to continue")
            time.sleep(1001)
            streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
            streamingAPI.filter(track=[keywords])

        # Stop temporarily when getting other errors
        except tweepy.TweepError as e:
            if 'Failed to send request:' in e.reason:
                print("Time out error caught.")
                time.sleep(1001)
                streamingAPI = tweepy.streaming.Stream(auth, StreamListener())
                streamingAPI.filter(track=[keywords])
            else:
                print("Other error with this user...passing")
                pass
Ejemplo n.º 22
0
 def __init__(self):
     StreamListener.__init__(self)
     try:
         username = settings.TWITTER_USERNAME
         password = settings.TWITTER_PASSWORD
     except AttributeError:
         print "TWITTER_USERNAME and TWITTER_PASSWORD not defined in private_settings.py"
         username = raw_input("Twitter username: "******"Twitter password: ")
     self.status_lock = RLock()
     self.statuses = []
     self.query_builder = gen_query_builder()
     self.stream = Stream(
         username,
         password,
         self,  # this object implements StreamListener
         timeout=600,  # reconnect if no messages in 600s
         retry_count=20,  # try reconnecting 20 times
         retry_time=10.0,  # wait 10s if no HTTP 200
         snooze_time=1.0,
     )  # wait 1s if timeout in 600s
Ejemplo n.º 23
0
    def __init__(self, drop_publisher, predicates, firehose=None,
                 predicate_type=None):
        StreamListener.__init__(self)

        self.drop_publisher = drop_publisher

        self.firehose = firehose
        self.predicate_type = predicate_type
        self.predicate_dict = dict(predicates)
        
        # Whether to match follow predicates
        self.follow_match = self.predicate_type == 'follow'

        # Flatten the fiter predicates
        self.predicate_list = utils.flatten_filter_predicates(predicates)

        self.drop_queue = Queue()
        # Spawn a predicate match worker
        FilterPredicateMatcher(self.drop_publisher,
                               self.predicate_list,
                               self.drop_queue,
                               self.follow_match)
Ejemplo n.º 24
0
    def __init__(self, drop_publisher, predicates, firehose=None,
                 predicate_type=None):
        StreamListener.__init__(self)

        self.drop_publisher = drop_publisher

        self.firehose = firehose
        self.predicate_type = predicate_type
        self.predicate_dict = dict(predicates)
        
        # Whether to match follow predicates
        self.follow_match = self.predicate_type == 'follow'

        # Flatten the fiter predicates
        self.predicate_list = utils.flatten_filter_predicates(predicates)

        self.drop_queue = Queue()
        # Spawn a predicate match worker
        FilterPredicateMatcher(self.drop_publisher,
                               self.predicate_list,
                               self.drop_queue,
                               self.follow_match)
Ejemplo n.º 25
0
rawConn.close()
class StreamListener(StreamListener):
	mentionCounter = 0
	def on_status(self, status): #function acts as a generator/iterator
		if status.retweeted:
			return
		bodyText = (str(status.text)).lower()	
		i = 0
		for i in range (0,len(coinList)):
			if ((tally(bodyText, coinList[i], coinInitialList[i]) != 0) and not("https://" in bodyText) and not("rt" in bodyText)): #checks to see if comment is relevant before writing
				rawConn = sqlite3.connect('twitterRawData.db')
				rawCursor = rawConn.cursor()
				
				currentTime = datetime.datetime.now()
				rawCursor.execute("INSERT INTO twitterRawData VALUES (?,?,0)", (bodyText,currentTime))
				rawCursor.close()
				rawConn.commit()
				rawConn.close()
				print(bodyText)
				

	def error(self, statusCode): #handles excessive request error
		if statusCode == 420:
			return False
mentionCounter = 0
stream_Listener = StreamListener()
stream = Stream(auth=api.auth, listener = stream_Listener)
stream.filter(track=trackingList)


Ejemplo n.º 26
0
 def __init__(self, api: API, me: User):
     StreamListener.__init__(self, api)
     self.me = me
Ejemplo n.º 27
0
 def __init__(self, required_size):
     StreamListener.__init__(self)
     self.required_size = required_size
     self.sample = []
Ejemplo n.º 28
0
    # access_token = keys.access_token
    # access_token_secret = keys.access_token_secret

    # auth1 = tweepy.OAuthHandler(consumer_key, consumer_secret)
    # auth1.set_access_token(access_token, access_token_secret)
    #auth
    # LOCATIONS are the longitude, latitude coordinate corners for a box that restricts the
    # geographic area from which you will stream tweets. The first two define the southwest
    # corner of the box and the second two define the northeast corner of the box.
    LOCATIONS = [
        -124.7771694,
        24.520833,
        -66.947028,
        49.384472,  # Contiguous US
        -164.639405,
        58.806859,
        -144.152365,
        71.76871,  # Alaska
        -160.161542,
        18.776344,
        -154.641396,
        22.878623
    ]  # Hawaii

    stream_listener = StreamListener(api=tweepy.API(wait_on_rate_limit=True))
    stream = tweepy.Stream(auth=auth,
                           listener=stream_listener,
                           tweet_mode='extended')
    stream.filter(track=[
        'Rabobank', '#Rabo', '#interestrate', 'negative interest', 'banking'
    ])
Ejemplo n.º 29
0
 def __init__(self, query):
     """Initialize twitter stream"""
     StreamListener.__init__(self)
     self._mongo = pymongo.MongoClient().minerva[query]
     self._filters = []
Ejemplo n.º 30
0
 def __init__(self, api, sample=15, test=False):
     StreamListener.__init__(self, api)
     self.index_RT: int = 1
     self.sample: int = sample
     self.test: bool = test
Ejemplo n.º 31
0
        def __init__(self, mongo: Mongo, collection):
            StreamListener.__init__(self)

            self.counter = 0
            self.mongo = mongo
            self.collection = collection
Ejemplo n.º 32
0
    def on_status(self, status):
        if hasattr(status, 'retweeted_status'):
            return
        print type(status)
        print(status.text)

    def on_error(self, status_code):
        if status_code == 420:
            return False


#Since it wasn't explicitly mentioned, keeping the API keys as it is
#API keys
ckey = '*****'
csec = '*****'
atok = '*****'
asec = '*****'

#Auth
auth = OAuthHandler(ckey, csec)
auth.set_access_token(atok, asec)
api = API(auth)

#Query to search for tweets using emoticons values
#For example, \U0001F602 refers to 'happy'
#1F602 = happy, 1F62D = sad, 1F621= angry, 2764 = love, 1F61C = playful, 1F631 = confused
query = [u'\U0001F602']
#stream prints out the tweets matching the query to the terminal
stream = Stream(auth=api.auth, listener=StreamListener())
stream.filter(track=query, languages=["en"], stall_warnings=True, async=True)
Ejemplo n.º 33
0
            user_id = str(status.user.id)
            sentiment = sentiment_score(text)
            positivity = round(sentiment['pos'], 4)
            negativity = round(sentiment['neg'], 4)
            compound = round(sentiment['compound'], 4)
            polarity = round((TextBlob(text)).sentiment.polarity, 4)
            city_id = get_city_id(centroid_lat, centroid_long)
            get_or_create_tweet(user_id, loc, id_str, created, centroid_lat, centroid_long, text, city_id)

    def on_exception(self, exception):
           print(exception)
           return

    def on_error(self, status_code):
        if status_code == 420:
            return False

flu = ['flu', 'influenza', 'cough', 'fever', 'sore throat', 'headache',
        'phlegm', 'runny nose', 'stuffy nose', 'Robitussin',
        'dayquil', 'nyquil', 'tamiflu', 'vomit', 'body ache', 'mucinex',
        'pneumonia', 'vomit', 'bodyache', 'medicine']

stream_listener = StreamListener()
stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
while True:
    try:
        stream.filter(track=flu)
    except (ProtocolError, AttributeError, ReadTimeoutError):
        continue
test = stream.filter(track=flu)
Ejemplo n.º 34
0
    def __init__(self, time_limit):
        StreamListener.__init__(self)

        self.start_time = time.time()
        self.limit = time_limit
Ejemplo n.º 35
0
 def __init__(self, loop_number, tweet_collection):
     """ """
     StreamListener.__init__(self)
     self.loop_number = loop_number
     self.tweet_collection = tweet_collection
     self.processedTweet = ProcessingTweet(self.tweet_collection)
Ejemplo n.º 36
0
 def __init__(self, loop_number, tweet_collection):
     """ """
     StreamListener.__init__(self)
     self.loop_number = loop_number
     self.tweet_collection = tweet_collection
     self.processedTweet = ProcessingTweet(self.tweet_collection)
Ejemplo n.º 37
0
 def __init__(self, required_size):
     StreamListener.__init__(self)
     self.required_size = required_size
     self.sample = []
Ejemplo n.º 38
0
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_key, access_secret)

# Create a wrapper for the API provided by Twitter
api = tweepy.API(auth,wait_on_rate_limit=True)

# Setting up the keywords, hashtag or mentions we want to listen
keywords = ["#music", "music", "music review", "song", "album"] 

# Set the name for CSV file  where the tweets will be saved
filename = "musicreview"



#get myStream to sync up with Streamlistener and prepare a filter
myStream = tweepy.Stream(auth = api.auth, listener=StreamListener())


# We need to implement StreamListener to use Tweepy to listen to Twitter
class StreamListener(tweepy.StreamListener):

    tweet_counter = 0 # Static variable and counter of how many tweet downloaded

    
    def on_status(self, status):

        try:
            # saves the tweet object
            tweet_object = status

            # Checks if its a extended tweet (>140 characters)
Ejemplo n.º 39
0
 def on_data(self, raw_data):
     self.raw_data = raw_data
     return StreamListener.on_data(self, raw_data)
Ejemplo n.º 40
0
    def run(self):

        """
        Initializing a Stream & Use Search Engine with User's Hashtags.

        Args:
            None.

        Parameters:
            self.app.listener (StreamListener): Initializing & stores StreamListener object.
            self.app.stream (Stream): Initializing & Store Stream object with auth & listener args.
            tweets_list[] (list): Temporary list in order to store specific hashtag's tweets
            self.tweet_matrix (list): list of lists - Storing lists of tweets for each hashtag.
            self.pb (QProgressBar): Progess bar which present the current status of the search.
            self.tag_list (dict): Hashtags list itself for searching tweets.
            temp_value (float): Temporary value for the continuation of the progress bar.
            start (float): current time in seconds.
            clock (str): current time.
            self.signal (pyqtBoundSignal):Emiting the tweets list of lists back to data_and_analasys_to_excel method.

        Returns:
            None
        """

        # Handles Twitter authetification and the connection to Twitter Streaming API
        self.app.listener = StreamListener()
        self.app.stream = Stream(self.app.auth, self.app.listener)

        # Capture the Tweets in a Temporary List
        tweets_list = []

        try:
                # Creating the Progress Bar and present it.
                self.pb = self.__create_progress_bar(self)
                self.pb.show()

                # Temporary value for the continuation of the progress bar
                temp_value = 0

                for hashtag, search_item in zip(self.tag_list, range(1, len(self.tag_list) + 1)):

                    QApplication.processEvents()

                    # Starting time
                    start = time.time()
                    clock = datetime.now().strftime("%H:%M:%S")

                    cursor = Cursor(self.twitter_client.search, q=hashtag, result_type='mixed', tweet_mode='extended',
                                    include_entities=True, lang="en").items(self.num_of_tweets)

                    # Use Cursor to search for hashtag and copy it into a tweets_list
                    for tweet in cursor:

                        QApplication.processEvents()
                        # Capture a list of tweets
                        tweets_list.append(tweet)

                    # Progress Bar Continuation's Configuration
                    time.sleep(0.05)
                    self.pb.setValue(temp_value + 100/len(self.tag_list))
                    temp_value += 100/len(self.tag_list)

                    print("Search #{} took {} from {}".format(search_item, time.time() - start, clock))

                    # Copy to the List of lists which stores all the tweets for each hashtags & clean the temp list
                    self.tweet_matrix.append(tweets_list.copy())
                    tweets_list.clear()

                # Hiding the Progress Bar
                self.pb.close()

                # Disconnect from the stream
                self.app.stream.disconnect()

                # Terminate and delete the search thread.
                self.app.search_thread.terminate()
                del self.app.search_thread

                # Emiting the tweets list of lists back to data_and_analasys_to_excel method.
                self.signal.emit(self.tweet_matrix)

        except RateLimitError as limit:
            print('RateLimit Error: {0}'.format(limit))
        except TweepError as error:
            print('Tweepy Error: {0}'.format(error))
        except Exception as e:
            print('Search Error: {0}'.format(e))