Beispiel #1
0
 def monitor(self, parent, connection):
     ntpclient = ntplib.NTPClient()
     silentChannelsFile = open("./modules/static/NOTWEETCHANNELS", 'r')
     accountsToMonitorFile = open("./modules/static/TWITTERACCOUNTS", 'r')
     self.silentChannels = silentChannelsFile.read().splitlines()
     self.accounts = accountsToMonitorFile.read().splitlines()
     silentChannelsFile.close()
     accountsToMonitorFile.close()
     cm = ChannelMessager()
     cm.setup(self.silentChannels, self.accounts, connection, parent)
     keyFile = open("modules/pubmsg/TwitterKeys", 'r')
     keyList = keyFile.read().splitlines()
     keyFile.close()
     for entry in keyList:
         if entry.startswith('#'):
             keyList.remove(entry)
     consumerKey = keyList[0]
     consumerSecret = keyList[1]
     accessKey = keyList[2]
     accessSecret = keyList[3]
     auth = OAuthHandler(consumerKey, consumerSecret)
     auth.set_access_token(accessKey, accessSecret)
     while True:
         stream = Stream(auth, cm, timeout=60)
         try:
             stream.userstream(_with='followings')
         except:
             print "Error, restarting stream"
Beispiel #2
0
def start_stream(auth, l):
    while True:
        try:
            stream = Stream(auth, l)
            stream.sample()
        except:
            continue
Beispiel #3
0
    def run(self):
        l = StdOutListener(self)
        auth = OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token, access_token_secret)

        stream = Stream(auth, l)
        stream.filter(track=self.tags, languages=['en'])
Beispiel #4
0
def main():
#    dirname = os.path.dirname(inspect.getfile(inspect.currentframe()))
#    basename = os.path.basename(inspect.getfile(inspect.currentframe()))
    dirname = os.path.dirname(os.path.realpath(__file__))
    basename = os.path.basename(os.path.realpath(__file__))
    name_noextension = os.path.splitext(basename)[0]
    """Start log."""
    configinput = __import__("config" + name_noextension)
    outputDir = os.path.join(dirname, configinput.directory)
    if not os.path.exists(outputDir):
        os.makedirs(outputDir)
    logfilename = os.path.join(outputDir,basename) + ".log"
    logging.basicConfig(filename=logfilename,level=logging.DEBUG, format='%(asctime)s %(message)s')
    logging.info('Started')
    save_pid()


    """Execute the twitter api."""
    try:
        auth = OAuthHandler(configinput.consumer_key, configinput.consumer_secret)
        auth.set_access_token(configinput.access_token, configinput.access_secret)
        twitter_stream = Stream(auth, MyListener(os.path.join(dirname, configinput.directory), os.path.join(dirname, configinput.to_dir), basename))   # el segundo argumento es el nombre del archibvo json
        twitter_stream.filter(track=configinput.keyword_list_filter)
    except BaseException as e:
        logging.error('Failed to execute twitter api: ' + str(e))    
    
    logging.info('Finished')
Beispiel #5
0
class Streamer( object ):

	def __init__( self, queue, terms=[], consumer=None, consumer_secret=None,
			token=None, secret=None):

		if consumer == None or consumer_secret == None or token == None or secret == None:
			config = configparser.ConfigParser()
			config.readfp( open( os.path.expanduser( "~/.slackTwitter" ) ) )

			consumer = config.get( "twitter", "consumer" )
			consumer_secret = config.get( "twitter", "consumer_secret" )
			token = config.get( "twitter", "token" )
			secret = config.get( "twitter", "secret" )

		auth = OAuthHandler( consumer, consumer_secret )
		auth.set_access_token( token, secret )

		listener = StreamerListener()
		self.stream = TweepyStream( auth=auth, listener=listener )

		self._queue = queue
		self._terms = terms
	
	def start( self ):
		self.stream.filter( track=self._terms )
    def handle(self, *args, **options):
        
        politicians = Politician.objects.all();

        politician_keywords = []
        for politician in politicians:
            politician_keywords.append(politician.first_name + " " + politician.last_name)
            if politician.twitter_url:
                indexSlash = politician.twitter_url.rfind("/")
                indexQuestionMark = politician.twitter_url.rfind("?")
                if indexQuestionMark != -1:
                    twitter = politician.twitter_url[indexSlash+1:indexQuestionMark]
                else:
                    twitter = politician.twitter_url[indexSlash+1:]
                politician_keywords.append(twitter)
        
        # create instance of the tweepy tweet stream listener
        listener = TweetStreamListener()

        # set twitter keys/tokens
        auth = OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token, access_token_secret)

        # create instance of the tweepy stream
        stream = Stream(auth, listener)


        # search twitter for "congress" keyword
        stream.filter(track=politician_keywords)
        
Beispiel #7
0
def stream_twitter(battle_id):
    #Avoiding circular import
    from battle.models import Battle

    battle = Battle.objects.get(id=battle_id)
    if battle.end_time < timezone.now():
        return

    battle.battlehashtags_set.update(typos=0, words=0)
    battle_hashtags = battle.battlehashtags_set.all().prefetch_related('hashtag')
    if battle_hashtags.count() == 0:
        return

    hashtag_values = [x.hashtag.value for x in battle_hashtags]

    listener = TwitterStreamListener(battle_hashtags)
    auth = OAuthHandler(
        settings.TWITTER_CONSUMER_KEY,
        settings.TWITTER_CONSUMER_SECRET
    )

    auth.set_access_token(
        settings.TWITTER_ACCESS_TOKEN,
        settings.TWITTER_ACCESS_TOKEN_SECRET
    )

    stream = Stream(auth, listener)

    delay = battle.end_time - timezone.now()
    Timer(delay.total_seconds(), stream.disconnect).start()

    stream.filter(track=hashtag_values, languages=['en'])
def stream(buff, terms):
    l = StdOutListener(buff)
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    stream = Stream(auth, l)
    stream.filter(track=[terms])
Beispiel #9
0
def main():
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    backend = FileBackend("./test-db")
    stream = Stream(auth, l)
    stream.filter(track=['トレクル'])
def TwitterStream(kwords, lim, lang=['en'], loca=[-180, -90, 180, 90]):
    # print kwords, lang, lim, loca
    global limit
    if type(lim) != tuple:
        l = StdOutListener()
        limit = int(lim)
    else:
        day = int(lim[0])
        hour = int(lim[1])
        minute = int(lim[2])
        second = int(lim[3])
        l = StdOutListener_time()
        print time.time()
        limit = time.time() + 86400 * day + 3600 * \
            hour + 60 * minute + 1 * second
        print limit

    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    global results
    results = list()
    stream = Stream(auth, l)

    # print kwords, lang
    stream.filter(track=kwords, languages=['en'])
    # def filter(self, follow=None, track=None, async=False, locations=None,
    #            stall_warnings=False, languages=None, encoding='utf8'):
    return results
Beispiel #11
0
 def run(self,user,message):
     if utilities.getCommand() == "autotweet":
         streamListener = ReplyToTweet()
         streamListener.setAPI(self.twitterApi)
         streamListener.setUser(self.account_user_id)
         twitterStream = Stream(self.auth, streamListener)
         twitterStream.userstream(_with='user')
Beispiel #12
0
def run_twitter_query():
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, l)
    #names = list(np.array(get_companies())[:,1])
    #print names[num1:num2]
    d = hand_made_list()
    search_list = []
    for key, value in d.items():
        if key == 'SPY':
            search_list.append(value[0]) # append the full anme of the symbol
            search_list.append('#SP500') # Don't append #SPY because it's not helpful
            search_list.append('$SP500')
        elif key == 'F':
            # search_list.append(value[0]) # append the full name of the symbol
            search_list.append('Ford') # append the name of the symbol
        elif key == 'GE':
            search_list.append('General Electric') # append the full anme of the symbol
        elif key == 'S':
            search_list.append('Sprint') # append the full anme of the symbol
        elif key == 'T':
            search_list.append('AT&T') # append the full anme of the symbol
        elif key == 'MU':
            search_list.append('Micron Tech') # append the full anme of the symbol
        elif key == 'TRI':
            search_list.append('Thomson Reuters') # append the full anme of the symbol
        else:
            for cell in value:
                search_list.append(cell)

    stream.filter(track=search_list)
Beispiel #13
0
class TwitterPlayer(player.Player):
    def __init__(self, model, code, access_token, access_token_secret, opponent):
        player.Player.__init__(self, model, code)
        self._opponent = opponent
        self._last_id = None

        self._auth = OAuthHandler(auth.consumer_key, auth.consumer_secret)
        self._auth.set_access_token(access_token, access_token_secret)
        self._api = API(self._auth)
        self._listener = TwitterListener(self, self._api)
        self._stream = Stream(self._auth, self._listener)

    @property
    def username(self):
        return self._auth.get_username()

    def allow(self):
        print 'This is the opponent\'s turn...'
        self._stream.userstream()

    def update(self, event):
        if event.player == self.code:
            return
        message = '@%s %s' % (self._opponent, self._model.events[-1][1])
        self.tweet(message)

    def tweet(self, message):
        if self._last_id is None:
            self._api.update_status(message)
        else:
            self._api.update_status(message, self._last_id)
Beispiel #14
0
def main():
    auth = OAuthHandler(ckey, csecret)
    auth.set_access_token(atoken, asecret)
    twitterStream = Stream(auth, listener())
    if not file_exist('db_tweet.csv'):
        cabecalho('db_tweet.csv')
    twitterStream.filter(locations=[-46.825390,-24.008381,-46.364830,-23.357611])
Beispiel #15
0
def minetweets():
    line = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, line)
    # stream.filter(track=['Watson', 'Cognitive', 'Machine Learning'])
    stream.filter(track=args, languages=["en"])
Beispiel #16
0
def printStream():
    #This handles Twitter authetification and the connection to Twitter Streaming API
    l = StdOutListener()
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
    stream = Stream(auth, l)
    stream.sample()
    def startListen(self):
        auth = OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token, access_token_secret)
 
        stream = Stream(auth, self)
#     stream.filter(track=[positiveEmoticons])
        stream.filter(locations = [113.90625,-43.665217,157.148438,-13.35399])
def streaming_tweets(keywords, language=["en"]):
    """
    @keywords   ==  search keywords, e.g. ["ImWithHer", "Trump"]
    @languages  ==  desired language, e.g.: ["en"]

    """

    filename = keywords[0].replace(" ", "").replace("#", "") + ".csv"
    print(filename)

    try:
        start_csv()
        while True:
            try:
                #Initialize Tweepy Streamer
                twitterStream = Stream(auth, TwitterListener())
                twitterStream.filter(track=keywords, languages=language)
            except Exception as error:
                print(error)
                print("[*] saving results to {}".format(filename))
                os.rename("streaming_results.csv", filename)

    except KeyboardInterrupt:
        print("[*] User KeyboardInterrupt: Tweepy Streamer Haulted")
        print("[*] saving results to {}".format(filename))
        os.rename("streaming_results.csv", filename)
Beispiel #19
0
 def get_tweet_stream(self, track, game_id, game_name):
     index = self.get_auth()
     self.logger.info('Using auth: ' + str(self.auth.consumer_key))
     self.set_paths(track, game_id, game_name)
     stream = Stream(self.auth, self)
     stream.filter(track=[track], async=True)
     return stream, index
Beispiel #20
0
def interactive(username=None, password=None, filenames=None):
    if not username:
        username = raw_input('Username: ').strip()
    if not password:
        password = getpass("Password: ").strip()
    s = Stream(username, password, TWSSBuildClassifierListner())
    s.sample()
def mainloop():
    # Authenticate OAuth
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

    # Setup our API.
    api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
    if not api:
        print ("Can't authenticate")
        sys.exit(-1)

    # Iterate through all the usernames of the followers we want to follow
    # and extract their ID to be placed into an array of ID's our bot will
    # follow and automatically re-tweet.
    followers_array = []
    for screen_name in RETWEET_FOLLOWERS:
        profile = api.get_user(screen_name)
        followers_array.append(str(profile.id))

    for screen_name in LIKE_FOLLOWERS:
        profile = api.get_user(screen_name)
        followers_array.append(str(profile.id))

    # Run the streamer.
    stream = Stream(auth = api.auth, listener=ListenerAndRetweeter(api))
    stream.filter(follow=followers_array,track=[])
    def get_tweets(cls, keyword):

        global tweet_file

        # get the auth
        auth = cls.get_auth()
        # define the listener
        listener = StdOutListener()
        # define stream object
        stream = Stream(auth, listener)

        # define the api object
        api = tweepy.API(auth)

        current_milli_time = str(int(round(time.time() * 1000)))
        # open a file to write tweets
        tweet_file = open(keyword+'_'+current_milli_time+'.txt', 'a')

        try:
            # get past tweets, max 500
            result = tweepy.Cursor(api.search, q=keyword).items(10)
            for tweet in result:
                tweet_file.write(tweet.text.encode("UTF-8"))
                tweet_file.write('\n')
                #pprint(tweet)

            # Close the file
            # tweet_file.close()

            # run live feeds
            stream.filter(track=[keyword])
        except Exception as ex:
            print(ex.message, ex)
        finally:
            tweet_file.close()
class StreamConsumerThreadClass(threading.Thread):
    def __init__(self,term='',oauthfile=''):
        threading.Thread.__init__(self)
        self.searchterm = term
        self.name = term
        self.consume = True
        
        oauth = json.loads(open(oauthfile,'r').read())
        
        listener = MongoDBListener()
        auth = OAuthHandler(oauth['consumer_key'], oauth['consumer_secret'])
        auth.set_access_token(oauth['access_token'], oauth['access_token_secret'])
    
        self.stream = Stream(auth, listener,timeout=60)  
        
        
    def stopConsume(self):
        self.stream.disconnect()
      
    def run(self):
        now = datetime.datetime.now()
        print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)
        
        connected = False
        while True:
            try: 
                if not connected:
                    connected = True
                    self.stream.filter(track=[self.searchterm])
            except SSLError, e:
                print e
                connected = False            
Beispiel #24
0
def run(track_list):
    listener = StdOutListener()
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
     
    stream = Stream(auth, listener)
    stream.filter(track=track_list)
def stream_users(in_file, auth):    
    screen_names = [x.strip() for x in in_file]
    
    tw_list = user_ids(screen_names, auth)
    
    twitterStream = Stream(auth, Listener())
    twitterStream.filter(follow=tw_list)
Beispiel #26
0
    def run(self):
        global streamobj
        streamobj = Stream(self.auth, self.listener)

        #LOCATION OF USA = [-124.85, 24.39, -66.88, 49.38,] filter tweets from the USA, and are written in English
        streamobj.filter(locations = [-124.85, 24.39, -66.88, 49.38,], languages=['en'])
        return
Beispiel #27
0
    def get_streaming_data(self):
        tweets_grabbed = 0
        while (tweets_grabbed < self.num_tweets_to_grab):
            twitterStream = Stream(self.auth, listener(self.s, self.twit_utils, self.num_tweets_to_grab, self.retweet_count))
            try:               
                twitterStream.sample()
            except Exception as e:
                    print("Error. Restarting Stream.... Error: ")
                    print(e.__doc__)
                    #print(e.message)
                    print("Le Error! Restart")
                    time.sleep(3) # Sleep for 5 minutes if error ocurred
            finally:
                tweets_grabbed = self.s.get_tweets_grabbed()
                print("tweets_grabbed = ", tweets_grabbed)

        lang, top_lang,love_words, swear_words, top_tweets, countries = self.s.get_stats()

        print(Counter(lang))
        print(Counter(top_lang))
        print("Love Words {} Swear Words {}".format(love_words, swear_words))
        print(Counter(countries))

        self.c.execute("INSERT INTO lang_data VALUES (?,?, DATETIME('now'))", (str(list(Counter(lang).items())), str(list(Counter(top_lang).items()))))

        self.c.execute("INSERT INTO love_data VALUES (?,?, DATETIME('now'))", (love_words, swear_words))

        for t in top_tweets:
            self.c.execute("INSERT INTO twit_data VALUES (?, DATETIME('now'))", (t,))

        self.c.execute("INSERT INTO country_data VALUES (?, DATETIME('now'))", (str(list(Counter(countries).items())),))

        self.conn.commit()
def main(argv):
    query = ""
    try:
        opts, args = getopt.getopt(argv, "helps:w:i:", ["query="])
    except getopt.GetoptError:
        print ("stream.py --query <query>")
        sys.exit(2)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            print ("stream.py --query <query>")
            sys.exit()
        elif opt in ("-q", "--query"):
            query = arg


    array = []
    with open("keys.txt", "r") as ins:
        for line in ins:
            array.append(line.rstrip('\n'))
    l = StdOutListener()
    auth = OAuthHandler(array[0], array[1])
    auth.set_access_token(array[2], array[3])

    stream = Stream(auth, l)
    stream.filter(track=[query])
Beispiel #29
0
def setup_streams(auth):
    twitter_list = WBListener()
    #twitter_list2 = WBListener()
    global stream_obj

    stream_obj = Stream(auth, twitter_list)
    stream_obj.filter(track=['#trump'], async=True)
class TweetController:
    """docstring for Controller"""

    def __init__(self):
        self.settings = Settings()
        # self.auth = OAuthHandler(Listener.api_data["consumer_key"], Listener.api_data["consumer_secret"])
        # self.auth.set_access_token(Listener.api_data["access_token"], Listener.api_data["access_token_secret"])
        self.api = tweepy.API(self.settings.auth, parser=tweepy.parsers.JSONParser())

        self.db = DataBase()
        # self.tweet_gui = tweet_gui
        self.default_keyword = ['Obama', 'hillary ', 'Trump']
        self.db.create_table_if_not_exist()

    def start_stream(self):
        self.tweet_listener = Listener()
        self.stream = Stream(auth=self.settings.auth, listener=self.tweet_listener)
        self.stream.filter(track=self.default_keyword, async=True)

    def stop_stream(self):
        self.stream.disconnect()

    def set_keyword(self, default_keyword):
        self.default_keyword = default_keyword
        print(default_keyword)
        user_data = all_data["user"]

        location = user_data["location"]
        tweet = all_data["text"]
        sentiment_value = sentiment_analysis_module.sentiment(tweet)

        writer.writerow([tweet, location, sentiment_value])

        # just for me to track how many tweets I have saved
        self.count += 1

        if self.count % 100 == 0:
            print(self.count)

        return True

    def on_error(self, status):
        print(status)


searchTerm = "gun"

twitterStream = Stream(auth, listener())
with open('datasets/twitter_stream_data.csv', 'w', newline='') as csvfile:
    field_names = ['Tweet Text', 'Location', 'Sentiment Polarity']
    writer = csv.writer(csvfile)
    writer.writerow(field_names)
    twitterStream.filter(track=[searchTerm])


        compound=compound+senti        
        print (count)
        print (tweet.strip())
        print (senti)
        print (t)
        print (str(positive) + ' ' + str(negative) + ' ' + str(compound)) 
        
    
        plt.axis([ 0, 70, -20,20])
        plt.xlabel('Time')
        plt.ylabel('Sentiment')
        plt.plot([t],[positive],'go',[t] ,[negative],'ro',[t],[compound],'bo')
        plt.show()
        plt.pause(0.0001)
        if count==200:
            return False
        else:
            return True
        
    def on_error(self,status):
        print (status)


auth=OAuthHandler(ckey,csecret)
auth.set_access_token(atoken,asecret)

twitterStream=  Stream(auth, listener(count))
twitterStream.filter(track=["Donald Trump"])
      
 
Beispiel #33
0
            s = TextBlob(tweet)
            sentiment_value, confidence = s.sentiment.polarity,s.sentiment.subjectivity
            if ( sentiment_value > 0):
                sentiment_express = "pos"
            elif (sentiment_value <= 0 ):
                sentiment_express ="neg"
            print(tweet, sentiment_express, confidence)

            if confidence*100 >= 80:
                output = open("twitter-out.txt","a")
                output.write(sentiment_express)
                output.write('\n')
                output.close()

            return True
        except:
            return True

    def on_error(self, status):
        print(status)

auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)

twitterStream = Stream(auth, listener())
print(" \t \t \t  \t Subject Sentiment Analysis  Tool  -  BY Prashanth Kethavarapu
print("\n This Tool crawls over Twitter and analyze tweets of people based on the subject and gives out polarity of the sentiment based on the text  \n ")
print(" \t Enter Subject Title: ")
tracking = input(" ------>>\n")
twitterStream.filter(track=[tracking])
access_secret = 'qu9jNve0TvFCzbwcPETUMPVQRDFHdwcopaEhb10sGCq00'

auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth)

class MyListener(StreamListener):
    def on_data(self, data):
        try:
            with open('tweet_json/5_9_17_1_excited.txt', 'a') as f:
                f.write(data)
                return True
        except BaseException as e:
            print("Error on_data: %s" % str(e))

        return True

    def on_error(self, status):
        print(status)
        return True


while True:
    try:
        twitter_stream = Stream(auth, MyListener())
        twitter_stream.filter(track=['#excited','#happy','#veryhappy','#awesome','#amazing'])

    except:
        print('AGAIN ')
Beispiel #35
0
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import SimpleProducer, KafkaClient

access_token = "374527580-KZx5Xr7yHdiG3QtEqkQmT3wUg2uS6RQgvOo2Dd4N"
access_token_secret = "cPBNhI8OqyLj0nSh0VFfAgIztNIhWMMi7tnyxZwuvdZ1v"
consumer_key = "y025jxSQOFVdwdSPfTRjUNRdJ"
consumer_secret = "ANRM4lG1C1bXWAwiJ3uyLhBLE8rUSFn9xtmBhaTXvJxnllT87c"


class StdOutListener(StreamListener):
    def on_data(self, data):
        producer.send_messages("BVBB04", data.encode('utf-8'))
        print(data)
        return True

    def on_error(self, status):
        print(status)


kafka = KafkaClient("localhost:9092")
producer = SimpleProducer(kafka)
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=['#BVBB04'])
		# Ensure tweets have text, a user with a description, and aren't retweets or plain links
		if status.text and status.user.description and status.text[:2] != 'RT' and status.text[:3] != 'http':
			# Bundle the tweet data into json format
			data = {'id': status.id, 'text': status.text, 'description': status.user.description, 'followers': status.user.followers_count}
			data = json.dumps(data)

			# Update the count and print progress to console
			self.count += 1
			print(self.count, end='\r')

			# Send the json to the kinesis stream
			kinesis.put_record(
				StreamName='macs30123-final',
				Data=data,
				PartitionKey='partitionkey'
				)

	def on_error(self, status_code):
		print(status_code)
		return False

stream = Stream(auth=api.auth, listener=Listener())

try:
    print('Start streaming.')
    stream.filter(track=['Biden, #USElection2020'])
except KeyboardInterrupt:
    print("Stopped.")
finally:
    print('Done.')
    stream.disconnect()
            tweet = {'id':tweet_id, 'username':username, 'followers':followers, 'text':text, 'hashtags':hashtags, 'language':language, 'created':created}

        # Save the refined Tweet data to MongoDB
            collection.save(tweet)




        # Print the username / text to your console in realtime as they are pulled from stream
            print (username + ':' + ' ' + text)
            return True
        except:
            print(loadedtweet)

    # Prints error of code 
    def on_error(self, status):
        print (status)

# Pulls from variables at the top of the script
if __name__ == '__main__':
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    stream = Stream(auth, l)
    stream.filter(track=keywords, languages=language)




Beispiel #38
0
        tweet_dict['Elitebook'] = elitebook
        tweet_dict['Spectre'] = spectre
        tweet_dict['Hp2000'] = hp2000

        #        print tweet
        self.tweets.append(tweet_dict)
        if len(self.tweets) % 30 == 0:
            print len(self.tweets)
            twitterData = pd.DataFrame(self.tweets)
            twitterData.to_csv("tweets_HP_" + self.starttime + ".csv",
                               encoding="utf-8")
        if len(self.tweets) > 20000:
            exit(0)
        else:
            return True

    def on_error(self, status):
        print 'error' + status

    def on_timeout(self):
        print "timeout!"
        return False


lis = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)

stream = Stream(auth, lis, timeout=60)
stream.filter(track=['hp'])
consumer_secret = "YOUR TWITTER CONSUMER SECRET"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

#Tweepy= a Python library for accessing the Twitter API.  "api" allows us to use the data from Twitter
api = tweepy.API(auth)

# This allows for having the connection open and available if we want to keep streaming tweets.  We are saving the tweets to a json file called "trial.json'.  To narrow the scope of this project, we have closen to focus on the hashtag 'Bitcoin'.  However, if we wanted to look at another hashtag, this can easily be changed.


class MyListener(StreamListener):
    def on_data(self, data):
        try:
            with open('ethereum.json', 'a') as f:
                f.write(data)
                return True
        except BaseException as e:
            print("Error on_data: %s" % str(e))
        return True

    def on_error(self, status):
        print(status)
        return True


twitter_stream = Stream(auth, MyListener())
twitter_stream.filter(track=['#ethereum'])

#To check the output of this hashtag, uncomment the print statement below.  To get a good amount of twitter data, must let the stream.py run for some time.
#print('Streaming Twitter data. To stop streaming, press Ctrl and C', twitter_stream)
Beispiel #40
0
#consumer key, consumer secret, access token, access secret.
ckey = "xNjTg6GU87D2zmPlVV8h4EccP"
csecret = "lPPLUtHOs7MdWuZm2HG4aE7STZid4QZVdgYl3RiNBdUbRBFCwh"
atoken = "18593973-SPfJ0L4PDYP5UebUX4YSpVwSal1NUGLWB9FSZ8EYJ"
asecret = "6y106J4ftAI42QFk6awgGKqJNemiqL9dOr9hTRCXrZanZ"


class listener(StreamListener):
    def on_data(self, data):
        all_data = json.loads(data)
        tweet = all_data["text"]
        sentiment_value, confidence = s.sentiment(tweet)
        print(tweet, sentiment_value, confidence)

        if confidence * 100 >= 80:
            output = open("twitter-out.txt", "a")
            output.write(sentiment_value)
            output.write('\n')
            output.close()

        return (True)

    def on_error(self, status):
        print(status)


auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)

twitterStream = Stream(auth, listener())
twitterStream.filter(track=["happy"])
                            ['coordinates'][0][0][1])
            if lat and lon:
                print(lat)
                es.index(index=index_name,
                         id=id,
                         doc_type="tweet",
                         body={
                             "tweet": tweet,
                             "location": {
                                 "lat": lat,
                                 "lon": lon
                             }
                         })
        except Exception as e:
            print("ERROR: " + str(e))

    def on_error(self, status):
        print(status)


if __name__ == '__main__':
    # This handles Twitter authetification and the connection to Twitter Streaming API
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, l)

    # This line filter Twitter Streams to capture data by the keywords: 'python', 'javascript', 'ruby'
    stream.filter(
        track=['starbucks', 'android', 'national geographic', 'pets', 'music'])
        try:
            
            dictTweet["_id"] = str(dictTweet['id'])
            doc = db.save(dictTweet)
            print ("SAVED" + str(doc) +"=>" + str(data))
        except:
            print ("Already exists")
            pass
        return True
    
    def on_error(self, status):
        print (status)
        
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())

'''========couchdb'=========='''
server = couchdb.Server('http://*****:*****@localhost:5984/')  #('http://115.146.93.184:5984/')
try:
    db = server.create('elorogl')
except:
    db = server('elorogl')
    
'''===============LOCATIONS=============='''    

#Filtro por geolocalización
twitterStream.filter(locations=[-80.4678,-3.8901,-79.3654,-3.0471])
#Filtro por palabras
#twitterStream.filter(track=['Elecciones Ecuador 2021','Andres Arauz','CENTRO'])
twitterStream.filter(track=['Elecciones Ecuador 2021','Guillermo Lasso','CREO'])
Beispiel #43
0
            #Antes de guardar el documento puedes realizar parseo, limpieza y cierto analisis o filtrado de datos previo
            #a guardar en documento en la base de datos
            doc = db.save(
                dictTweet)  #Aqui se guarda el tweet en la base de couchDB
            print("Guardado " + "=> " + dictTweet["_id"])
        except:
            print("Documento ya existe")
            pass
        return True

    def on_error(self, status):
        print(status)


auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())

#Setear la URL del servidor de couchDB
server = couchdb.Server('http://localhost:5984/')
try:
    #Si no existe la Base de datos la crea
    db = server.create('abdalabucaram')
except:
    #Caso contrario solo conectarse a la base existente
    db = server['test']

#Aqui se define el bounding box con los limites geograficos donde recolectar los tweets
#twitterStream.filter(locations=[-92.21,-5.02,-75.19,1.88])
twitterStream.filter(track=['abdalabucaram'])
Beispiel #44
0
def sendData(c_socket):
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_secret)

    twitter_stream = Stream(auth, TweetsListener(c_socket))
    twitter_stream.filter(track=['python'])
        try:
            with open(file_name) as f:
                consumer = f.readline().rstrip('\n').split(',')
                access = f.readline().split(',')
        except IOError:
            print "Error  : Authentication file not found"

        if len(consumer) + len(access) != 4:
            print "Error : Expecting to retrieve 4 values"
        else:
            #consumer_key, consumer_secret
            self.auth = OAuthHandler(consumer[0], consumer[1])
            #access_token, access_token_secret
            self.auth.set_access_token(access[0], access[1])

    def get_auth(self):
        return self.auth


if __name__ == '__main__':
    # l = StdOutListener()
    l = StreamWatcherListener()

    myAuth = Authentification(oauth=True)
    #auth = OAuthHandler(consumer_key, consumer_secret)
    #auth.set_access_token(access_token, access_token_secret)
    #auth = BasicAuthHandler(username, password)

    stream = Stream(myAuth.get_auth(), l)
    stream.filter(track=['#ios'])
Beispiel #46
0
import sys

conf = cp.ConfigParser()
conf.read(sys.argv[1])
env = sys.argv[2]

consumer_key = conf.get(env, 'consumer_key')
consumer_secret = conf.get(env, 'consumer_secret')
access_token = conf.get(env, 'access_token')
access_token_secret = conf.get(env, 'access_token_secret')


# create a basic listener which would write received tweets to stdout
class StdOutListener(StreamListener):
    def on_data(self, data):
        print(data)
        return True

    def on_error(self, status):
        print(status)


if __name__ == '__main__':

    # This handles twitter authentication and connection to twitter streaming API
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, l)
    stream.filter(track=['kafka', 'apache spark'])
@author: Alex
"""
from __future__ import print_function
import tweepy
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener

ACCESS_TOKEN = '1201278023472840705-vXvkf0jOOturE1Kt6WxbT5bdMW7afP'
ACCESS_SECRET = '1R70xgn5OOBHYHxVcmCdr0vvX5DRdTzAuhaHdvIbT9hYJ'
CONSUMER_KEY = 'mUeXO93OhVbf8EqCOg2PmCKD4'
CONSUMER_SECRET = 'fJTzWcyPMSYvtiiFQZSK6x1Uf6FSV4HZEbg0sq9psU9H6im9Iz'

tweetDoc = open('twitterDoc.txt', 'w')

class listener(StreamListener):

    def on_data(self, data):
        print(data, file = tweetDoc) 
        return True

    def on_error(self, status):
        print(status)

auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
twitterStream = Stream(auth, listener(),tweet_mode="extended")
twitterStream.filter(track="Christmas")

tweetDoc.close()
Beispiel #48
0
            # Antes de guardar el documento puedes realizar parseo, limpieza y cierto analisis o filtrado de datos previo
            # a guardar en documento en la base de datos
            doc = db.save(
                dictTweet)  # Aqui se guarda el tweet en la base de couchDB
            print("Guardado " + "=> " + dictTweet["_id"])
        except:
            print("Documento ya existe")
            pass
        return True

    def on_error(self, status):
        print(status)


auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())

# Setear la URL del servidor de couchDB
server = couchdb.Server('http://*****:*****@localhost:5984/')
try:
    # Si no existe la Base de datos la crea
    db = server.create('santo_domingo')
except:
    # Caso contrario solo conectarse a la base existente
    db = server['santo_domingo']

# Aqui se define el bounding box con los limites geograficos donde recolectar los tweets
twitterStream.filter(track=["Santo Domingo", "turismo"])
# twitterStream.filter(locations=[-78.586922,-0.395161,-78.274155,0.021973])
                saveFile1 = open('tweetsDB.csv', 'a')
                saveFile1.write(saveThis)
                saveFile1.write('\n\n')
                saveFile1.close()

                saveFile2 = open('twitterDB.csv','a')
                saveFile2.write(data)
                saveFile2.close()
                return True
            
            except KeyboardInterrupt:
                keyInterrupt();            
            except BaseException as e:
                print('Failed: ', str(e))
                
        def on_error(self, status):
            print (status)
        
#Connecting to twitter and authorization            
try: 
    auth = OAuthHandler(ckey, csecret)
    auth.set_access_token(atoken, asecret)
    twitterStream = Stream(auth, listener())
except:
    print('Error: Authentication Failed')

try:
    twitterStream.filter(track=["india"])
except KeyboardInterrupt:
    keyInterrupt();
            #data = data.encode('utf-8')
            #data = data.decode('unicode_escape')
            print(data)
        except:
            pass
        return True

    def on_error(self, status):
        print('error:', status)


if __name__ == '__main__':
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, l)
    #stream.filter(track=['python', 'javascript', 'ruby'])

    raw_words = pd.read_csv('../data/twitter_filter/twt_filter_0.csv',
                            header=0,
                            quoting=3,
                            encoding='cp949',
                            error_bad_lines=False)
    raw_words = raw_words['words']
    values = raw_words.values

    filter_words = []
    for word in values:
        filter_words.append(word[0])

    stream.filter(track=filter_words)
Beispiel #51
0
 def tweetStream(self, word):
     twitter_stream = Stream(self.auth, MyListener())
     twitter_stream.filter(track=[word])
def sendData(c_socket):
    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
    twitter_stream = Stream(auth, TweetsListener(c_socket))
    twitter_stream.filter(track=['trending'])
Beispiel #53
0
def score_to_sentiment(score):
    try:
        if score <= -5: return "VERY_NEGATIVE"
        elif (score < 0 and score > -5): return "NEGATIVE"
        elif score == 0: return "NEUTRAL"
        elif score > 0 and score <= 5: return "POSITIVE"
        else: return "VERY_POSITIVE"

    except TypeError:
        return "NEUTRAL"


if __name__ == "__main__":

    soc = socket.socket()
    host = "127.0.0.1"
    port = 6666
    soc.bind((host, port))
    print("Listening on port: %s" % str(port))

    soc.listen(10)
    con_socket, ip_socket = soc.accept()
    print("Sending to: %s" % str(ip_socket))

    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_secret)

    afinn = Afinn()
    twitter_stream = Stream(auth, TweetListener(con_socket))
    twitter_stream.filter(languages=['en'], track=['BigData'])
Beispiel #54
0
            self.tweets = []
        self.count += 1
        # if we've grabbed more than total_tweets tweets, exit the script.
        # If this script is being run in the context of a kubernetes
        # replicationController, the pod will be restarted fresh when
        # that happens.
        if self.count > self.total_tweets:
            return False
        if (self.count % 1000) == 0:
            print 'count is: %s at %s' % (self.count, datetime.datetime.now())
        return True

    def on_error(self, status):
        print status


if __name__ == '__main__':
    print '....'
    listener = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    stream = Stream(auth, listener)
    stream.filter(track=[
        'blackfriday', 'blackfriday2016', 'blackfridaysale', 'cybermonday',
        'givingtuesday', 'thanksgiving', 'handsupdontspend',
        'blackoutblackfriday', 'notonedime', 'deals', 'giveaway', 'sale',
        'christmas', 'retail', 'amazon', 'walmart', 'apple', 'best buy',
        'bestbuy', 'target', 'kohls', 'bigsale'
    ])
Beispiel #55
0
            return False

    def on_error(self, status):
        if status == 420:
            return False

    def getData(self, data):
        return json.loads(data)


if __name__ == "__main__":
    #server="localhost:9092"
    #topicName="corona"
    #producer = KafkaProducer(bootstrap_servers=[server])

    listener = StdOutListener()
    auth = OAuthHandler(myCred.API_KEY, myCred.API_SECRET_KEY)
    auth.set_access_token(myCred.ACCESS_TOKEN, myCred.ACCESS_TOKEN_SECRET)
    stream = Stream(auth, listener)
    stream.filter(track=[topicName])

    #producer = KafkaProducer(bootstrap_servers=[server],
    #			 value_serializer=lambda v: json.dumps(v).encode('utf-8'))

    #producer.send(topicName,data)
    #producer.flush()
    #for e in range(1000):
    #	data = {'number' : e}
    #	producer.send(topicName, value=stream.filter(track=[topicName]).encode())
    #	time.sleep(5)
consumer_key = "i4yTWoWw1XVm5VkZfs2JFCjP9"
consumer_secret = "EwOMz7DIuNXczlGLh2mw2V9CXiWKfnzAiGyuieJYjCA0pz9VL2"

access_token = "1112707342876008450-POGZDGukHkPdghlBLVBBTsKjWEaTqq"
access_token_secret = "elUIcqWqi2FZqjuAACQI9CticYBJpYVtYiCNUYKSU5yqx"

# Define output file to store the collected tweets
todays_date = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
out = open(f"collected_tweets_{todays_date}.txt", "w")


# Implementar classe para conexao com o twitter
class MyListener(StreamListener):
    def on_data(self, data):
        itemString = json.dumps(data)
        out.write(itemString + "\n")
        return True

    def on_error(self, status):
        print(status)


# Implementar a funcao Main
if __name__ == "__main__":
    l = MyListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    stream = Stream(auth, l)
    stream.filter(track=["Trump"])
Beispiel #57
0
# In[ ]:

# consumer key authentication
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# set up the API with the authentication handler
api = API(auth)

# set up words to hear
keywords_to_hear = ['#Facebook', '#Apple', '#Instagram']

# instantiate the SListener object
listen = SListener(api)

# instantiate the stream object
stream = Stream(auth, listen)

# # create a engine to the database
# engine = create_engine("sqlite:///tweets.sqlite")
# # if the database does not exist
# if not database_exists(engine.url):
#     # create a new database
#     create_database(engine.url)

# begin collecting data
while True:
    # maintian connection unless interrupted
    try:
        stream.filter(track=keywords_to_hear)
    # reconnect automantically if error arise
    # due to unstable network connection
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import KafkaProducer
from os import environ

access_token = environ.get('twiiter_access_token')
access_token_secret = environ.get('twitter_access_token_secret')
consumer_key = environ.get('twitter_consumer_key')
consumer_secret = environ.get('twitter_consumer_secret')


class StdOutListener(StreamListener):
    def on_data(self, data):
        producer.send("covid-data", value=data.encode('utf-8'))
        print(data)
        return True

    def on_error(self, status):
        print(status)


if __name__ == '__main__':
    producer = KafkaProducer(bootstrap_servers=["localhost:9092"])
    out = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, out)
    stream.filter(track="covid")
access_token_secret = "pGjJkZOd7y2nkQim401WUhlLwsHzkcSKSt3KH5u5W5nyK"
consumer_key = "KZSmj4904W5KrhYb0jr1jTNrs"
consumer_secret = "KIPFzApbSpy78a9xIxQ7yhgk62NkgjE7Q4oCQ1iCATK4Pwowax"


#Tweepy listener prints out to system
class StdOutListener(StreamListener):
    def on_data(self, data):
        print(data)
        return True

    def on_status(self, status):
        print('full_text:', status.extended_tweet['full_text'])


#Connects to Twitter API
if __name__ == '__main__':

    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(
        auth, l,
        tweet_mode='extended')  #necessary tweet_mode param to get full text

    #Filter by keywords
    stream.filter(track=[
        '#DeepLearning', '#MachineLearning', '#ArtificialIntelligence',
        'Neural Networks'
    ])
Beispiel #60
0
from tweepy import OAuthHandler
from tweepy import Stream

#Variables that contains the user credentials to access Twitter API
access_token = "18196659-DBEYBxdsCHrBL0eUwVZnUx27onK5wJHZBONmsqThM"
access_token_secret = "TnoLLNka4YJue8c7khFYscg4Ari28Xs3YX7UBcb5CXT9d"
consumer_key = "qWR2fyEjOgTeKiUJjN0PmwQlX"
consumer_secret = "0JFag4zj5qunoZHNRHyXUKubs5LhGjWXrWz3K6fP6oApI4dzGI"


#This is a basic listener that just prints received tweets to stdout.
class StdOutListener(StreamListener):
    def on_data(self, data):
        print data
        return True

    def on_error(self, status):
        print status


if __name__ == '__main__':

    #This handles Twitter authetification and the connection to Twitter Streaming API
    l = StdOutListener()
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    stream = Stream(auth, l)

    #This line filter Twitter Streams to capture data by the keywords: 'python', 'javascript', 'ruby'
    #Changed to look for the words vaccine, vaccination
    stream.filter(track=['vaccine', 'vaccination', 'vaccin'])