def monitor(self, parent, connection):
ntpclient = ntplib.NTPClient()
silentChannelsFile = open("./modules/static/NOTWEETCHANNELS", 'r')
accountsToMonitorFile = open("./modules/static/TWITTERACCOUNTS", 'r')
self.silentChannels = silentChannelsFile.read().splitlines()
self.accounts = accountsToMonitorFile.read().splitlines()
silentChannelsFile.close()
accountsToMonitorFile.close()
cm = ChannelMessager()
cm.setup(self.silentChannels, self.accounts, connection, parent)
keyFile = open("modules/pubmsg/TwitterKeys", 'r')
keyList = keyFile.read().splitlines()
keyFile.close()
for entry in keyList:
if entry.startswith('#'):
keyList.remove(entry)
consumerKey = keyList[0]
consumerSecret = keyList[1]
accessKey = keyList[2]
accessSecret = keyList[3]
auth = OAuthHandler(consumerKey, consumerSecret)
auth.set_access_token(accessKey, accessSecret)
while True:
stream = Stream(auth, cm, timeout=60)
try:
stream.userstream(_with='followings')
except:
print "Error, restarting stream"
def start_stream(auth, l):
while True:
try:
stream = Stream(auth, l)
stream.sample()
except:
continue
def run(self):
l = StdOutListener(self)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=self.tags, languages=['en'])
def main():
# dirname = os.path.dirname(inspect.getfile(inspect.currentframe()))
# basename = os.path.basename(inspect.getfile(inspect.currentframe()))
dirname = os.path.dirname(os.path.realpath(__file__))
basename = os.path.basename(os.path.realpath(__file__))
name_noextension = os.path.splitext(basename)[0]
"""Start log."""
configinput = __import__("config" + name_noextension)
outputDir = os.path.join(dirname, configinput.directory)
if not os.path.exists(outputDir):
os.makedirs(outputDir)
logfilename = os.path.join(outputDir,basename) + ".log"
logging.basicConfig(filename=logfilename,level=logging.DEBUG, format='%(asctime)s %(message)s')
logging.info('Started')
save_pid()
"""Execute the twitter api."""
try:
auth = OAuthHandler(configinput.consumer_key, configinput.consumer_secret)
auth.set_access_token(configinput.access_token, configinput.access_secret)
twitter_stream = Stream(auth, MyListener(os.path.join(dirname, configinput.directory), os.path.join(dirname, configinput.to_dir), basename)) # el segundo argumento es el nombre del archibvo json
twitter_stream.filter(track=configinput.keyword_list_filter)
except BaseException as e:
logging.error('Failed to execute twitter api: ' + str(e))
logging.info('Finished')
class Streamer( object ): def __init__( self, queue, terms=[], consumer=None, consumer_secret=None, token=None, secret=None): if consumer == None or consumer_secret == None or token == None or secret == None: config = configparser.ConfigParser() config.readfp( open( os.path.expanduser( "~/.slackTwitter" ) ) ) consumer = config.get( "twitter", "consumer" ) consumer_secret = config.get( "twitter", "consumer_secret" ) token = config.get( "twitter", "token" ) secret = config.get( "twitter", "secret" ) auth = OAuthHandler( consumer, consumer_secret ) auth.set_access_token( token, secret ) listener = StreamerListener() self.stream = TweepyStream( auth=auth, listener=listener ) self._queue = queue self._terms = terms def start( self ): self.stream.filter( track=self._terms )
def handle(self, *args, **options):
politicians = Politician.objects.all();
politician_keywords = []
for politician in politicians:
politician_keywords.append(politician.first_name + " " + politician.last_name)
if politician.twitter_url:
indexSlash = politician.twitter_url.rfind("/")
indexQuestionMark = politician.twitter_url.rfind("?")
if indexQuestionMark != -1:
twitter = politician.twitter_url[indexSlash+1:indexQuestionMark]
else:
twitter = politician.twitter_url[indexSlash+1:]
politician_keywords.append(twitter)
# create instance of the tweepy tweet stream listener
listener = TweetStreamListener()
# set twitter keys/tokens
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# create instance of the tweepy stream
stream = Stream(auth, listener)
# search twitter for "congress" keyword
stream.filter(track=politician_keywords)
def stream_twitter(battle_id):
#Avoiding circular import
from battle.models import Battle
battle = Battle.objects.get(id=battle_id)
if battle.end_time < timezone.now():
return
battle.battlehashtags_set.update(typos=0, words=0)
battle_hashtags = battle.battlehashtags_set.all().prefetch_related('hashtag')
if battle_hashtags.count() == 0:
return
hashtag_values = [x.hashtag.value for x in battle_hashtags]
listener = TwitterStreamListener(battle_hashtags)
auth = OAuthHandler(
settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET
)
auth.set_access_token(
settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_ACCESS_TOKEN_SECRET
)
stream = Stream(auth, listener)
delay = battle.end_time - timezone.now()
Timer(delay.total_seconds(), stream.disconnect).start()
stream.filter(track=hashtag_values, languages=['en'])
def stream(buff, terms):
l = StdOutListener(buff)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=[terms])
def main():
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
backend = FileBackend("./test-db")
stream = Stream(auth, l)
stream.filter(track=['トレクル'])
def TwitterStream(kwords, lim, lang=['en'], loca=[-180, -90, 180, 90]):
# print kwords, lang, lim, loca
global limit
if type(lim) != tuple:
l = StdOutListener()
limit = int(lim)
else:
day = int(lim[0])
hour = int(lim[1])
minute = int(lim[2])
second = int(lim[3])
l = StdOutListener_time()
print time.time()
limit = time.time() + 86400 * day + 3600 * \
hour + 60 * minute + 1 * second
print limit
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
global results
results = list()
stream = Stream(auth, l)
# print kwords, lang
stream.filter(track=kwords, languages=['en'])
# def filter(self, follow=None, track=None, async=False, locations=None,
# stall_warnings=False, languages=None, encoding='utf8'):
return results
def run(self,user,message):
if utilities.getCommand() == "autotweet":
streamListener = ReplyToTweet()
streamListener.setAPI(self.twitterApi)
streamListener.setUser(self.account_user_id)
twitterStream = Stream(self.auth, streamListener)
twitterStream.userstream(_with='user')
def run_twitter_query():
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#names = list(np.array(get_companies())[:,1])
#print names[num1:num2]
d = hand_made_list()
search_list = []
for key, value in d.items():
if key == 'SPY':
search_list.append(value[0]) # append the full anme of the symbol
search_list.append('#SP500') # Don't append #SPY because it's not helpful
search_list.append('$SP500')
elif key == 'F':
# search_list.append(value[0]) # append the full name of the symbol
search_list.append('Ford') # append the name of the symbol
elif key == 'GE':
search_list.append('General Electric') # append the full anme of the symbol
elif key == 'S':
search_list.append('Sprint') # append the full anme of the symbol
elif key == 'T':
search_list.append('AT&T') # append the full anme of the symbol
elif key == 'MU':
search_list.append('Micron Tech') # append the full anme of the symbol
elif key == 'TRI':
search_list.append('Thomson Reuters') # append the full anme of the symbol
else:
for cell in value:
search_list.append(cell)
stream.filter(track=search_list)
class TwitterPlayer(player.Player):
def __init__(self, model, code, access_token, access_token_secret, opponent):
player.Player.__init__(self, model, code)
self._opponent = opponent
self._last_id = None
self._auth = OAuthHandler(auth.consumer_key, auth.consumer_secret)
self._auth.set_access_token(access_token, access_token_secret)
self._api = API(self._auth)
self._listener = TwitterListener(self, self._api)
self._stream = Stream(self._auth, self._listener)
@property
def username(self):
return self._auth.get_username()
def allow(self):
print 'This is the opponent\'s turn...'
self._stream.userstream()
def update(self, event):
if event.player == self.code:
return
message = '@%s %s' % (self._opponent, self._model.events[-1][1])
self.tweet(message)
def tweet(self, message):
if self._last_id is None:
self._api.update_status(message)
else:
self._api.update_status(message, self._last_id)
def main():
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
if not file_exist('db_tweet.csv'):
cabecalho('db_tweet.csv')
twitterStream.filter(locations=[-46.825390,-24.008381,-46.364830,-23.357611])
def minetweets():
line = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, line)
# stream.filter(track=['Watson', 'Cognitive', 'Machine Learning'])
stream.filter(track=args, languages=["en"])
def printStream():
#This handles Twitter authetification and the connection to Twitter Streaming API
l = StdOutListener()
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
stream = Stream(auth, l)
stream.sample()
def startListen(self):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, self)
# stream.filter(track=[positiveEmoticons])
stream.filter(locations = [113.90625,-43.665217,157.148438,-13.35399])
def streaming_tweets(keywords, language=["en"]):
"""
@keywords == search keywords, e.g. ["ImWithHer", "Trump"]
@languages == desired language, e.g.: ["en"]
"""
filename = keywords[0].replace(" ", "").replace("#", "") + ".csv"
print(filename)
try:
start_csv()
while True:
try:
#Initialize Tweepy Streamer
twitterStream = Stream(auth, TwitterListener())
twitterStream.filter(track=keywords, languages=language)
except Exception as error:
print(error)
print("[*] saving results to {}".format(filename))
os.rename("streaming_results.csv", filename)
except KeyboardInterrupt:
print("[*] User KeyboardInterrupt: Tweepy Streamer Haulted")
print("[*] saving results to {}".format(filename))
os.rename("streaming_results.csv", filename)
def get_tweet_stream(self, track, game_id, game_name):
index = self.get_auth()
self.logger.info('Using auth: ' + str(self.auth.consumer_key))
self.set_paths(track, game_id, game_name)
stream = Stream(self.auth, self)
stream.filter(track=[track], async=True)
return stream, index
def interactive(username=None, password=None, filenames=None):
if not username:
username = raw_input('Username: ').strip()
if not password:
password = getpass("Password: ").strip()
s = Stream(username, password, TWSSBuildClassifierListner())
s.sample()
def mainloop():
# Authenticate OAuth
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
# Setup our API.
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
if not api:
print ("Can't authenticate")
sys.exit(-1)
# Iterate through all the usernames of the followers we want to follow
# and extract their ID to be placed into an array of ID's our bot will
# follow and automatically re-tweet.
followers_array = []
for screen_name in RETWEET_FOLLOWERS:
profile = api.get_user(screen_name)
followers_array.append(str(profile.id))
for screen_name in LIKE_FOLLOWERS:
profile = api.get_user(screen_name)
followers_array.append(str(profile.id))
# Run the streamer.
stream = Stream(auth = api.auth, listener=ListenerAndRetweeter(api))
stream.filter(follow=followers_array,track=[])
def get_tweets(cls, keyword):
global tweet_file
# get the auth
auth = cls.get_auth()
# define the listener
listener = StdOutListener()
# define stream object
stream = Stream(auth, listener)
# define the api object
api = tweepy.API(auth)
current_milli_time = str(int(round(time.time() * 1000)))
# open a file to write tweets
tweet_file = open(keyword+'_'+current_milli_time+'.txt', 'a')
try:
# get past tweets, max 500
result = tweepy.Cursor(api.search, q=keyword).items(10)
for tweet in result:
tweet_file.write(tweet.text.encode("UTF-8"))
tweet_file.write('\n')
#pprint(tweet)
# Close the file
# tweet_file.close()
# run live feeds
stream.filter(track=[keyword])
except Exception as ex:
print(ex.message, ex)
finally:
tweet_file.close()
class StreamConsumerThreadClass(threading.Thread):
def __init__(self,term='',oauthfile=''):
threading.Thread.__init__(self)
self.searchterm = term
self.name = term
self.consume = True
oauth = json.loads(open(oauthfile,'r').read())
listener = MongoDBListener()
auth = OAuthHandler(oauth['consumer_key'], oauth['consumer_secret'])
auth.set_access_token(oauth['access_token'], oauth['access_token_secret'])
self.stream = Stream(auth, listener,timeout=60)
def stopConsume(self):
self.stream.disconnect()
def run(self):
now = datetime.datetime.now()
print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)
connected = False
while True:
try:
if not connected:
connected = True
self.stream.filter(track=[self.searchterm])
except SSLError, e:
print e
connected = False
def run(track_list):
listener = StdOutListener()
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, listener)
stream.filter(track=track_list)
def stream_users(in_file, auth):
screen_names = [x.strip() for x in in_file]
tw_list = user_ids(screen_names, auth)
twitterStream = Stream(auth, Listener())
twitterStream.filter(follow=tw_list)
def run(self):
global streamobj
streamobj = Stream(self.auth, self.listener)
#LOCATION OF USA = [-124.85, 24.39, -66.88, 49.38,] filter tweets from the USA, and are written in English
streamobj.filter(locations = [-124.85, 24.39, -66.88, 49.38,], languages=['en'])
return
def get_streaming_data(self):
tweets_grabbed = 0
while (tweets_grabbed < self.num_tweets_to_grab):
twitterStream = Stream(self.auth, listener(self.s, self.twit_utils, self.num_tweets_to_grab, self.retweet_count))
try:
twitterStream.sample()
except Exception as e:
print("Error. Restarting Stream.... Error: ")
print(e.__doc__)
#print(e.message)
print("Le Error! Restart")
time.sleep(3) # Sleep for 5 minutes if error ocurred
finally:
tweets_grabbed = self.s.get_tweets_grabbed()
print("tweets_grabbed = ", tweets_grabbed)
lang, top_lang,love_words, swear_words, top_tweets, countries = self.s.get_stats()
print(Counter(lang))
print(Counter(top_lang))
print("Love Words {} Swear Words {}".format(love_words, swear_words))
print(Counter(countries))
self.c.execute("INSERT INTO lang_data VALUES (?,?, DATETIME('now'))", (str(list(Counter(lang).items())), str(list(Counter(top_lang).items()))))
self.c.execute("INSERT INTO love_data VALUES (?,?, DATETIME('now'))", (love_words, swear_words))
for t in top_tweets:
self.c.execute("INSERT INTO twit_data VALUES (?, DATETIME('now'))", (t,))
self.c.execute("INSERT INTO country_data VALUES (?, DATETIME('now'))", (str(list(Counter(countries).items())),))
self.conn.commit()
def main(argv):
query = ""
try:
opts, args = getopt.getopt(argv, "helps:w:i:", ["query="])
except getopt.GetoptError:
print ("stream.py --query <query>")
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
print ("stream.py --query <query>")
sys.exit()
elif opt in ("-q", "--query"):
query = arg
array = []
with open("keys.txt", "r") as ins:
for line in ins:
array.append(line.rstrip('\n'))
l = StdOutListener()
auth = OAuthHandler(array[0], array[1])
auth.set_access_token(array[2], array[3])
stream = Stream(auth, l)
stream.filter(track=[query])
def setup_streams(auth):
twitter_list = WBListener()
#twitter_list2 = WBListener()
global stream_obj
stream_obj = Stream(auth, twitter_list)
stream_obj.filter(track=['#trump'], async=True)
class TweetController:
"""docstring for Controller"""
def __init__(self):
self.settings = Settings()
# self.auth = OAuthHandler(Listener.api_data["consumer_key"], Listener.api_data["consumer_secret"])
# self.auth.set_access_token(Listener.api_data["access_token"], Listener.api_data["access_token_secret"])
self.api = tweepy.API(self.settings.auth, parser=tweepy.parsers.JSONParser())
self.db = DataBase()
# self.tweet_gui = tweet_gui
self.default_keyword = ['Obama', 'hillary ', 'Trump']
self.db.create_table_if_not_exist()
def start_stream(self):
self.tweet_listener = Listener()
self.stream = Stream(auth=self.settings.auth, listener=self.tweet_listener)
self.stream.filter(track=self.default_keyword, async=True)
def stop_stream(self):
self.stream.disconnect()
def set_keyword(self, default_keyword):
self.default_keyword = default_keyword
print(default_keyword)
user_data = all_data["user"]
location = user_data["location"]
tweet = all_data["text"]
sentiment_value = sentiment_analysis_module.sentiment(tweet)
writer.writerow([tweet, location, sentiment_value])
# just for me to track how many tweets I have saved
self.count += 1
if self.count % 100 == 0:
print(self.count)
return True
def on_error(self, status):
print(status)
searchTerm = "gun"
twitterStream = Stream(auth, listener())
with open('datasets/twitter_stream_data.csv', 'w', newline='') as csvfile:
field_names = ['Tweet Text', 'Location', 'Sentiment Polarity']
writer = csv.writer(csvfile)
writer.writerow(field_names)
twitterStream.filter(track=[searchTerm])
compound=compound+senti
print (count)
print (tweet.strip())
print (senti)
print (t)
print (str(positive) + ' ' + str(negative) + ' ' + str(compound))
plt.axis([ 0, 70, -20,20])
plt.xlabel('Time')
plt.ylabel('Sentiment')
plt.plot([t],[positive],'go',[t] ,[negative],'ro',[t],[compound],'bo')
plt.show()
plt.pause(0.0001)
if count==200:
return False
else:
return True
def on_error(self,status):
print (status)
auth=OAuthHandler(ckey,csecret)
auth.set_access_token(atoken,asecret)
twitterStream= Stream(auth, listener(count))
twitterStream.filter(track=["Donald Trump"])
s = TextBlob(tweet)
sentiment_value, confidence = s.sentiment.polarity,s.sentiment.subjectivity
if ( sentiment_value > 0):
sentiment_express = "pos"
elif (sentiment_value <= 0 ):
sentiment_express ="neg"
print(tweet, sentiment_express, confidence)
if confidence*100 >= 80:
output = open("twitter-out.txt","a")
output.write(sentiment_express)
output.write('\n')
output.close()
return True
except:
return True
def on_error(self, status):
print(status)
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
print(" \t \t \t \t Subject Sentiment Analysis Tool - BY Prashanth Kethavarapu
print("\n This Tool crawls over Twitter and analyze tweets of people based on the subject and gives out polarity of the sentiment based on the text \n ")
print(" \t Enter Subject Title: ")
tracking = input(" ------>>\n")
twitterStream.filter(track=[tracking])
access_secret = 'qu9jNve0TvFCzbwcPETUMPVQRDFHdwcopaEhb10sGCq00'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
class MyListener(StreamListener):
def on_data(self, data):
try:
with open('tweet_json/5_9_17_1_excited.txt', 'a') as f:
f.write(data)
return True
except BaseException as e:
print("Error on_data: %s" % str(e))
return True
def on_error(self, status):
print(status)
return True
while True:
try:
twitter_stream = Stream(auth, MyListener())
twitter_stream.filter(track=['#excited','#happy','#veryhappy','#awesome','#amazing'])
except:
print('AGAIN ')
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import SimpleProducer, KafkaClient
access_token = "374527580-KZx5Xr7yHdiG3QtEqkQmT3wUg2uS6RQgvOo2Dd4N"
access_token_secret = "cPBNhI8OqyLj0nSh0VFfAgIztNIhWMMi7tnyxZwuvdZ1v"
consumer_key = "y025jxSQOFVdwdSPfTRjUNRdJ"
consumer_secret = "ANRM4lG1C1bXWAwiJ3uyLhBLE8rUSFn9xtmBhaTXvJxnllT87c"
class StdOutListener(StreamListener):
def on_data(self, data):
producer.send_messages("BVBB04", data.encode('utf-8'))
print(data)
return True
def on_error(self, status):
print(status)
kafka = KafkaClient("localhost:9092")
producer = SimpleProducer(kafka)
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=['#BVBB04'])
# Ensure tweets have text, a user with a description, and aren't retweets or plain links
if status.text and status.user.description and status.text[:2] != 'RT' and status.text[:3] != 'http':
# Bundle the tweet data into json format
data = {'id': status.id, 'text': status.text, 'description': status.user.description, 'followers': status.user.followers_count}
data = json.dumps(data)
# Update the count and print progress to console
self.count += 1
print(self.count, end='\r')
# Send the json to the kinesis stream
kinesis.put_record(
StreamName='macs30123-final',
Data=data,
PartitionKey='partitionkey'
)
def on_error(self, status_code):
print(status_code)
return False
stream = Stream(auth=api.auth, listener=Listener())
try:
print('Start streaming.')
stream.filter(track=['Biden, #USElection2020'])
except KeyboardInterrupt:
print("Stopped.")
finally:
print('Done.')
stream.disconnect()
tweet = {'id':tweet_id, 'username':username, 'followers':followers, 'text':text, 'hashtags':hashtags, 'language':language, 'created':created}
# Save the refined Tweet data to MongoDB
collection.save(tweet)
# Print the username / text to your console in realtime as they are pulled from stream
print (username + ':' + ' ' + text)
return True
except:
print(loadedtweet)
# Prints error of code
def on_error(self, status):
print (status)
# Pulls from variables at the top of the script
if __name__ == '__main__':
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=keywords, languages=language)
tweet_dict['Elitebook'] = elitebook
tweet_dict['Spectre'] = spectre
tweet_dict['Hp2000'] = hp2000
# print tweet
self.tweets.append(tweet_dict)
if len(self.tweets) % 30 == 0:
print len(self.tweets)
twitterData = pd.DataFrame(self.tweets)
twitterData.to_csv("tweets_HP_" + self.starttime + ".csv",
encoding="utf-8")
if len(self.tweets) > 20000:
exit(0)
else:
return True
def on_error(self, status):
print 'error' + status
def on_timeout(self):
print "timeout!"
return False
lis = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, lis, timeout=60)
stream.filter(track=['hp'])
consumer_secret = "YOUR TWITTER CONSUMER SECRET"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
#Tweepy= a Python library for accessing the Twitter API. "api" allows us to use the data from Twitter
api = tweepy.API(auth)
# This allows for having the connection open and available if we want to keep streaming tweets. We are saving the tweets to a json file called "trial.json'. To narrow the scope of this project, we have closen to focus on the hashtag 'Bitcoin'. However, if we wanted to look at another hashtag, this can easily be changed.
class MyListener(StreamListener):
def on_data(self, data):
try:
with open('ethereum.json', 'a') as f:
f.write(data)
return True
except BaseException as e:
print("Error on_data: %s" % str(e))
return True
def on_error(self, status):
print(status)
return True
twitter_stream = Stream(auth, MyListener())
twitter_stream.filter(track=['#ethereum'])
#To check the output of this hashtag, uncomment the print statement below. To get a good amount of twitter data, must let the stream.py run for some time.
#print('Streaming Twitter data. To stop streaming, press Ctrl and C', twitter_stream)
#consumer key, consumer secret, access token, access secret.
ckey = "xNjTg6GU87D2zmPlVV8h4EccP"
csecret = "lPPLUtHOs7MdWuZm2HG4aE7STZid4QZVdgYl3RiNBdUbRBFCwh"
atoken = "18593973-SPfJ0L4PDYP5UebUX4YSpVwSal1NUGLWB9FSZ8EYJ"
asecret = "6y106J4ftAI42QFk6awgGKqJNemiqL9dOr9hTRCXrZanZ"
class listener(StreamListener):
def on_data(self, data):
all_data = json.loads(data)
tweet = all_data["text"]
sentiment_value, confidence = s.sentiment(tweet)
print(tweet, sentiment_value, confidence)
if confidence * 100 >= 80:
output = open("twitter-out.txt", "a")
output.write(sentiment_value)
output.write('\n')
output.close()
return (True)
def on_error(self, status):
print(status)
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
twitterStream.filter(track=["happy"])
['coordinates'][0][0][1])
if lat and lon:
print(lat)
es.index(index=index_name,
id=id,
doc_type="tweet",
body={
"tweet": tweet,
"location": {
"lat": lat,
"lon": lon
}
})
except Exception as e:
print("ERROR: " + str(e))
def on_error(self, status):
print(status)
if __name__ == '__main__':
# This handles Twitter authetification and the connection to Twitter Streaming API
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
# This line filter Twitter Streams to capture data by the keywords: 'python', 'javascript', 'ruby'
stream.filter(
track=['starbucks', 'android', 'national geographic', 'pets', 'music'])
try:
dictTweet["_id"] = str(dictTweet['id'])
doc = db.save(dictTweet)
print ("SAVED" + str(doc) +"=>" + str(data))
except:
print ("Already exists")
pass
return True
def on_error(self, status):
print (status)
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
'''========couchdb'=========='''
server = couchdb.Server('http://*****:*****@localhost:5984/') #('http://115.146.93.184:5984/')
try:
db = server.create('elorogl')
except:
db = server('elorogl')
'''===============LOCATIONS=============='''
#Filtro por geolocalización
twitterStream.filter(locations=[-80.4678,-3.8901,-79.3654,-3.0471])
#Filtro por palabras
#twitterStream.filter(track=['Elecciones Ecuador 2021','Andres Arauz','CENTRO'])
twitterStream.filter(track=['Elecciones Ecuador 2021','Guillermo Lasso','CREO'])
#Antes de guardar el documento puedes realizar parseo, limpieza y cierto analisis o filtrado de datos previo
#a guardar en documento en la base de datos
doc = db.save(
dictTweet) #Aqui se guarda el tweet en la base de couchDB
print("Guardado " + "=> " + dictTweet["_id"])
except:
print("Documento ya existe")
pass
return True
def on_error(self, status):
print(status)
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
#Setear la URL del servidor de couchDB
server = couchdb.Server('http://localhost:5984/')
try:
#Si no existe la Base de datos la crea
db = server.create('abdalabucaram')
except:
#Caso contrario solo conectarse a la base existente
db = server['test']
#Aqui se define el bounding box con los limites geograficos donde recolectar los tweets
#twitterStream.filter(locations=[-92.21,-5.02,-75.19,1.88])
twitterStream.filter(track=['abdalabucaram'])
def sendData(c_socket):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(track=['python'])
try:
with open(file_name) as f:
consumer = f.readline().rstrip('\n').split(',')
access = f.readline().split(',')
except IOError:
print "Error : Authentication file not found"
if len(consumer) + len(access) != 4:
print "Error : Expecting to retrieve 4 values"
else:
#consumer_key, consumer_secret
self.auth = OAuthHandler(consumer[0], consumer[1])
#access_token, access_token_secret
self.auth.set_access_token(access[0], access[1])
def get_auth(self):
return self.auth
if __name__ == '__main__':
# l = StdOutListener()
l = StreamWatcherListener()
myAuth = Authentification(oauth=True)
#auth = OAuthHandler(consumer_key, consumer_secret)
#auth.set_access_token(access_token, access_token_secret)
#auth = BasicAuthHandler(username, password)
stream = Stream(myAuth.get_auth(), l)
stream.filter(track=['#ios'])
import sys
conf = cp.ConfigParser()
conf.read(sys.argv[1])
env = sys.argv[2]
consumer_key = conf.get(env, 'consumer_key')
consumer_secret = conf.get(env, 'consumer_secret')
access_token = conf.get(env, 'access_token')
access_token_secret = conf.get(env, 'access_token_secret')
# create a basic listener which would write received tweets to stdout
class StdOutListener(StreamListener):
def on_data(self, data):
print(data)
return True
def on_error(self, status):
print(status)
if __name__ == '__main__':
# This handles twitter authentication and connection to twitter streaming API
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=['kafka', 'apache spark'])
@author: Alex
"""
from __future__ import print_function
import tweepy
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener
ACCESS_TOKEN = '1201278023472840705-vXvkf0jOOturE1Kt6WxbT5bdMW7afP'
ACCESS_SECRET = '1R70xgn5OOBHYHxVcmCdr0vvX5DRdTzAuhaHdvIbT9hYJ'
CONSUMER_KEY = 'mUeXO93OhVbf8EqCOg2PmCKD4'
CONSUMER_SECRET = 'fJTzWcyPMSYvtiiFQZSK6x1Uf6FSV4HZEbg0sq9psU9H6im9Iz'
tweetDoc = open('twitterDoc.txt', 'w')
class listener(StreamListener):
def on_data(self, data):
print(data, file = tweetDoc)
return True
def on_error(self, status):
print(status)
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
twitterStream = Stream(auth, listener(),tweet_mode="extended")
twitterStream.filter(track="Christmas")
tweetDoc.close()
# Antes de guardar el documento puedes realizar parseo, limpieza y cierto analisis o filtrado de datos previo
# a guardar en documento en la base de datos
doc = db.save(
dictTweet) # Aqui se guarda el tweet en la base de couchDB
print("Guardado " + "=> " + dictTweet["_id"])
except:
print("Documento ya existe")
pass
return True
def on_error(self, status):
print(status)
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
# Setear la URL del servidor de couchDB
server = couchdb.Server('http://*****:*****@localhost:5984/')
try:
# Si no existe la Base de datos la crea
db = server.create('santo_domingo')
except:
# Caso contrario solo conectarse a la base existente
db = server['santo_domingo']
# Aqui se define el bounding box con los limites geograficos donde recolectar los tweets
twitterStream.filter(track=["Santo Domingo", "turismo"])
# twitterStream.filter(locations=[-78.586922,-0.395161,-78.274155,0.021973])
saveFile1 = open('tweetsDB.csv', 'a')
saveFile1.write(saveThis)
saveFile1.write('\n\n')
saveFile1.close()
saveFile2 = open('twitterDB.csv','a')
saveFile2.write(data)
saveFile2.close()
return True
except KeyboardInterrupt:
keyInterrupt();
except BaseException as e:
print('Failed: ', str(e))
def on_error(self, status):
print (status)
#Connecting to twitter and authorization
try:
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
except:
print('Error: Authentication Failed')
try:
twitterStream.filter(track=["india"])
except KeyboardInterrupt:
keyInterrupt();
#data = data.encode('utf-8')
#data = data.decode('unicode_escape')
print(data)
except:
pass
return True
def on_error(self, status):
print('error:', status)
if __name__ == '__main__':
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#stream.filter(track=['python', 'javascript', 'ruby'])
raw_words = pd.read_csv('../data/twitter_filter/twt_filter_0.csv',
header=0,
quoting=3,
encoding='cp949',
error_bad_lines=False)
raw_words = raw_words['words']
values = raw_words.values
filter_words = []
for word in values:
filter_words.append(word[0])
stream.filter(track=filter_words)
def tweetStream(self, word):
twitter_stream = Stream(self.auth, MyListener())
twitter_stream.filter(track=[word])
def sendData(c_socket):
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
twitter_stream = Stream(auth, TweetsListener(c_socket))
twitter_stream.filter(track=['trending'])
def score_to_sentiment(score):
try:
if score <= -5: return "VERY_NEGATIVE"
elif (score < 0 and score > -5): return "NEGATIVE"
elif score == 0: return "NEUTRAL"
elif score > 0 and score <= 5: return "POSITIVE"
else: return "VERY_POSITIVE"
except TypeError:
return "NEUTRAL"
if __name__ == "__main__":
soc = socket.socket()
host = "127.0.0.1"
port = 6666
soc.bind((host, port))
print("Listening on port: %s" % str(port))
soc.listen(10)
con_socket, ip_socket = soc.accept()
print("Sending to: %s" % str(ip_socket))
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
afinn = Afinn()
twitter_stream = Stream(auth, TweetListener(con_socket))
twitter_stream.filter(languages=['en'], track=['BigData'])
self.tweets = []
self.count += 1
# if we've grabbed more than total_tweets tweets, exit the script.
# If this script is being run in the context of a kubernetes
# replicationController, the pod will be restarted fresh when
# that happens.
if self.count > self.total_tweets:
return False
if (self.count % 1000) == 0:
print 'count is: %s at %s' % (self.count, datetime.datetime.now())
return True
def on_error(self, status):
print status
if __name__ == '__main__':
print '....'
listener = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, listener)
stream.filter(track=[
'blackfriday', 'blackfriday2016', 'blackfridaysale', 'cybermonday',
'givingtuesday', 'thanksgiving', 'handsupdontspend',
'blackoutblackfriday', 'notonedime', 'deals', 'giveaway', 'sale',
'christmas', 'retail', 'amazon', 'walmart', 'apple', 'best buy',
'bestbuy', 'target', 'kohls', 'bigsale'
])
return False
def on_error(self, status):
if status == 420:
return False
def getData(self, data):
return json.loads(data)
if __name__ == "__main__":
#server="localhost:9092"
#topicName="corona"
#producer = KafkaProducer(bootstrap_servers=[server])
listener = StdOutListener()
auth = OAuthHandler(myCred.API_KEY, myCred.API_SECRET_KEY)
auth.set_access_token(myCred.ACCESS_TOKEN, myCred.ACCESS_TOKEN_SECRET)
stream = Stream(auth, listener)
stream.filter(track=[topicName])
#producer = KafkaProducer(bootstrap_servers=[server],
# value_serializer=lambda v: json.dumps(v).encode('utf-8'))
#producer.send(topicName,data)
#producer.flush()
#for e in range(1000):
# data = {'number' : e}
# producer.send(topicName, value=stream.filter(track=[topicName]).encode())
# time.sleep(5)
consumer_key = "i4yTWoWw1XVm5VkZfs2JFCjP9"
consumer_secret = "EwOMz7DIuNXczlGLh2mw2V9CXiWKfnzAiGyuieJYjCA0pz9VL2"
access_token = "1112707342876008450-POGZDGukHkPdghlBLVBBTsKjWEaTqq"
access_token_secret = "elUIcqWqi2FZqjuAACQI9CticYBJpYVtYiCNUYKSU5yqx"
# Define output file to store the collected tweets
todays_date = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
out = open(f"collected_tweets_{todays_date}.txt", "w")
# Implementar classe para conexao com o twitter
class MyListener(StreamListener):
def on_data(self, data):
itemString = json.dumps(data)
out.write(itemString + "\n")
return True
def on_error(self, status):
print(status)
# Implementar a funcao Main
if __name__ == "__main__":
l = MyListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=["Trump"])
# In[ ]:
# consumer key authentication
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# set up the API with the authentication handler
api = API(auth)
# set up words to hear
keywords_to_hear = ['#Facebook', '#Apple', '#Instagram']
# instantiate the SListener object
listen = SListener(api)
# instantiate the stream object
stream = Stream(auth, listen)
# # create a engine to the database
# engine = create_engine("sqlite:///tweets.sqlite")
# # if the database does not exist
# if not database_exists(engine.url):
# # create a new database
# create_database(engine.url)
# begin collecting data
while True:
# maintian connection unless interrupted
try:
stream.filter(track=keywords_to_hear)
# reconnect automantically if error arise
# due to unstable network connection
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from kafka import KafkaProducer
from os import environ
access_token = environ.get('twiiter_access_token')
access_token_secret = environ.get('twitter_access_token_secret')
consumer_key = environ.get('twitter_consumer_key')
consumer_secret = environ.get('twitter_consumer_secret')
class StdOutListener(StreamListener):
def on_data(self, data):
producer.send("covid-data", value=data.encode('utf-8'))
print(data)
return True
def on_error(self, status):
print(status)
if __name__ == '__main__':
producer = KafkaProducer(bootstrap_servers=["localhost:9092"])
out = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, out)
stream.filter(track="covid")
access_token_secret = "pGjJkZOd7y2nkQim401WUhlLwsHzkcSKSt3KH5u5W5nyK"
consumer_key = "KZSmj4904W5KrhYb0jr1jTNrs"
consumer_secret = "KIPFzApbSpy78a9xIxQ7yhgk62NkgjE7Q4oCQ1iCATK4Pwowax"
#Tweepy listener prints out to system
class StdOutListener(StreamListener):
def on_data(self, data):
print(data)
return True
def on_status(self, status):
print('full_text:', status.extended_tweet['full_text'])
#Connects to Twitter API
if __name__ == '__main__':
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(
auth, l,
tweet_mode='extended') #necessary tweet_mode param to get full text
#Filter by keywords
stream.filter(track=[
'#DeepLearning', '#MachineLearning', '#ArtificialIntelligence',
'Neural Networks'
])
from tweepy import OAuthHandler
from tweepy import Stream
#Variables that contains the user credentials to access Twitter API
access_token = "18196659-DBEYBxdsCHrBL0eUwVZnUx27onK5wJHZBONmsqThM"
access_token_secret = "TnoLLNka4YJue8c7khFYscg4Ari28Xs3YX7UBcb5CXT9d"
consumer_key = "qWR2fyEjOgTeKiUJjN0PmwQlX"
consumer_secret = "0JFag4zj5qunoZHNRHyXUKubs5LhGjWXrWz3K6fP6oApI4dzGI"
#This is a basic listener that just prints received tweets to stdout.
class StdOutListener(StreamListener):
def on_data(self, data):
print data
return True
def on_error(self, status):
print status
if __name__ == '__main__':
#This handles Twitter authetification and the connection to Twitter Streaming API
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#This line filter Twitter Streams to capture data by the keywords: 'python', 'javascript', 'ruby'
#Changed to look for the words vaccine, vaccination
stream.filter(track=['vaccine', 'vaccination', 'vaccin'])