def minetweets():
line = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, line)
# stream.filter(track=['Watson', 'Cognitive', 'Machine Learning'])
stream.filter(track=args, languages=["en"])
def tweet(title, url, location=None, parsed_location=None):
auth = OAuthHandler(app.config['TWITTER_CONSUMER_KEY'], app.config['TWITTER_CONSUMER_SECRET'])
auth.set_access_token(app.config['TWITTER_ACCESS_KEY'], app.config['TWITTER_ACCESS_SECRET'])
api = API(auth)
urllength = 23 # Current Twitter standard for HTTPS (as of Oct 2014)
maxlength = 140 - urllength - 1 # == 116
locationtag = u''
if parsed_location:
locationtags = []
for token in parsed_location.get('tokens', []):
if 'geoname' in token and 'token' in token:
locname = token['token'].strip()
if locname:
locationtags.append(u'#' + locname.title().replace(u' ', ''))
locationtag = u' '.join(locationtags)
if locationtag:
maxlength -= len(locationtag) + 1
if not locationtag and location:
# Make a hashtag from the first word in the location. This catches
# locations like 'Anywhere' which have no geonameid but are still valid
locationtag = u'#' + re.split('\W+', location)[0]
maxlength -= len(locationtag) + 1
if len(title) > maxlength:
text = title[:maxlength-1] + u'…'
else:
text = title[:maxlength]
text = text + ' ' + url # Don't shorten URLs, now that there's t.co
if locationtag:
text = text + ' ' + locationtag
api.update_status(text)
def unwrapped_callback(self, resp):
if resp is None:
raise LoginCallbackError(_("You denied the request to login"))
# Try to read more from the user's Twitter profile
auth = TwitterOAuthHandler(self.consumer_key, self.consumer_secret)
if self.access_key is not None and self.access_secret is not None:
auth.set_access_token(self.access_key, self.access_secret)
else:
auth.set_access_token(resp['oauth_token'], resp['oauth_token_secret'])
api = TwitterAPI(auth)
try:
twinfo = api.lookup_users(user_ids=[resp['user_id']])[0]
fullname = twinfo.name
avatar_url = twinfo.profile_image_url_https.replace('_normal.', '_bigger.')
except TweepError:
fullname = None
avatar_url = None
return {'userid': resp['user_id'],
'username': resp['screen_name'],
'fullname': fullname,
'avatar_url': avatar_url,
'oauth_token': resp['oauth_token'],
'oauth_token_secret': resp['oauth_token_secret'],
'oauth_token_type': None, # Twitter doesn't have token types
}
class MainRunner:
def __init__(self, conf):
self._conf = conf
self._out = None
self.listeners = []
self._auth = OAuthHandler(conf.consumer_key, conf.consumer_secret)
self._auth.set_access_token(conf.access_token, conf.access_token_secret)
self._run_listener()
def _run_listener(self):
listener = Listener(self.out, self._conf.places)
stream = Stream(self._auth, listener)
locations = []
for city in self._conf.places:
locations.extend(city['southwest'].values())
locations.extend(city['northeast'].values())
stream.filter(locations=locations)
@property
def out(self):
if self._out is None:
try:
self._out = open(self._conf.output, 'a')
except FileNotFoundError:
if path.exists('output.txt'):
self._out = open('output.txt', 'a')
else:
self._out = open('output.txt', 'a')
return self._out
def __del__(self):
self.out.close()
def twitter_login(request):
"""
ログインを行う。
:param request: リクエストオブジェクト
:type request: django.http.HttpRequest
:return: 遷移先を示すレスポンスオブジェクト
:rtype: django.http.HttpResponse
"""
# 認証URLを取得する
oauth_handler = OAuthHandler(
settings.CONSUMER_KEY,
settings.CONSUMER_SECRET,
request.build_absolute_uri(reverse(twitter_callback))
)
authorization_url = oauth_handler.get_authorization_url()
# リクエストトークンをセッションに保存する
request.session['request_token'] = oauth_handler.request_token
# ログイン完了後のリダイレクト先URLをセッションに保存する
if 'next' in request.GET:
request.session['next'] = request.GET['next']
# 認証URLにリダイレクトする
return HttpResponseRedirect(authorization_url)
def authenticate(self): """ Set up the connection """ auth = OAuthHandler(self.consumer_key, self.consumer_secret) auth.set_access_token(self.access_token, self.access_token_secret) return(auth)
def __init__(self):
global API
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
self.twitterStream = Stream(auth, listener())
API = tweepy.API(auth)
verifyDir(IMAGE_DIR)
def stream_twitter(battle_id):
#Avoiding circular import
from battle.models import Battle
battle = Battle.objects.get(id=battle_id)
if battle.end_time < timezone.now():
return
battle.battlehashtags_set.update(typos=0, words=0)
battle_hashtags = battle.battlehashtags_set.all().prefetch_related('hashtag')
if battle_hashtags.count() == 0:
return
hashtag_values = [x.hashtag.value for x in battle_hashtags]
listener = TwitterStreamListener(battle_hashtags)
auth = OAuthHandler(
settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET
)
auth.set_access_token(
settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_ACCESS_TOKEN_SECRET
)
stream = Stream(auth, listener)
delay = battle.end_time - timezone.now()
Timer(delay.total_seconds(), stream.disconnect).start()
stream.filter(track=hashtag_values, languages=['en'])
def run(self):
l = StdOutListener(self)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=self.tags, languages=['en'])
def main():
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
api = tweepy.API(auth)
# specify the main machine's server address
couch = couchdb.Server('http://*****:*****@115.146.84.141:5984/')
try:
database = couch['train']
except:
database = couch.create('train')
cursor = tweepy.Cursor(api.search, geocode="39.091919,-94.5757195,1000km", since="2015-05-03",
until="2015-05-10", lang="en").items()
while True:
try:
tweet = cursor.next()
database.save(tweet)
except tweepy.TweepError:
print 'waiting...'
time.sleep(60*15)
except StopIteration:
break
def __call__(self):
self.items = queue.Queue()
auth = OAuthHandler(mykeys.ckey, mykeys.csecret)
auth.set_access_token(mykeys.atoken, mykeys.asecret)
self.stream = Stream(auth, self)
self.stream.filter(track=self.terms, async=True)
return self
def handle(self, *args, **options):
politicians = Politician.objects.all();
politician_keywords = []
for politician in politicians:
politician_keywords.append(politician.first_name + " " + politician.last_name)
if politician.twitter_url:
indexSlash = politician.twitter_url.rfind("/")
indexQuestionMark = politician.twitter_url.rfind("?")
if indexQuestionMark != -1:
twitter = politician.twitter_url[indexSlash+1:indexQuestionMark]
else:
twitter = politician.twitter_url[indexSlash+1:]
politician_keywords.append(twitter)
# create instance of the tweepy tweet stream listener
listener = TweetStreamListener()
# set twitter keys/tokens
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# create instance of the tweepy stream
stream = Stream(auth, listener)
# search twitter for "congress" keyword
stream.filter(track=politician_keywords)
def __init__(self,slacker):
# auth
auth = OAuthHandler(settings.twitter_consumer_key, settings.twitter_consumer_secret)
auth.set_access_token(settings.twitter_access_token, settings.twitter_access_token_secret)
# out
l = StdOutListener(slacker)
# stream
stream = Stream(auth, l)
print("opening twitter stream")
# if only a certain list
if FILTER_LIST:
api = API(auth)
employees = api.list_members(LIST_USER,LIST)
list = map(lambda val: str(val),employees.ids())
#print(list)
print("only List: "+LIST)
stream.filter(follow=list)
elif FILTER_KEYWORDS:
print("only Keywords: "+str(KEYWORDS))
stream.filter(track=KEYWORDS)
else:
print("your timeline")
stream.userstream()
def main():
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
backend = FileBackend("./test-db")
stream = Stream(auth, l)
stream.filter(track=['トレクル'])
def unwrapped_callback(self, resp):
if resp is None:
raise LoginCallbackError(_("You denied the request to login"))
# Try to read more from the user's Twitter profile
auth = TwitterOAuthHandler(self.consumer_key, self.consumer_secret)
auth.set_access_token(resp['oauth_token'], resp['oauth_token_secret'])
api = TwitterAPI(auth)
try:
twinfo = api.verify_credentials(include_entities='false', skip_status='true', include_email='true')
fullname = twinfo.name
avatar_url = twinfo.profile_image_url_https.replace('_normal.', '_bigger.')
email = getattr(twinfo, 'email', None)
except TweepError:
fullname = None
avatar_url = None
email = None
return {'email': email,
'userid': resp['user_id'],
'username': resp['screen_name'],
'fullname': fullname,
'avatar_url': avatar_url,
'oauth_token': resp['oauth_token'],
'oauth_token_secret': resp['oauth_token_secret'],
'oauth_token_type': None, # Twitter doesn't have token types
}
def _get_twitter_access(self, username):
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
url = auth.get_authorization_url()
print 'Go here: %s and enter the corresponding PIN' % url
pin = raw_input('PIN:')
auth.get_access_token(pin)
return (auth.access_token.key, auth.access_token.secret)
def get_tweets():
access_token,access_secret,consumer_key,consumer_secret = read_config()
auth = OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_secret)
global hashes
count = 0
api = tweepy.API(auth)
hashes = hashes.replace("'","").split(",")
for hashtag in hashes:
tweets = api.search(hashtag)
for tweet in tweets:
#print tweet.text
twitter_json = {}
twitter_json["created_at"] = str(tweet.created_at)
twitter_json["caption"] = tweet.text
twitter_json["username"] = tweet.user.name
twitter_json["thumbs"] = sentiment.check_sentiments(tweet.text)
twitter_json["source"] = "twitter"
twitter_json["link"] = "https://twitter.com/"+str(tweet.user.screen_name)+"/status/"+str(tweet.id)
print twitter_json["link"]
if 'media' in tweet.entities:
twitter_json["url"] = tweet.entities['media'][0]['media_url']
else:
twitter_json["url"] = ""
push_mongo(twitter_json)
def results(request):
global tweets,review,confidence,positive,negative
tweets = []
review = []
confidence = []
positive = []
negative = []
global auth,api
auth = OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_secret)
api = tweepy.API(auth)
ky = 'purplestem'
dys = 1
if 'kywrd' in request.GET and request.GET['kywrd']:
ky = request.GET['kywrd']
if 'days' in request.GET and request.GET['days']:
dys = request.GET['days']
past(ky,dys)
compute()
return render(request,'final.html',{'satisfaction':satisfaction,'pos':positive,'neg':negative})
def stream(buff, terms):
l = StdOutListener(buff)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
stream.filter(track=[terms])
def get_tweets(tweeter_id, from_id = None):
#Setup
#l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = API(auth)
#Get tweets (status) from Twitter A
if from_id == None:
status = api.user_timeline(user_id = tweeter_id, count=amont_of_tweets)
else:
status = api.user_timeline(user_id = tweeter_id, count=amont_of_tweets, max_id = from_id)
status.pop(0) #Remove duplicate first tweet, max_id not included.
tweetlist = []
last_id = 0
#print("Cleaning tweets from :", status.user.screen_name, "id: ", tweeter_id)
for items in status:
tweet = {}
tweet["id"] = items.id
tweet["user_id"] = tweeter_id
tweet["nickname"] = items.user.screen_name
tweet["realname"] = items.user.name
tweet["date"] = datetime.strptime(str(items.created_at), "%Y-%m-%d %H:%M:%S")
tweet["text"] = items.text
tweetlist.append(tweet)
last_id = items.id
print("Last ID: ", last_id, "\n")
return tweetlist, last_id
class TweetPlugin:
''' Catches the "autotweet" command '''
def __init__(self):
''' Initialize twitter '''
config = ConfigParser.ConfigParser()
config.read('.twitter')
consumer_key = config.get('apikey', 'key')
consumer_secret = config.get('apikey', 'secret')
access_token = config.get('token', 'token')
access_token_secret = config.get('token', 'secret')
stream_rule = config.get('app', 'rule')
account_screen_name = config.get('app', 'account_screen_name').lower()
self.account_user_id = config.get('app', 'account_user_id')
self.auth = OAuthHandler(consumer_key, consumer_secret)
self.auth.set_access_token(access_token, access_token_secret)
self.twitterApi = API(self.auth)
def list(self):
pass
def run(self,user,message):
if utilities.getCommand() == "autotweet":
streamListener = ReplyToTweet()
streamListener.setAPI(self.twitterApi)
streamListener.setUser(self.account_user_id)
twitterStream = Stream(self.auth, streamListener)
twitterStream.userstream(_with='user')
def setUp(self):
self.auth = OAuthHandler(apikeys.TWITTER_CONSUMER_KEY, apikeys.TWITTER_CONSUMER_SECRET)
self.auth.set_access_token(apikeys.TWITTER_ACCESS_TOKEN, apikeys.TWITTER_ACCESS_KEY)
self.invalid_auth = OAuthHandler("WRONG_CONSUMER_KEY", "WRONG_CONSUMER_SECRET")
self.invalid_auth.set_access_token("WRONG_ACCESS_TOKEN", "WRONG_ACCESS_KEY")
self.blank_auth = OAuthHandler("", "")
self.blank_auth.set_access_token("", "")
def init_api(self):
oauth_handler = TweepyOAuthHandler(self._consumer_key,
self._consumer_secret,
secure=configuration.twitter['use_https'])
oauth_handler.set_access_token(self._access_token_key,
self._access_token_secret)
self._api = BaseTweepyApi(oauth_handler, secure=configuration.twitter['use_https'])
def main(self):
#twitter authorization
auth = OAuthHandler(AuthDetails.consumer_key, AuthDetails.consumer_secret)
auth.set_access_token(AuthDetails.access_token, AuthDetails.access_token_secret)
language = 'en'
pt = ProcessTweet()
searchTerm = pt.unicodetostring(self.searchTerm)
stopAt = pt.unicodetostring(self.stopAt)
#calls method to train the classfier
tr = Training()
(priors, likelihood) = tr.starttraining()
#stream tweets from twitter
twitterStream = Stream(auth, Listener(searchTerm, stopAt))
twitterStream.filter(track=[searchTerm], languages=[language])
sen = Sentiment()
sentiment_tally = Counter()
(sentiment_tally, tweet_list) = sen.gettweetstoanalyse(priors, likelihood, searchTerm)
tr = Training()
sen = Sentiment()
(neutral, positive, negative) = sen.analyse(sentiment_tally)
tweet_list = self.edittweetlists(tweet_list)
#truncate streamtweets table
self.removetweetsfromdatabase()
#save training data
tr.savetrainingdatatodb(priors, likelihood)
return (neutral, positive, negative, tweet_list)
class TwitterPlayer(player.Player):
def __init__(self, model, code, access_token, access_token_secret, opponent):
player.Player.__init__(self, model, code)
self._opponent = opponent
self._last_id = None
self._auth = OAuthHandler(auth.consumer_key, auth.consumer_secret)
self._auth.set_access_token(access_token, access_token_secret)
self._api = API(self._auth)
self._listener = TwitterListener(self, self._api)
self._stream = Stream(self._auth, self._listener)
@property
def username(self):
return self._auth.get_username()
def allow(self):
print 'This is the opponent\'s turn...'
self._stream.userstream()
def update(self, event):
if event.player == self.code:
return
message = '@%s %s' % (self._opponent, self._model.events[-1][1])
self.tweet(message)
def tweet(self, message):
if self._last_id is None:
self._api.update_status(message)
else:
self._api.update_status(message, self._last_id)
def process(self, statement):
confidence = self.classifier.classify(statement.text.lower())
tokens = nltk.word_tokenize(str(statement))
tagged = nltk.pos_tag(tokens)
nouns = [word for word, pos in tagged if
(pos == 'NN' or pos == 'NNP' or pos =='JJ' or pos == 'NNS' or pos == 'NNPS')]
downcased = [x.lower() for x in nouns]
searchTerm = " ".join(downcased).encode('utf-8')
#"http://where.yahooapis.com/v1/places.q('Place name')?appid=yourappidhere"
st=""
if len(nouns) != 0:
auth = OAuthHandler(twitter_consumer_key, twitter_consumer_secret)
auth.set_access_token(twitter_access_key, twitter_access_secret)
api = tweepy.API(auth)
for status in tweepy.Cursor(api.search, q='#'+searchTerm).items(20):
st = st+status.text
response = Statement("Jarvis: "+st)
else:
response = Statement("Jarvis: "+"Sorry sir, Nothing Found")
return confidence, response
#what's trending in city
#movie reviews
#people talking about some topic
def _request_tweets(self, search_word, since_id):
auth = OAuthHandler(self.consumer_key, self.consumer_secret)
auth.set_access_token(self.access_token, self.access_secret)
tweets = []
api = tweepy.API(auth)
max_id = None
new_since_id = None
total = 0
logger.info("start search by %s" % search_word)
while True:
tweets_batch = api.search(search_word, max_id=max_id, since_id=since_id)
logger.info("get " + str(len(tweets_batch)) + " tweets by '" + search_word + "'")
if not new_since_id: new_since_id = tweets_batch.since_id
if max_id == tweets_batch.max_id: break
max_id = tweets_batch.max_id
total += len(tweets_batch)
for tweet in tweets_batch:
tweets.append(tweet._json)
if not max_id:
break
logger.info("done with search found %s new tweets" % total)
return new_since_id, tweets
class IRCListener(StreamListener):
def __init__(self, config, bot):
self.bot = bot
self.auth = OAuthHandler(config["auth"]["consumer_key"], config["auth"]["consumer_secret"])
self.auth.set_access_token(config["auth"]["access_token"], config["auth"]["access_token_secret"])
api = tweepy.API(self.auth)
stream = Stream(self.auth, self)
self.users = [str(api.get_user(u).id) for u in config["follow"]]
stream.filter(follow=self.users, async=True)
log.debug("a twitter.IRCListener instance created")
def on_data(self, data):
parsed = json.loads(data)
if "text" in parsed and parsed["user"]["id_str"] in self.users:
# TODO: use Twisted color formatting
ourtweeter = parsed["user"]["name"]
ourtweet = parsed["text"]
statusLinkPart = " - https://twitter.com/" + parsed["user"]["screen_name"] + "/status/" + parsed["id_str"]
self.bot.announce(ourtweeter, " tweeted ", ourtweet, statusLinkPart, specialColors=(None, None, attributes.fg.blue, None))
return True
def on_error(self, status):
log.debug("Twitter error: " + str(status))
def run_twitter_query():
l = StdOutListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
#names = list(np.array(get_companies())[:,1])
#print names[num1:num2]
d = hand_made_list()
search_list = []
for key, value in d.items():
if key == 'SPY':
search_list.append(value[0]) # append the full anme of the symbol
search_list.append('#SP500') # Don't append #SPY because it's not helpful
search_list.append('$SP500')
elif key == 'F':
# search_list.append(value[0]) # append the full name of the symbol
search_list.append('Ford') # append the name of the symbol
elif key == 'GE':
search_list.append('General Electric') # append the full anme of the symbol
elif key == 'S':
search_list.append('Sprint') # append the full anme of the symbol
elif key == 'T':
search_list.append('AT&T') # append the full anme of the symbol
elif key == 'MU':
search_list.append('Micron Tech') # append the full anme of the symbol
elif key == 'TRI':
search_list.append('Thomson Reuters') # append the full anme of the symbol
else:
for cell in value:
search_list.append(cell)
stream.filter(track=search_list)
def TwitterStream(kwords, lim, lang=['en'], loca=[-180, -90, 180, 90]):
# print kwords, lang, lim, loca
global limit
if type(lim) != tuple:
l = StdOutListener()
limit = int(lim)
else:
day = int(lim[0])
hour = int(lim[1])
minute = int(lim[2])
second = int(lim[3])
l = StdOutListener_time()
print time.time()
limit = time.time() + 86400 * day + 3600 * \
hour + 60 * minute + 1 * second
print limit
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
global results
results = list()
stream = Stream(auth, l)
# print kwords, lang
stream.filter(track=kwords, languages=['en'])
# def filter(self, follow=None, track=None, async=False, locations=None,
# stall_warnings=False, languages=None, encoding='utf8'):
return results
# Importing libraries
import pandas as pd
import numpy as np
from textblob import TextBlob
from matplotlib import pyplot as plt
import seaborn as sns
from tweepy import Cursor, OAuthHandler
import re
# For using twitter api, below i used essential information of my app.
key = "key"
secret = "secret"
token = "token"
token_secret = "token_secret"
auth = OAuthHandler(key, secret)
auth.set_access_token(token, token_secret)
api = API(auth)
# Adding twitter account names and DataFrame column names
accounts = [
'BBCWorld', 'NHKWORLD_News', 'trtworld', 'cnni', 'foxnews', 'dwnews',
'ajenglish', 'FRANCE24', 'rt_com', 'cgtnofficial'
]
names = [
'bbc', 'nhk', 'trt', 'cnn', 'foxnews', 'dw', 'aj', 'fr24', 'rt', 'cgtn'
]
accounts = accounts + names
tw = pd.DataFrame()
lists = [bbc, nhk, trt, cnn, foxnews, dw, aj, fr24, rt, cgtn]
accounts = accounts + lists
def on_status(self, status):
json_str = json.dumps(status._json)
print(json_str) #printing the Json string that is being fetched...
#appending the JSON response to the end of a file
try:
with open("bitcointweets_test.json", "a") as file:
#data to the file
file.write(json_str + ",")
except:
pass
return True
def on_error(self, status_code):
print("Exception has occurred, here is the access code: ", status_code)
if status_code == 420:
#returning False in on_error disconnects the stream
return False
if __name__ == "__main__":
print("I am here")
listener = Listen() #listener object of Listener class
print(OAuthHandler)
authentication = OAuthHandler(consumer_token, consumer_token_secret)
authentication.set_access_token(access_token, access_token_secret)
stream = Stream(authentication,
listener) #stream object with authentication
#use follow in order to track an ID
list_of_search_terms = ['bitcoin', 'cryptocurrencies', 'Ethereum']
stream.filter(track=list_of_search_terms)
def main():
couch = couchdb.Server('http://115.146.89.12:5984/')
try:
db = couch.create('tweets')
except couchdb.http.PreconditionFailed as e:
db = couch['tweets']
client25 = testTweepy.clientKeys(
"2723420030-7Rcvp9nXIIPGWB8IsOCMkts4YHcXPom36FCtT9L",
"gjg5RFYRHJrU6aEMT8MvKzCTORgyc0iYrfYB0KwdtpqJB",
"b3mRv2HYjPoekuE0AjHVhxHFg",
"zY42dCT8c2PhWj4qBE7ZKTbVIIEGTRfvyRXsAZ4ZQYo6O1s673", 25)
api25 = client25.OAuth()
client26 = testTweepy.clientKeys(
"2723420030-WSRBT6DX4y8yNf2FnKPQxgomXcirYscQQf7E2DF",
"073SmrI7FQ3NKZcRUfyOYkRBbM3Y9kh39nvkQYaBuEasq",
"dIZDF4GTsaua8q00hky8S9w96",
"w2NX30Gg6yPPtwR8GerXCkDag8MewY6GqLWHQSEqxmw59EXyjn", 26)
api26 = client26.OAuth()
client27 = testTweepy.clientKeys(
"2723420030-dbLrRNNcPdDVVN9vU6gNe6ezmJIjBiFSTA4tZIK",
"iQDLE1UeBMsO1AAH12jxHz4eYscKbqYBCOzYPz4v8Wrm6",
"b9lVAy5VBbePQPsqxQvCW6bbn",
"C8ZMEtz6BOF8kNkotmLmJULvgyZW50r7FirDn5AJkpyjtHi61a", 27)
api27 = client27.OAuth()
client28 = testTweepy.clientKeys(
"727698246571724800-pTS7vQrpXzkcZy2Xa252bA91V9eyAuX",
"Auw6gec17e93VNaT3RZylhEJcpZS6U2yoTRR8JgNet8Kg",
"wkVCCl1aXTrDXHiAqqUW5rSKs",
"heZKoWPP4dbCUf905gnx6pSrXugPYLOYuCLHTUtB8rMVy6SFgm", 28)
api28 = client28.OAuth()
client29 = testTweepy.clientKeys(
"727698246571724800-nGxWtkrotWU82YaPjLqxqT8fYfgFq8H",
"9IOjOsWzfzsSG9FiTsIRCmt949bwnygAI4x5m1ne8PCar",
"HafWlU6lhWNSnPBZ6LOzQTTPk",
"cLwesptc8NELKP9vOGIjlN8j33i7EZkBmE7nJRwNOHdIck0sRg", 29)
api29 = client29.OAuth()
client30 = testTweepy.clientKeys(
"727698246571724800-yaJtodv2gaAFzw5DNZSebKHYwD5RTBB",
"R8oqgADtOIOHhxfVtAfQIH47X4lJf9Txc1XIY093OxzwQ",
"RbR0qPH7aB3OPY5c1MxX1ydwV",
"eEL4AUFYfJb2esMCW32f228ycdvOd9lujKUBKWJyFLXF5tmFbE", 24)
api30 = client30.OAuth()
#read CSV File
list1 = []
with open('list7.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
list1.append(row)
list2 = []
with open('list8.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
list2.append(row)
userTimelineCompany = []
with open('smallData.csv', 'rb') as f:
reader = csv.reader(f)
for row in reader:
userTimelineCompany.append(row)
#step1:comany post, new thread
collectUserTimeline_thread = testTweepy.getUserTimeline(
db, api25, userTimelineCompany, "screenName")
collectUserTimeline_thread.start()
print "company user timelime start"
#step2:find all followers
followersIds1 = getFollowersIds(api26, list1)
followersIds2 = getFollowersIds(api26, list2)
#break point
#step3:followers post,new thread
collectFollowersTimeline_thread1 = testTweepy.getUserTimeline(
db, api27, followersIds1, "user_id")
collectFollowersTimeline_thread1.start()
collectFollowersTimeline_thread2 = testTweepy.getUserTimeline(
db, api28, followersIds2, "user_id")
collectFollowersTimeline_thread2.start()
print "followers timelime start"
#step4:search, new thread
search_thread1 = testTweepy.search(db, api29, list1)
search_thread1.start()
search_thread2 = testTweepy.search(db, api30, list2)
search_thread2.start()
print "search start"
#step5:streaming,realtime data
#This handles Twitter authetification and the connection to Twitter Streaming API
# print "search ok~"
#[[c1,a1],[c2,a2]]
listOfCompany = list1
listOfCompany.extend(list2)
nameOfCompany = []
tmp = []
#[c1,a1,c2,a2]
streamFilter = []
for each in listOfCompany:
tmp.append(each[0])
tmp.append(each[2])
nameOfCompany.append(tmp)
streamFilter.append(each[0])
streamFilter.append(each[2])
tmp = []
l = StdOutListener(db, nameOfCompany)
auth = OAuthHandler('8FWIMWkMWaLKtbSyUkDcsAO8Z',
'JTPVp6izcJ73PSL4fbTaVWSFGFiJM4m0TZj1J5gSKtiuL6vaCA')
auth.set_access_token('2723420030-WFrcjeXsVIV1AHgWZ1YhTVb7RCxmftZQB8D8IOW',
'EvAUoawFAdVPEfTjiRZY7xxAuyqNuLv4WVI86Eh797HUK')
stream = Stream(auth, l)
print "streaming start"
# #This line filter Twitter Streams to capture data by the keywords: 'python', 'javascript', 'ruby'
stream.filter(track=streamFilter, languages=["en"])
print "main over"
import requests as req
from bs4 import BeautifulSoup
from lxml import html
import scrapy
import re
new_tweet = "========New_News=============================="
from tweepy import OAuthHandler
consumer_key = 'nhNFDac7wxpIdUmJjMgrKKB68'
consumer_secret = '6f1jQpHTK52U8PP9JHB15zaSx6kQZn6zMBBNr6Csc883SBTRtI'
access_token = '1013780071499223045-FYFso89tQ6GXUouRIjZ4RUTP6xnmVA'
access_secret = 'UW4waSBwOZS16Q2zLRyRoXGsRxHbEj0FgO5RghHl5oc6Y'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth, wait_on_rate_limit=True)
#twitter_stream = Stream(auth, MyListener())
listOfLinks = []
clean_display = ""
for i,status in enumerate(tweepy.Cursor(api.home_timeline).items(7)):
try:
listOfLinks.append(re.search("(?P<url>https?://[^\s]+)", status.text).group("url"))
except AttributeError:
print("A link does not exist")
print "sp_collector_protocol:", args.sp_collector_protocol
print "sp_collector_port:", args.sp_collector_port
print "sp_collector_method:", args.sp_collector_method
print "sp_app_id:", args.sp_app_id
print "--"
print "Tracker setup..."
global emiter
global tracker
emiter = Emitter(buffer_size=50,
endpoint=args.sp_collector_uri,
protocol=args.sp_collector_protocol,
port=int(args.sp_collector_port),
method=args.sp_collector_method)
tracker = Tracker(emitters=emiter,
namespace="cf",
app_id=args.sp_app_id,
encode_base64=True)
print "Starting listening to Twitter..."
l = StdOutListener()
auth = OAuthHandler(args.consumer_key, args.consumer_secret)
auth.set_access_token(args.access_token, args.access_token_secret)
stream = Stream(auth, l)
follow_arr = args.follow.split(",")
print "follow_arr:", follow_arr
stream.filter(follow=follow_arr)
def authenticateTwitterApi():
auth = OAuthHandler(credential.API_KEY, credential.API_SECRET_KEY)
auth.set_access_token(credential.ACCESS_TOKEN,
credential.ACCESS_TOKEN_SECRET)
return auth
from dotenv import load_dotenv
load_dotenv()
# TWITTER_USERS = ['calebhicks', 'elonmusk', 'rrherr', 'SteveMartinToGo',
# 'alyankovic', 'nasa', 'sadserver', 'jkhowland', 'austen',
# 'common_squirrel', 'KenJennings', 'conanobrien',
# 'big_ben_clock', 'IAM_SHAKESPEARE']
TWITTER_API_KEY = os.getenv('TWITTER_API_KEY')
TWITTER_API_SECRET = os.getenv('TWITTER_API_SECRET')
TWITTER_ACCESS_TOKEN = os.getenv('TWITTER_ACCESS_TOKEN')
TWITTER_ACCESS_TOKEN_SECRET = os.getenv('TWITTER_ACCESS_TOKEN_SECRET')
auth = OAuthHandler(TWITTER_API_KEY, TWITTER_API_SECRET)
auth.set_access_token(TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET)
api = API(auth)
print("API CLIENT", api)
if __name__ == "__main__":
user = api.get_user('elonmusk')
print("TWITTER USER:", type(user))
print(user.id)
print(user.screen_name)
print(user.name)
tweets = api.user_timeline('elonmusk', tweet_mode='extended')
print(tweet)
print(fetch_location)
print(tweet_time)
conn.execute(
'INSERT INTO tweet (time, username, tweet_data, location, hashtag) VALUES (%s,%s,%s,%s,%s)',
(str(tweet_time), name, tweet, str(fetch_location),
str(hashtags)))
connection.commit()
producer.send_messages("stream", data.encode('utf-8'))
print(data)
return True
return True
except Exception as e:
print(e)
return True
return False
def on_error(self, status):
print(status)
kafka = KafkaClient("localhost:9092")
producer = SimpleProducer(kafka)
listen = kafka_listener()
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
stream = Stream(auth, listen)
stream.filter(track="covid19")
class TwitterTipBot():
auth = None
api = None
stream = None
twitter_uid = "1190625904717959169"
bot_twitter_handle = "prarysoft"
dataStore = None
explorer_url = 'https://explorer.pops.one/#' #'https://explorer.harmony.one/#'
def __init__(self):
try:
self.auth = OAuthHandler(Secretes._twitterConsumerApiKey,
Secretes._twitterConsumerApiSecret)
self.auth.secure = True
self.auth.set_access_token(Secretes._twitterAccessToken,
Secretes._twitterAccessTokenSecret)
self.api = API(self.auth)
self.dataStore = DataStore()
except BaseException as e:
print("Error in main()", e)
self.tweetCount = 0
def startTwitterTipBot(self):
while 1:
twitter_event_details = self.dataStore.getNotAddressedTwitterEvents(
)
if twitter_event_details != None:
try:
text = twitter_event_details['event_text']
if f'@{self.bot_twitter_handle} !tip' in text:
self.process_tip(
twitter_event_details['event_id'], text,
twitter_event_details['sender_handle'],
twitter_event_details['receiver_handle'])
elif text == '!history':
self.history(twitter_event_details['sender_handle'],
twitter_event_details['sender_id'])
elif text == '!help':
self.help(twitter_event_details['sender_id'])
elif text == '!balance':
self.balance(twitter_event_details['sender_handle'],
twitter_event_details['sender_id'])
elif text.startswith("!withdraw"):
self.withdraw(text,
twitter_event_details['sender_handle'],
twitter_event_details['sender_id'])
elif text.startswith("!deposit"):
self.deposit(twitter_event_details['sender_handle'],
twitter_event_details['sender_id'])
except Exception as ex:
print(ex)
finally:
if 'event_id' in twitter_event_details:
twitter_event_details['addressed'] = True
self.dataStore.saveTwitterEventDetails(
twitter_event_details)
sleep(10)
# When someone wants to deposit one to his account
def deposit(self, sender_handle, sender_id):
try:
user_details = self.dataStore.getUserDetailsByTwitterHandle(
f'@{sender_handle}')
if user_details != None:
one_address = user_details['one_address']
try:
img = self.api.media_upload(
Utility.getQRCodeImageFilePath(one_address))
self.api.send_direct_message(
text=f'Your Deposit address {one_address}',
recipient_id=sender_id,
attachment_type='media',
attachment_media_id=img.media_id)
except Exception as e:
self.api.send_direct_message(
text=f'Your Deposit address {one_address}',
recipient_id=sender_id)
print(e)
else:
self.api.send_direct_message(
text=
"You\'re not registered!, please register to deposit ONE, please register using Telegram bot (https://t.me/onetipbot)",
recipient_id=sender_id)
except Exception as ex:
print(ex)
def withdraw(self, text, sender_handle, sender_id):
parts = text.split(" ")
withdraw_address = ""
amount = 0
inputInValid = False
reply_message = ""
user_details = self.dataStore.getUserDetailsByTwitterHandle(
f'@{sender_handle}')
from_address = user_details['one_address']
if len(parts) >= 3:
if parts[0] == "!withdraw":
print(text)
try:
amount = float(parts[1])
except Exception as ex:
inputInValid = True
reply_message = "Invalid withdrawal amount."
print(ex)
if not inputInValid:
if amount < 0.00000000:
reply_message = f"Withdraw amount cannot be negative value."
print(reply_message)
inputInValid = True
else:
if HmyClient.getBalace(
from_address) + 0.00000021 < float(amount):
inputInValid = True
reply_message = f"Please make sure you have enough funds for transfer {amount} in your account including fees, please reduce the withdraw amount and try again."
print(reply_message)
if not inputInValid:
withdraw_address = parts[2]
if not HmyClient.validateONEAdress(withdraw_address):
inputInValid = True
reply_message = "Invalid ONE address, transfer cancelled"
print(reply_message)
if not inputInValid:
res = HmyClient.transfer(from_address, withdraw_address, amount)
res = eval(res)
if 'transaction-hash' in res:
reply_message = f"Withdraw is complete, here is the receipt {self.explorer_url}/tx/{res['transaction-hash']}"
else:
reply_message = "Withdraw is failed with unknown error"
else:
if reply_message == "":
reply_message = "Unknown Error!"
print(f'Withdraw reply :{reply_message}')
self.api.send_direct_message(text=reply_message,
recipient_id=sender_id)
# When someone wants to deposit one to his account
def balance(self, sender_handle, sender_id):
try:
# If they're not registered nor have they received any tips without an account
user_details = self.dataStore.getUserDetailsByTwitterHandle(
f'@{sender_handle}')
if user_details != None:
one_address = user_details['one_address']
balance = HmyClient.getBalace(one_address)
self.api.send_direct_message(
text=f'Your Wallet Balace \n{balance}',
recipient_id=sender_id)
else:
self.api.send_direct_message(
text=
f'You\'re not registered!, please register to get balance please register using Telegram bot (https://t.me/onetipbot), if you are already registered please link you twitter handle.',
recipient_id=sender_id)
# Save the data
except Exception as ex:
print(ex)
# When someone wants to see the History of his account
def history(self, sender_handle, sender_id):
try:
# If they're not registered nor have they received any tips without an account
user_details = self.dataStore.getUserDetailsByTwitterHandle(
f'@{sender_handle}')
if user_details != None:
one_address = user_details['one_address']
self.api.send_direct_message(
text=
f'Account History\n{self.explorer_url}/address/{one_address}',
recipient_id=sender_id)
else:
self.api.send_direct_message(
text=
f'You\'re not registered!, please register to get Account History please register using Telegram bot (https://t.me/onetipbot)',
recipient_id=sender_id)
except Exception as ex:
print(ex)
def help(self, sender_id):
try:
help_text = u"Deposit \n----------------\n\nTo get started using Telegram bot (https://t.me/onetipbot) you need to deposit funds to your address. DM \" !deposite\" to to find your deposit address.\n\n\nWithdraw\n----------------\n\nTo Withdraw funds from your @OneTipBot you need to send \"!withdraw <amount> <one_address>\" Make sure you have enough balance to cover the network fees and your withdrawal amount.\n\n\nTip\n-----------------\n\nYou can tip anyone by Tweeting @OneTipBot !tip <tip_amount> <receivers_handle> \n\n\nRegister/Update Twitter handle\n-----------------\n\n\nTo register or update your Twitter handle you need to user our Telegram bot @OneTipBot at (https://t.me/onetipbot) then click on \"Register twitter handle/update twitter handle\". Follow the prompts and you will be able to register/update your twitter handle.\n\n\nDisclaimer\n-----------------\n\nPrivate keys are managed by @OneTipBot and securely stored. The bot uses the private key to create transactions on your behalf via telegram bot. It is not recommended to store large quantities of your crypto on @OneTipBot."
self.api.send_direct_message(text=help_text,
recipient_id=sender_id)
except Exception as ex:
print(ex)
def process_tip(self, tweet_id, text, sender_handle, receiver):
tip = 0
success = "yes"
failure_reason = ""
tweet_type = "tweet"
reply_text = ""
print(text)
print(f'@{self.bot_twitter_handle} !tip')
if f'@{self.bot_twitter_handle} !tip' in text:
if not self.dataStore.checkIftweetDataExists(tweet_id):
sender_details = self.dataStore.getUserDetailsByTwitterHandle(
f'@{sender_handle}')
if sender_details != None:
parts = text.split(" ")
for i in range(0, len(parts)):
if parts[i] == "!tip":
if i + 1 < len(parts):
tip = float(parts[i + 1])
break
from_address = sender_details['one_address']
sender = sender_handle
if receiver != "":
# Can't tip yourself
if sender != receiver:
# Can't tip more than you have
from_balance = HmyClient.getBalace(from_address)
if tip + 0.00000021 > from_balance:
reply_text = f'@{sender_handle}, your balance is low to tip {tip} ONE'
else:
receiver_details = self.dataStore.getUserDetailsByTwitterHandle(
f'@{receiver}')
if receiver_details == None:
reply_text = f"@{sender_handle}, @{receiver} is not registered with ONE Tipping bot, please register using Telegram bot (https://t.me/onetipbot)"
else:
if 'one_address' in receiver_details:
res = HmyClient.transfer(
from_address,
receiver_details['one_address'],
tip)
res = eval(res)
if 'transaction-hash' in res:
reply_text = f"Hi @{receiver}, @{sender_handle} just tipped you {tip} ONE"
else:
print(
f"Tip failed from {sender} to {receiver} tip {tip} ONE"
)
else:
print(
'Receiver ONE address is missing')
else:
reply_text = f'@{sender_handle} You can\'t tip yourself!'
else:
reply_text = f'@{sender_handle} Please mention a receiver to tip.'
else:
success = "no"
failure_reason = "account does not exists"
reply_text = f'@{sender_handle} You are not registered with ONE Tipping bot, please register using Telegram bot (https://t.me/onetipbot).'
if reply_text != "":
self.api.update_status(status=reply_text,
in_reply_to_status_id=tweet_id)
print(reply_text)
tweetDetails = {
'tweet_id': tweet_id,
'sender': sender_handle,
'receiver': receiver,
'tip': tip,
'text': text,
'replied': "yes",
'success': success,
'failure_reason': failure_reason,
'tweet_type': tweet_type,
'reply_text': reply_text
}
self.dataStore.saveTweetDetails(tweetDetails)
else:
print("Tweet already served")
else:
print("Not a Tipping tweet")
class TwitterClient(object):
def __init__(self):
# keys and tokens from the Twitter Dev Console
consumer_key = '.................'
consumer_secret = '.......................'
access_token = '......................'
access_token_secret = '....................'
# attempt authentication
try:
# create OAuthHandler object
self.auth = OAuthHandler(consumer_key, consumer_secret)
# set access token and secret
self.auth.set_access_token(access_token, access_token_secret)
# create tweepy API object to fetch tweets
self.api = tweepy.API(self.auth)
print('auth. success')
except:
print("Error: Authentication Failed")
def clean_tweet(self, tweet):
'''
Utility function to clean tweet text by removing links, special characters
using simple regex statements.
'''
return ' '.join(
re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t]) |(\w+:\/\/\S+)", " ",
tweet).split())
def get_tweet_sentiment(self, tweet):
'''
Utility function to classify sentiment of passed tweet
using textblob's sentiment method
'''
# create TextBlob object of passed tweet text
analysis = TextBlob(self.clean_tweet(tweet))
# set sentiment
if analysis.sentiment.polarity > 0:
return 'positive'
elif analysis.sentiment.polarity == 0:
return 'neutral'
else:
return 'negative'
def get_tweets(self, query, count=10):
'''
Main function to fetch tweets and parse them.
'''
# empty list to store parsed tweets
tweets = []
try:
# call twitter api to fetch tweets
fetched_tweets = self.api.search(q=query, count=count)
# parsing tweets one by one
for tweet in fetched_tweets:
# empty dictionary to store required params of a tweet
parsed_tweet = {}
# saving text of tweet
parsed_tweet['text'] = tweet.text
# saving sentiment of tweet
parsed_tweet['sentiment'] = self.get_tweet_sentiment(
tweet.text)
# appending parsed tweet to tweets list
if tweet.retweet_count > 0:
# if tweet has retweets, ensure that it is appended only once
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
else:
tweets.append(parsed_tweet)
# return parsed tweets
return tweets
except tweepy.TweepError as e:
# print error (if any)
print("Error : " + str(e))
con_secret = ""
con_secret_key = ""
# RethinkDB
r.connect('localhost', 28015).repl()
try:
r.db_create('twitter').run()
r.db('twitter').table_create('tweet').run()
except ReqlRuntimeError:
pass
class MyStreamListener(StreamListener):
def on_data(self, raw_data):
json_data = json.loads(raw_data)
if 'text' in json_data:
print(json.dumps(json_data))
tweet_id = json_data["id"]
r.db('twitter').table('tweet').insert(json_data).run()
return True
return False
def on_error(self, status_code):
print(status_code)
auth = OAuthHandler(con_secret, con_secret_key)
auth.set_access_token(token, token_key)
twitterStream = Stream(auth, MyStreamListener())
twitterStream.filter(locations=[139.60, 35.60, 139.90, 35.80])
def on_data(self, data):
dictTweet = json.loads(data)
try:
dictTweet["_id"] = str(dictTweet['id'])
doc = db.save(dictTweet)
print ("SAVED" + str(doc) +"=>" + str(data))
except:
print ("Already exists")
pass
return True
def on_error(self, status):
print (status)
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
'''========couchdb'=========='''
server = couchdb.Server('http://*****:*****@localhost:5984/') #('http://115.146.93.184:5984/')
try:
db = server.create('cotopaxigl')
except:
db = server('cotopaxigl')
'''===============LOCATIONS=============='''
#Filtro por geolocalización
twitterStream.filter(locations=[-79.3393,-1.22,-78.3809,-0.3308])
#Filtro por palabras
class TwitterClient(object):
'''
Generic Twitter Class for sentiment analysis.
'''
def __init__(self):
'''
Class constructor or initialization method.
'''
# keys and tokens from the Twitter Dev Console
consumer_key = 'tDHKXx4j2wXlm1j4y97FpTFPn'
consumer_secret = 'aL92keHuYjydc2K8oXAPfT1cwO4Elr3upLIecYNTfdVEjx59hH'
access_token = '1660471136-gTl1XwgI1yVTBCQwhQBAknEhqmeEHKjhXaJyutR'
access_token_secret = 'iaKFeL2OtS15ddTjFurWeVpUrligzJHJ9PgYqfBkIDAw2'
# attempt authentication
try:
# create OAuthHandler object
self.auth = OAuthHandler(consumer_key, consumer_secret)
# set access token and secret
self.auth.set_access_token(access_token, access_token_secret)
# create tweepy API object to fetch tweets
self.api = tweepy.API(self.auth)
except:
print("Error: Authentication Failed")
def clean_tweet(self, tweet):
'''
Utility function to clean tweet text by removing links, special characters
using simple regex statements.
'''
return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\ / \ / \S+)", " ", tweet).split())
def get_tweet_sentiment(self, tweet):
'''
Utility function to classify sentiment of passed tweet
using textblob's sentiment method
'''
# create TextBlob object of passed tweet text
analysis = TextBlob(self.clean_tweet(tweet))
# set sentiment
if analysis.sentiment.polarity > 0:
return 'positive'
elif analysis.sentiment.polarity == 0:
return 'neutral'
else:
return 'negative'
def get_tweets(self, query, count=10):
'''
Main function to fetch tweets and parse them.
'''
# empty list to store parsed tweets
tweets = []
try:
# call twitter api to fetch tweets
fetched_tweets = self.api.search(q=query, count=count)
# parsing tweets one by one
for tweet in fetched_tweets:
# empty dictionary to store required params of a tweet
parsed_tweet = {}
# saving text of tweet
parsed_tweet['text'] = tweet.text
# saving sentiment of tweet
parsed_tweet['sentiment'] = self.get_tweet_sentiment(tweet.text)
# appending parsed tweet to tweets list
if tweet.retweet_count > 0:
# if tweet has retweets, ensure that it is appended only once
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
else:
tweets.append(parsed_tweet)
# return parsed tweets
return tweets
except tweepy.TweepError as e:
# print error (if any)
print("Error : " + str(e))
def sendData(c_socket):
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
twitter_stream = Stream(auth, TweetListener(c_socket))
twitter_stream.filter(track=['covid'])
def authenticate_twitter_app(self):
auth = OAuthHandler(twitter_credentials.CONSUMER_KEY,
twitter_credentials.CONSUMER_SECRET)
auth.set_access_token(twitter_credentials.ACCESS_TOKEN,
twitter_credentials.ACCESS_TOKEN_SECRET)
return auth
saveFile = io.open('raw_tweets.json', 'w', encoding='utf-8')
saveFile.write(u'[\n')
saveFile.write(','.join(self.tweet_data))
saveFile.write(u'\n]')
saveFile.close()
exit()
def on_error(self, status):
print status
def on_disconnect(self, notice):
print 'bye'
#Beginning of the specific code
start_time = time.time() #grabs the system time
keyword_list = ['emoji'] #track list
auth = OAuthHandler(ckey, consumer_secret) #OAuth object
auth.set_access_token(access_token_key, access_token_secret)
twitterStream = Stream(auth, listener(start_time, time_limit=200)) #initialize Stream object with a time out limit
twitterStream.filter(track=keyword_list, languages=['en']) #call the filter method to run the Stream Listener
next = random.choice(next_candidates)
prev = current[m - n]
if n - 1 - m == 0:
current = (*next, )
else:
current = current[m + 1 - n:] + (*next, )
for c in current[-m:]:
result.append(c)
if current[-1] in ['.', '!', '?'] and len(result) > 5:
return ' '.join(result)
#load twitter keys
twitter_keys = p.load(open('tk.pkl', 'rb'))
auth = OAuthHandler(twitter_keys['consumer_key'],
twitter_keys['consumer_secret'])
auth.set_access_token(twitter_keys['access_token'],
twitter_keys['access_secret'])
api = tweepy.API(auth)
#grab the time
today = str(datetime.now(timezone('US/Eastern')))[:10]
del twitter_keys, auth
while True:
#At midnight start a new day of tweeting
if today != str(datetime.now(timezone('US/Eastern')))[:10]:
#Grab Trumps tweets and store them in a json dictionary
class TwitterClient(object):
'''
Generic Twitter Class for the App
'''
def __init__(self, query, retweets_only=False, with_sentiment=False):
# keys and tokens from the Twitter Dev Console
#consumer_key = os.environ['CONSUMER_KEY']
#consumer_secret = os.environ['CONSUMER_SECRET']
#access_token = os.environ['ACCESS_TOKEN']
#access_token_secret = os.environ['ACCESS_TOKEN_SECRET']
consumer_key = credvar.CONSUMER_KEY
consumer_secret = credvar.CONSUMER_SECRET
access_token = credvar.ACCESS_TOKEN
access_token_secret = credvar.ACCESS_TOKEN_SECRET
# Attempt authentication
try:
self.auth = OAuthHandler(consumer_key, consumer_secret)
self.auth.set_access_token(access_token, access_token_secret)
self.query = query
self.retweets_only = retweets_only
self.with_sentiment = with_sentiment
self.api = tweepy.API(self.auth)
self.tweet_count_max = 8000
#self.startDate = datetime.datetime(2006, 1, 1, 0, 0, 0)
#self.endDate = datetime.datetime(2019, 30, 9, 0, 0, 0)
#startDate = datetime.datetime(2006, 1, 1, 0, 0, 0)
#endDate = datetime.datetime(2019, 9, 30, 0, 0, 0)
except:
print("Error: Authentication Failed")
# startDate = datetime.datetime(2006, 1, 1, 0, 0, 0)
# endDate = datetime.datetime(2019, 30, 9, 0, 0, 0)
# def set_styr(self, st=''):
# #print("hello")
# sy = int(st)
# #self.startDate = datetime.datetime(sy, 1, 1, 0, 0, 0)
# global startDate
# startDate = datetime.datetime(sy, 1, 1, 0, 0, 0)
# #print(startDate)
def set_yer(self, dt='', mn='', yer=''):
d = int(dt)
m = int(mn)
y = int(yer)
#self.endDate = datetime.datetime(ey, 9, 30, 0, 0, 0)
global startDate
startDate = datetime.datetime(y, m, d, 0, 0, 0)
global endDate
endDate = datetime.datetime(y, m, d, 23, 59, 0)
#print("hai")
#print(endDate)
def set_query(self, query=''):
self.query = query
def set_retweet_checking(self, retweets_only='false'):
self.retweets_only = retweets_only
def set_with_sentiment(self, with_sentiment='false'):
self.with_sentiment = with_sentiment
def clean_tweet(self, tweet):
return ' '.join(
re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ",
tweet).split())
def get_tweet_sentiment(self, tweet):
analysis = TextBlob(self.clean_tweet(tweet))
if analysis.sentiment.polarity > 0:
return 'positive'
elif analysis.sentiment.polarity == 0:
return 'neutral'
else:
return 'negative'
def get_tweets(self):
tweets = []
try:
recd_tweets = self.api.user_timeline(screen_name=self.query,
count=self.tweet_count_max)
if not recd_tweets:
pass
for tweet in recd_tweets:
parsed_tweet = {}
# if tweet.created_at < self.endDate and tweet.created_at > self.startDate:
if tweet.created_at < endDate and tweet.created_at > startDate:
#print(startDate)
#print(endDate)
parsed_tweet['text'] = tweet.text
parsed_tweet['user'] = tweet.user.screen_name
if self.with_sentiment == 1:
parsed_tweet['sentiment'] = self.get_tweet_sentiment(
tweet.text)
else:
parsed_tweet['sentiment'] = 'unavailable'
if tweet.retweet_count > 0 and self.retweets_only == 1:
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
elif not self.retweets_only:
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
return tweets
except tweepy.TweepError as e:
print("Error : " + str(e))
_, _inputs_, _, keep_prob, _, _, _, _, _, saver, predictions = \
LSTM(n_words, graph, lstm_size, lstm_layers, batch_size, learning_rate).get()
c_key, c_secret, acc_token, acc_secret = Setup().get()
kyword = str(input('Enter keyword (eg: Trump)\n>> '))
class Listener(StreamListener):
def on_data(self, data):
all_data = json.loads(data)
tweet = all_data["text"]
print(tweet)
features, _ = Process().process_reviews(tweet)
for i in features:
i = np.reshape(i, (1, len(i)))
with tf.Session(graph=graph) as sess:
saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))
feed = {_inputs_: i, keep_prob: 1}
pred = sess.run([predictions], feed_dict=feed)
print(pred[0][0][0])
with open(os.path.dirname(os.path.realpath(path)) + 'points/'+kyword+'-plot.txt', 'a') as file:
file.write(str(pred[0][0][0]) + '\n')
return True
def on_error(self, status):
print(status)
auth = OAuthHandler(c_key, c_secret)
auth.set_access_token(acc_token, acc_secret)
twitterStream = Stream(auth, Listener())
twitterStream.filter(track=[kyword])
def getStatistics(self):
import time
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import json
from textblob import TextBlob
import matplotlib.pyplot as plt
import re
import test
"# -- coding: utf-8 --"
def calctime(a):
return time.time() - a
positive = 0
negative = 0
compound = 0
count = 0
initime = time.time()
plt.ion()
consumer_key = "rbjc5Bc9L5cO7Xtq9IN3oHekg"
consumer_secret = "njTD2N1ANx9piP4PUjwCgRhmwihLUzTH46jUykJZKnkmiKmut9"
access_token = "4866833991-iA61aJuWNRpxN7PkGNCOL9zp6vvQ1VqTQJbuZEf"
access_secret = "jfiU8S5kNb7hgaecwO9ZP0bJobT6RoXyI6HbMXz3upkQ6"
class listener(StreamListener):
def on_data(self, data):
global initime
t = int(calctime(initime))
all_data = json.loads(data)
tweet = all_data["text"].encode("utf-8")
# username=all_data["user"]["screen_name"]
tweet = " ".join(
re.findall(
b"[a-zA-Z].decode('utf-8', 'backslashreplace')+",
tweet))
blob = TextBlob(tweet.strip())
global positive
global negative
global compound
global count
count = count + 1
senti = 0
for sen in blob.sentences:
senti = senti + sen.sentiment.polarity
if sen.sentiment.polarity >= 0:
positive = positive + sen.sentiment.polarity
else:
negative = negative + sen.sentiment.polarity
compound = compound + senti
print(count)
print(tweet.strip())
print(senti)
print(t)
print(
str(positive) + ' ' + str(negative) + ' ' + str(compound))
plt.axis([0, 70, -20, 20])
plt.xlabel('Time')
plt.ylabel('Sentiment')
plt.plot([t], [positive], 'go', [t], [negative], 'ro', [t],
[compound], 'bo')
plt.pause(0.0001)
plt.show()
if count == 200:
return False
else:
return True
def on_error(self, status):
print(status)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
twitterStream = Stream(auth, listener(count))
twitterStream.filter(track=["Donald trump"])
class TwitterClient(object):
'''
Generic Twitter Class for sentiment analysis.
'''
def __init__(self):
'''
Class constructor or initialization method.
'''
# keys and tokens from the Twitter Dev Console
consumer_key = 'a6WWzOxrE2sdWf2KEhqV6NvEb'
consumer_secret = 'O3195fFCqHz3aRVNSi99ugaPMvVPTEOO4xJ9kSenVz87QwHwBA'
access_token = '622747161-Z2yLC31uBPh7ZBvWJAX9VRQoYnM5SA4bPshXzljx'
access_token_secret = 'Ghq2z4VjPGHUqJJO7JHfhSqK9LR72pFORFxXbEEgKN7rA'
# attempt authentication
try:
# create OAuthHandler object
self.auth = OAuthHandler(consumer_key, consumer_secret)
# set access token and secret
self.auth.set_access_token(access_token, access_token_secret)
# create tweepy API object to fetch tweets
self.api = tweepy.API(self.auth)
except:
print("Error: Authentication Failed")
def clean_tweet(self, tweet):
'''
Utility function to clean tweet text by removing links, special characters
using simple regex statements.
'''
return ' '.join(
re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])| (\w+:\ / \ / \S+)", " ",
tweet).split())
def get_tweet_sentiment(self, tweet):
'''
Utility function to classify sentiment of passed tweet
using textblob's sentiment method
'''
# create TextBlob object of passed tweet text
analysis = TextBlob(self.clean_tweet(tweet))
# set sentiment
if analysis.sentiment.polarity > 0:
return 'positive'
elif analysis.sentiment.polarity == 0:
return 'neutral'
else:
return 'negative'
def get_tweets(self, query, count=10):
'''
Main function to fetch tweets and parse them.
'''
# empty list to store parsed tweets
tweets = []
try:
# call twitter api to fetch tweets
fetched_tweets = self.api.search(q=query, count=count)
# parsing tweets one by one
for tweet in fetched_tweets:
# empty dictionary to store required params of a tweet
parsed_tweet = {}
# saving text of tweet
parsed_tweet['text'] = tweet.text
# saving sentiment of tweet
parsed_tweet['sentiment'] = self.get_tweet_sentiment(
tweet.text)
# appending parsed tweet to tweets list
if tweet.retweet_count > 0:
# if tweet has retweets, ensure that it is appended only once
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
else:
tweets.append(parsed_tweet)
# return parsed tweets
return tweets
except tweepy.TweepError as e:
# print error (if any)
print("Error : " + str(e))
class TwitterClient():
'''
Generic Twitter Class for the App
'''
def __init__(self, query, retweets_only=False, with_sentiment=False):
# keys and tokens from the Twitter Dev Console
consumer_key = 'GnoSX5gtHMBpx0w8WU2GngWR0'
consumer_secret = 'OUoz9zJlSjnbv7b7jwLkYCZagBfgQpnn2W0ZpwVipXKwRt0QHl'
access_token = '1048444945864880128-XNRsd5FXXFnmqMIk1u2WR5iMDRKdzn'
access_token_secret = 'J4l2Z2W7xlvLke1cCr4BX1oLVoTQfKqD9jPZgICLynz22'
# Attempt authentication
try:
self.auth = OAuthHandler(consumer_key, consumer_secret)
self.auth.set_access_token(access_token, access_token_secret)
self.query = query
self.retweets_only = retweets_only
self.with_sentiment = with_sentiment
self.api = tweepy.API(self.auth)
self.tweet_count_max = 100 # To prevent Rate Limiting
except:
print("Error: Authentication Failed")
def set_query(self, query=''):
self.query = query
def set_retweet_checking(self, retweets_only='false'):
self.retweets_only = retweets_only
def set_with_sentiment(self, with_sentiment='false'):
self.with_sentiment = with_sentiment
def clean_tweet(self, tweet):
return ' '.join(
re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ",
tweet).split())
def get_tweet_sentiment(self, tweet):
analysis = TextBlob(self.clean_tweet(tweet))
if analysis.sentiment.polarity > 0:
return 'positive'
elif analysis.sentiment.polarity == 0:
return 'neutral'
else:
return 'negative'
def get_tweets(self):
tweets = []
try:
recd_tweets = self.api.search(q=self.query,
count=self.tweet_count_max)
for tweet in recd_tweets:
parsed_tweet = {}
parsed_tweet['text'] = tweet.text
parsed_tweet['user'] = tweet.user.screen_name
if self.with_sentiment == 1:
parsed_tweet['sentiment'] = self.get_tweet_sentiment(
tweet.text)
else:
parsed_tweet['sentiment'] = 'unavailable'
if tweet.retweet_count > 0 and self.retweets_only == 1:
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
elif not self.retweets_only:
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
return tweets
except tweepy.TweepError as e:
print("Error : " + str(e))
def fetchTweets(keyword, filename):
auth = OAuthHandler(config.ckey, config.csecret)
auth.set_access_token(config.atoken, config.asecret)
twitterStream = Stream(auth, listener(filename))
twitterStream.filter(track=[keyword])
def get_auth_user():
ad = twitterAuth
auth = OAuthHandler(ad['api']['key'], ad['api']['secret'])
auth.set_access_token(ad['access']['token'], ad['access']['token-secret'])
return auth
class TwitterAuthentication(object):
"""Twitter authentication object.
This is basically just a wrapper for Twitter API keys to prevent a bunch of variables being scattered everywhere.
"""
@staticmethod
def autodetect_twitter_auth(auth_filepath="~/secrets.json"):
# type: (str) -> TwitterAuthentication
"""
Attempts to autodetect Twitter API keys from a file called 'secrets.json'.
Using this method is inadvisable and only exists to aid our test cases.
"""
print("WARNING: API key autodetection is inadvisable.")
auth_filename = os.path.basename(auth_filepath)
auth_filepath = os.path.expanduser(auth_filepath)
if not os.path.isfile(auth_filepath):
print("Auto-detection of {} failed. Searched this path for {}:".
format(auth_filename, auth_filename))
print(auth_filepath)
print(
"Either initialize {} with API keys, or make the file located at the above path."
.format(TwitterAuthentication.__name__))
raise ValueError("No API keys in {} initializer".format(
TwitterAuthentication.__name__))
else: # Path to auth file exists.
return TwitterAuthentication.from_json(auth_filepath)
@staticmethod
def from_json(filepath):
"""
Creates a TwitterAuthentication object from a JSON file.
:param filepath: The path to the JSON file.
:return: A TwitterAuthentication object.
Uses the following keys:
- consumer_key
- consumer_secret
- access_token
- access_token_secret
"""
file = open(filepath, 'r')
json_object = json.load(file)
file.close()
return TwitterAuthentication(
consumer_key=json_object['consumer_key'],
consumer_secret=json_object['consumer_secret'],
access_token=json_object['access_token'],
access_token_secret=json_object['access_token_secret'],
)
def __init__(self, consumer_key, consumer_secret, access_token,
access_token_secret):
self.consumer_key = consumer_key
self.consumer_secret = consumer_secret
self.access_token = access_token
self.access_token_secret = access_token_secret
self.oauth_handler = OAuthHandler(self.consumer_key,
self.consumer_secret)
self.oauth_handler.set_access_token(self.access_token,
self.access_token_secret)
def authenticate_twitter_app(self):
auth = OAuthHandler(auth_twitter.CONSUMER_KEY, auth_twitter.CONSUMER_SECRET)
auth.set_access_token(auth_twitter.ACCESS_TOKEN, auth_twitter.ACCESS_TOKEN_SECRET)
return auth
def __init__(self):
self.auth = OAuthHandler(self.consumer_key, self.consumer_secret)
self.auth.set_access_token(self.access_token, self.access_secret)
self.api = tweepy.API(self.auth)
# Description: script that collects twitter tweets using Tweepy/Twitter API #
#############################################################################
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import sentiment_analysis_module
import json
import csv
consumerKey = "jH7AHkg7xR5BIce9JQkgJIqwZ"
consumerSecret = "89bDpRvlsWptNL6P1cwy1GrXwtaExE3PRuVXpk0a0yYQXiduDP"
accessToken = "2562821789-Lha1HBpzPQTziFQ6vkJEqpbLjpCd2BYPGohCJCZ"
accessTokenSecret = "TS1y9mTEWd3ckdg9UAKm1mpuf9vsxc9umEpzXzuaPgqcW"
auth = OAuthHandler(consumerKey, consumerSecret)
auth.set_access_token(accessToken, accessTokenSecret)
# class streams tweets and writes the text, location, and sentiment value of the text to a csv file
class listener(StreamListener):
sentimment_index = 0
count = 0
def on_data(self, data):
all_data = json.loads(data)
user_data = all_data["user"]
location = user_data["location"]
tweet = all_data["text"]
class TwitterClient(object):
'''
Generic Twitter Class for sentiment analysis.
'''
def __init__(self):
'''
Class constructor or initialization method.
'''
# keys and tokens from the Twitter Dev Console
consumer_key = 'McfEpmGoArTag1y0YEgLmAw9B'
consumer_secret = 'nhh8YrANBIS1CcjM6T1XZr9dDuBOFBK69fyR3vIISlNKQUzmTF'
access_token = '580508392-TGvre1m0EqXppjxZoiNoB7B4bfDBxG61m4rOwIdo'
access_token_secret = 'ZWLk0U1KilShIZHeJnYGshMhxI11JLHL9WurC9IsNck0m'
# attempt authentication
try:
# create OAuthHandler object
self.auth = OAuthHandler(consumer_key, consumer_secret)
# set access token and secret
self.auth.set_access_token(access_token, access_token_secret)
# create tweepy API object to fetch tweets
self.api = tweepy.API(self.auth)
except:
print("Error: Authentication Failed")
def clean_tweet(self, tweet):
'''
Utility function to clean tweet text by removing links, special characters
using simple regex statements.
'''
return ' '.join(
re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ",
tweet).split())
def get_tweet_sentiment(self, tweet):
'''
Utility function to classify sentiment of passed tweet
using textblob's sentiment method
'''
# create TextBlob object of passed tweet text
analysis = TextBlob(self.clean_tweet(tweet))
# set sentiment
if analysis.sentiment.polarity > 0:
return 'positive'
elif analysis.sentiment.polarity == 0:
return 'neutral'
else:
return 'negative'
def get_tweets(self, query, count=10):
'''
Main function to fetch tweets and parse them.
'''
# empty list to store parsed tweets
tweets = []
try:
# call twitter api to fetch tweets
fetched_tweets = self.api.search(q=query, count=count)
# parsing tweets one by one
for tweet in fetched_tweets:
# empty dictionary to store required params of a tweet
parsed_tweet = {}
# saving text of tweet
parsed_tweet['text'] = tweet.text
# saving sentiment of tweet
parsed_tweet['sentiment'] = self.get_tweet_sentiment(
tweet.text)
parsed_tweet['fav'] = tweet.favorite_count
# appending parsed tweet to tweets list
if tweet.retweet_count > 0:
# if tweet has retweets, ensure that it is appended only once
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
else:
tweets.append(parsed_tweet)
# return parsed tweets
return tweets
except tweepy.TweepError as e:
# print error (if any)
print("Error : " + str(e))
class TwitterClient(object):
def __init__(self):
consumer_key = 'w9G7yqJXhkfhaBRlLhnSHMQET'
consumer_secret = 'trrSp2hJefDzcTdgsQv4DNhuZR8tGyj8fQJF5m2juBLHygfJsi'
access_token = '85805130-AK7uXWpTbtXJ9CqKWv584tH7iUQcBcNyhtm0mLM5U'
access_token_secret = 'zWF6ASX4WIOfFieyG45xfaHEYrPujg7U1VGDqPLKxv3R5'
# attempt authentication
try:
# create OAuthHandler object
self.auth = OAuthHandler(consumer_key, consumer_secret)
# set access token and secret
self.auth.set_access_token(access_token, access_token_secret)
# create tweepy API object to fetch tweets
self.api = tweepy.API(self.auth)
except:
print("Error: Authentication Failed")
def get_tweets(self, query, lat, log, radius, since, until, count=10):
'''
Main function to fetch tweets and parse them.
'''
# empty list to store parsed tweets
tweets = []
if since != '':
since = datetime.datetime.strptime(since, "%Y-%m-%d")
since = '{0}-{1}-{2}'.format(since.year, since.month, since.day)
if until != '':
until = datetime.datetime.strptime(until, "%Y-%m-%d")
until = '{0}-{1}-{2}'.format(until.year, until.month, until.day)
try:
# call twitter api to fetch tweets
geo = "" + lat + "," + log + "," + radius + "mi"
#fetched_tweets = self.api.search(q=query,geocode=geo,count=count)
fetched_tweets = tweepy.Cursor(self.api.search,
q=query,
lang='en',
since=since,
until=until,
geo=geo).items(count)
# parsing tweets one by one
for tweet in fetched_tweets:
# empty dictionary to store required params of a tweet
parsed_tweet = {}
# saving text of tweet
parsed_tweet['text'] = tweet.text
parsed_tweet['id'] = tweet.id
parsed_tweet['location'] = tweet.user.location
parsed_tweet['retweetcount'] = tweet.retweet_count
parsed_tweet['gender'] = Gclf.classify(
GenderAlgo.gender_features(tweet.user.name))
parsed_tweet['sentiment'] = self.get_tweet_sentiment(
tweet.text)
parsed_tweet['coordinates'] = tweet.coordinates
# appending parsed tweet to tweets list
if tweet.retweet_count > 0:
# if tweet has retweets, ensure that it is appended only once
if parsed_tweet not in tweets:
tweets.append(parsed_tweet)
else:
tweets.append(parsed_tweet)
# return parsed tweets
return tweets
except tweepy.TweepError as e:
# print error (if any)
print("Error : " + str(e))
def get_tweet_sentiment(self, tweet=[]):
tweet = ' '.join(self.clean_tweet(tweet))
ls = [tweet]
tweet = tfidf.transform(ls).toarray()
return clf.predict(tweet)
def clean_tweet(self, tweet):
text = re.sub(r'((www\.[^\s]+)|(https?://[^\s]+))', ' ', tweet)
text = re.sub(r'@[A-Za-z0-9_]+', ' ', text)
text = text.lower()
text = re.sub(r"RT", " ", text)
text = re.sub(r"that's", "that is", text)
text = re.sub(r"there's", "there is", text)
text = re.sub(r"what's", "what is", text)
text = re.sub(r"where's", "where is", text)
text = re.sub(r"it's", "it is", text)
text = re.sub(r"who's", "who is", text)
text = re.sub(r"i'm", "i am", text)
text = re.sub(r"she's", "she is", text)
text = re.sub(r"he's", "he is", text)
text = re.sub(r"they're", "they are", text)
text = re.sub(r"who're", "who are", text)
text = re.sub(r"ain't", "am not", text)
text = re.sub(r"wouldn't", "would not", text)
text = re.sub(r"shouldn't", "should not", text)
text = re.sub(r"couldn't", "could not", text)
text = re.sub(r"can't", "can not", text)
text = re.sub(r"won't", "will not", text)
text = re.sub(r'@[^\s]+', ' ', text)
text = re.sub(r"\W", " ", text)
text = re.sub(r"\d", " ", text)
text = re.sub(r"^\s", " ", text)
text = re.sub(' +', ' ', text)
word_tokens = word_tokenize(text)
filtered_sentence = [w for w in word_tokens if not w in stop_words]
return filtered_sentence
def wordcount(self, tweet=[]):
counts = dict()
for word in tweet:
if word in counts:
counts[word] += 1
else:
counts[word] = 1
return counts
def take(n, iterable):
"Return first n items of the iterable as a list"
return list(islice(iterable, n))