def poll(self):
tso = TwitterSearchOrder()
tso.set_keywords(self._config["query"])
language = self._config.get("language", None)
if language:
tso.set_language(language)
tso.set_result_type("recent")
tso.set_count(self._config.get("count", 30))
tso.set_include_entities(False)
last_id = self._get_last_id()
if last_id:
tso.set_since_id(int(last_id))
try:
tweets = self._client.search_tweets(tso)
tweets = tweets["content"]["statuses"]
except Exception as e:
self._logger.exception("Polling Twitter failed: %s" % (str(e)))
return
tweets = list(reversed(tweets))
if tweets:
self._set_last_id(last_id=tweets[-1]["id"])
for tweet in tweets:
self._dispatch_trigger_for_tweet(tweet=tweet)
def coleta_tweets():
try:
ts = TwitterSearch(
consumer_key = '',
consumer_secret = '',
access_token = '',
access_token_secret = ''
)
tso = TwitterSearchOrder()
tso.set_keywords(['Harry potter'])
tso.set_language('pt')
df = []
for tweet in ts.search_tweets_iterable(tso):
df.append('@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'])+',')
#print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text']) )
print('Coleta finalizada!')
df = pd.DataFrame(df)
#df.to_csv('tweets.txt')
#print('Arquivo salvo.')
return df
except TwitterSearchException as e:
print(e)
def cmd(send, msg, args):
"""
Search the Twitter API.
Syntax: {command} <query> <--user username> <--count 1>
"""
if not msg:
send('What do you think I am, a bird?')
return
parser = arguments.ArgParser(args['config'])
parser.add_argument('query', nargs='*')
group = parser.add_mutually_exclusive_group()
group.add_argument('--user', dest='user', default=None)
group.add_argument('--count', dest='count', type=int, default=1)
group.add_argument('--random', action='store_true', default=False)
try:
cmdargs = parser.parse_args(msg)
except arguments.ArgumentException as e:
send(str(e))
return
api = get_search_api(args['config'])
query = TwitterSearchOrder()
keywords = [' '.join(cmdargs.query)]
if cmdargs.user:
keywords += [f'from:{cmdargs.user}']
query.set_keywords(keywords)
query.set_language('en')
query.set_result_type('recent')
query.set_include_entities(False)
query.set_count(cmdargs.count)
results = list(api.search_tweets_iterable(query))
if not results:
send('No tweets here!')
return
if cmdargs.random:
shuffle(results)
max_chan_tweets = 5
max_pm_tweets = 25
if cmdargs.count > max_pm_tweets:
send(
f"That's too many tweets! The maximum allowed through PM is {max_pm_tweets}"
)
return
if cmdargs.count > max_chan_tweets:
send(
f"That's a lot of tweets! The maximum allowed in a channel is {max_chan_tweets}"
)
for i in range(0, min(cmdargs.count, max_pm_tweets)):
if cmdargs.count <= max_chan_tweets:
send(tweet_text(results[i]))
else:
send(tweet_text(results[i]), target=args['nick'])
def post(self):
query_string = self.request.body_arguments.get('query')
query = TwitterSearchOrder()
query.set_keywords(query_string)
query.set_language('en')
query.set_include_entities(False)
results = TwitterClient.search_tweets(query)
tweets = [tweet['text'] for tweet in results['content']['statuses']]
tweet_results = indicoio.batch_text_tags(tweets)
n_tweets = float(len(tweet_results))
scores = defaultdict(float)
for tweet in tweet_results:
for category, score in tweet.items():
scores[category] += score / n_tweets
category = max(scores, key=lambda x: scores[x])
data = {
'scores': scores,
'category': category
}
self.write(json.dumps(data))
def rq_tweets(game):
"""
Request tweet metadata according to a game using the Twitter API
"""
search = TwitterSearchOrder()
search.set_keywords(game.c_name.split())
search.set_language('en')
try:
for tweet_json in API.search_tweets_iterable(search):
# Unit filtering
if not validate_tweet(tweet_json):
continue
# Relevancy filtering
if not relevant_tweet(game, tweet_json):
continue
# Remove unwanted information
tweet_json = {
'id': tweet_json['id'],
'text': tweet_json['text'],
'user': {
'name': tweet_json['user']['name']
},
'created_at': tweet_json['created_at']
}
# Finally add the tweet
TC['Tweet.game_id'].add(
CachedTweet(game_id=game.game_id, twitter_data=tweet_json))
except TwitterSearchException:
TC['Tweet.game_id'].flush()
reload_api()
def poll(self):
tso = TwitterSearchOrder()
tso.set_keywords(self._config['query'])
language = self._config.get('language', None)
if language:
tso.set_language(language)
tso.set_result_type('recent')
tso.set_count(self._config.get('count', 30))
tso.set_include_entities(False)
last_id = self._get_last_id()
if last_id:
tso.set_since_id(int(last_id))
try:
tweets = self._client.search_tweets(tso)
tweets = tweets['content']['statuses']
except Exception as e:
self._logger.exception('Polling Twitter failed: %s' % (str(e)))
return
tweets = list(reversed(tweets))
if tweets:
self._set_last_id(last_id=tweets[-1]['id'])
for tweet in tweets:
self._dispatch_trigger_for_tweet(tweet=tweet)
def init_tw_search_lib(self, domain_keyword):
"""
Init TwitterSearch Library
(Copyright (C) 2013 Christian Koepp
https://github.com/ckoepp/TwitterSearch/tree/master)
Arguments:
domain_keyword {str} -- The keyword from <domain_keywords_dict>
that will be used to search in Twitter
Returns:
[TwitterSearch] -- TwitterSearch object with our secret tokens
[TwitterSearchOrder] -- TwitterSearchOrder object with initialized attributes
"""
try:
tso = TwitterSearchOrder() # create a TwitterSearchOrder object
tso.add_keyword(
domain_keyword) # add keyword for search in Twitter
tso.set_language('en') # we want to see English tweets only
tso.set_include_entities(
False) # and don't give us all those entity information
# it's about time to create a TwitterSearch object with our secret tokens
ts = TwitterSearch(
consumer_key='<your-CONSUMER_KEY>',
consumer_secret='<your-CONSUMER_SECRET>',
access_token='<your-ACCESS_TOKEN>',
access_token_secret='<your-ACCESS_TOKEN_SECRET>')
except TwitterSearchException as e: # take care of all those ugly errors if there are some
print(e)
return ts, tso
def poll(self):
tso = TwitterSearchOrder()
tso.set_keywords(self._config['query'], True)
language = self._config.get('language', None)
if language:
tso.set_language(language)
tso.set_result_type('recent')
tso.set_count(self._config.get('count', 30))
tso.set_include_entities(False)
last_id = self._get_last_id()
if last_id:
tso.set_since_id(int(last_id))
try:
tweets = self._client.search_tweets(tso)
tweets = tweets['content']['statuses']
except Exception as e:
self._logger.exception('Polling Twitter failed: %s' % (str(e)))
return
tweets = list(reversed(tweets))
if tweets:
self._set_last_id(last_id=tweets[-1]['id'])
for tweet in tweets:
self._dispatch_trigger_for_tweet(tweet=tweet)
def coleta_tweets():
try:
ts = TwitterSearch(consumer_key='',
consumer_secret='',
access_token='',
access_token_secret='')
tso = TwitterSearchOrder()
tso.set_keywords(['Harry potter'])
tso.set_language('pt')
df = []
for tweet in ts.search_tweets_iterable(tso):
df.append('@%s tweeted: %s' %
(tweet['user']['screen_name'], tweet['text']) + ',')
#print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text']) )
print('Coleta finalizada!')
df = pd.DataFrame(df)
#df.to_csv('tweets.txt')
#print('Arquivo salvo.')
return df
except TwitterSearchException as e:
print(e)
def SearchOnTwitter(keywords, language):
"""
Allows to test twitter search library -> Print tweets of interest.
Parameters:
- keywords : string array that tweets must contain
- language : string indicating the language of the interest tweets
Return :
- array of tweets
"""
tweets = []
try:
tso = TwitterSearchOrder() # create a TwitterSearchOrder object
tso.set_keywords(
keywords
) # let's define all words we would like to have a look for
tso.set_language(language) # we want to see German tweets only
tso.set_include_entities(
False) # and don't give us all those entity information
# it's about time to create a TwitterSearch object with our secret tokens
ts = TwitterSearch(consumer_key=consumer_key,
consumer_secret=consumer_secret,
access_token=access_token,
access_token_secret=access_token_secret)
# this is where the fun actually starts :)
for tweet in ts.search_tweets_iterable(tso):
tweets.append(tweet['text'])
except TwitterSearchException as e: # take care of all those ugly errors if there are some
print(e)
return tweets
def cmd(send, msg, args):
"""
Search the Twitter API.
Syntax: {command} <query> <--user username> <--count 1>
"""
if not msg:
send('What do you think I am, a bird?')
return
parser = arguments.ArgParser(args['config'])
parser.add_argument('query', nargs='*')
group = parser.add_mutually_exclusive_group()
group.add_argument('--user', dest='user', default=None)
group.add_argument('--count', dest='count', type=int, default=1)
group.add_argument('--random', action='store_true', default=False)
try:
cmdargs = parser.parse_args(msg)
except arguments.ArgumentException as e:
send(str(e))
return
api = get_search_api(args['config'])
query = TwitterSearchOrder()
keywords = [' '.join(cmdargs.query)]
if cmdargs.user:
keywords += ['from:{}'.format(cmdargs.user)]
query.set_keywords(keywords)
query.set_language('en')
query.set_result_type('recent')
query.set_include_entities(False)
query.set_count(cmdargs.count)
results = list(api.search_tweets_iterable(query))
if not results:
send('No tweets here!')
return
if cmdargs.random:
shuffle(results)
max_chan_tweets = 5
max_pm_tweets = 25
if cmdargs.count > max_pm_tweets:
send("That's too many tweets! The maximum allowed through PM is {}".format(max_pm_tweets))
return
if cmdargs.count > max_chan_tweets:
send("That's a lot of tweets! The maximum allowed in a channel is {}".format(max_chan_tweets))
for i in range(0, min(cmdargs.count, max_pm_tweets)):
if cmdargs.count <= max_chan_tweets:
send(tweet_text(results[i]))
else:
send(tweet_text(results[i]), target=args['nick'])
def getTweets(politician_id, searchOnlySexistWords):
try:
politician = Politician.objects.get(id=politician_id)
politician_names = [
politician.first_name + " " + politician.last_name,
politician.username
]
tso = TwitterSearchOrder()
searchTerms = []
if searchOnlySexistWords:
sexistWords = CONFIG['SEXISTWORDS']
for word in sexistWords:
for politician_name in politician_names:
searchTerms.append(word + ' ' + politician_name)
elif searchOnlySexistWords is False:
searchTerms = politician_names
tso.set_keywords(searchTerms, or_operator=True)
tso.set_language("en")
tso.set_include_entities(False)
querystr = tso.create_search_url()
tso.set_search_url(querystr + "&tweet_mode=extended")
ts = TwitterSearch(consumer_key=CONFIG['CONSUMER_KEY'],
consumer_secret=CONFIG['CONSUMER_SECRET'],
access_token=CONFIG['ACCESS_TOKEN'],
access_token_secret=CONFIG['ACCESS_TOKEN_SECRET'])
print("**Processing tweets for " +
str(politician.first_name + " " + politician.last_name) + "**")
if searchOnlySexistWords:
tweets = ts.search_tweets_iterable(tso)
return tweets
else:
# will limit to 100 if not only searching sexist words
tweets = ts.search_tweets(tso)
return tweets['content']['statuses']
except TwitterSearchException as e:
logging.exception("Unable to get new tweets because of" + str(e))
def gettwitter(query):
try:
tso = TwitterSearchOrder()
tso.set_language('en')
tso.set_locale('en')
tso.set_keywords([query])
url = "https://twitter.com/search"+tso.create_search_url()
print url
except TwitterSearchException as e:
print(e)
html = getHtml(url)
soup = BeautifulSoup(html)
twits = soup.find_all("p",class_="TweetTextSize")
twitters=[]
for t in twits:
dr = re.compile(r'<[^>]+>',re.S)
replacedStr = dr.sub('',str(t))
replacedStr = re.sub(r"([a-zA-z]+://\S*\s{0,1})", "url", replacedStr)
twitters.append(replacedStr+"\n")
return twitters
def post(self):
query_string = self.request.body_arguments.get('query')
query = TwitterSearchOrder()
query.set_keywords(query_string)
query.set_language('en')
query.set_include_entities(False)
results = TwitterClient.search_tweets(query)
tweets = [tweet['text'] for tweet in results['content']['statuses']]
tweet_results = indicoio.batch_text_tags(tweets)
n_tweets = float(len(tweet_results))
scores = defaultdict(float)
for tweet in tweet_results:
for category, score in tweet.items():
scores[category] += score / n_tweets
category = max(scores, key=lambda x: scores[x])
data = {'scores': scores, 'category': category}
self.write(json.dumps(data))
def post(self):
query_string = self.request.body_arguments.get('query')
query = TwitterSearchOrder()
query.set_keywords(query_string)
query.set_language('en')
query.set_include_entities(False)
results = TwitterClient.search_tweets(query)
tweets = [tweet['text'] for tweet in results['content']['statuses']]
sentiment = indicoio.batch_sentiment(tweets)
pairs = sorted(zip(sentiment, tweets))
n_tweets = float(len(pairs))
top_n = 5
most_negative = pairs[:top_n]
most_positive = list(reversed(pairs[-top_n:]))
data = {
'most_positive': most_positive,
'most_negative': most_negative,
'average': sum(sentiment)/n_tweets
}
self.write(json.dumps(data))
def post(self):
query_string = self.request.body_arguments.get('query')
query = TwitterSearchOrder()
query.set_keywords(query_string)
query.set_language('en')
query.set_include_entities(False)
results = TwitterClient.search_tweets(query)
tweets = [tweet['text'] for tweet in results['content']['statuses']]
sentiment = indicoio.batch_sentiment(tweets)
pairs = sorted(zip(sentiment, tweets))
n_tweets = float(len(pairs))
top_n = 5
most_negative = pairs[:top_n]
most_positive = list(reversed(pairs[-top_n:]))
data = {
'most_positive': most_positive,
'most_negative': most_negative,
'average': sum(sentiment) / n_tweets
}
self.write(json.dumps(data))
def getTweets(politician_id):
try:
politician = Politician.objects.get(id=politician_id)
politician_names = [politician.first_name + " " + politician.last_name, politician.last_name, politician.username]
print("Getting Tweets for " + str(politician.first_name + " " + politician.last_name))
tso = TwitterSearchOrder()
sexistWords = ['bitch', 'skank', 'rape']
searchTerms = []
for word in sexistWords:
for politician in politician_names:
searchTerms.append(word + ' ' + politician)
tso.set_keywords(searchTerms, or_operator=True)
print(searchTerms)
tso.set_language("en")
tso.set_include_entities(False)
querystr = tso.create_search_url()
tso.set_search_url(querystr + "&tweet_mode=extended")
ts = TwitterSearch(
consumer_key = os.environ.get('CONSUMER_KEY', CONFIG['CONSUMER_KEY']),
consumer_secret = os.environ.get('CONSUMER_SECRET', CONFIG['CONSUMER_SECRET']),
access_token = os.environ.get('ACCESS_TOKEN', CONFIG['ACCESS_TOKEN']),
access_token_secret = os.environ.get('ACCESS_TOKEN_SECRET', CONFIG['ACCESS_TOKEN_SECRET'])
)
return ts.search_tweets_iterable(tso)
except TwitterSearchException as e:
logging.exception("Unable to get new tweets because of" + str(e))
# if __name__ == "__main__":
# getTweets()
def search(self, query, lang='en', n=10**5):
"""
Search twitter for specified query.
Function returns n tweets or as many as can be found for that query.
Parameters:
query -- Search query (String)
lang -- Specify language of tweets, optional, default: 'en' (String)
n -- Number of tweets to return, optional, default: 10**3 (Int)
Returns:
tweets_out -- Pandas series of tweets of length n
"""
# Initialise container
tweets_out = []
# Setup twitter search
tso = TwitterSearchOrder()
tso.set_keywords([query])
tso.set_language(lang)
tso.set_include_entities(False)
# Begin search
sys.stdout.write("Tweet number out of {0}: ".format(n))
for i, tweet in enumerate(self.twitter.search_tweets_iterable(tso)):
# Break from loop when n tweets are reached
if i == n:
break
# Output progress
if i % 100 == 0:
sys.stdout.write('{0} '.format(i))
sys.stdout.flush()
# Add the next tweet to the container
tweets_out.append('%s' % (tweet['text']))
print
# Return as pandas series as it's easier to work with
return pd.Series(tweets_out)
database = couchdb.Server()[COUCH_DATABASE_NAME]
# Setup a twitter connection and configure its credentials:
twitter_connection = TwitterSearch(**TWITTER_CREDENTIALS)
# The twitter client may stop iterating the tweets at some point.
# In order to automatically continue at the last position, we put the
# import in a "while"-loop which will be stopped when there are no new
# tweets to import.
while True:
# First, let's build a search query:
twitter_query = TwitterSearchOrder()
twitter_query.set_keywords(TWITTER_SEARCH_KEYWORDS)
# Only import english tweets as our sentiment analysis will only work
# with the English language for now.
twitter_query.set_language('en')
# We do not require entities (e.g. extracted URLs) as we are only
# interested in the raw text of the tweet.
twitter_query.set_include_entities(False)
document_ids = tuple(filter(lambda id_: not id_.startswith('_'), database))
if len(document_ids) > 0:
# If we already have imported tweets, we should continue with the oldest
# tweet we know and work our way to older tweets from there.
# We do that by setting the max_id query parameter to the oldest tweet
# we know.
oldest_id = min(document_ids)
twitter_query.set_max_id(int(oldest_id))
print('Continuing initial import from tweet {}'.format(oldest_id))
else:
print('Starting initial import on fresh database.')
def collect_tweets(keyword, count, force=False):
from TwitterSearch import TwitterSearch
from TwitterSearch import TwitterSearchOrder
import pymongo
from dateutil.parser import parse
from alchemyapi import AlchemyAPI
import ConfigParser
# try:
# keyword = sys.argv[1]
# count = int(sys.argv[2])
# except IndexError:
# e_too_few_args = "You did not enter enough arguments. Two are required: keyword, and count"
# raise Exception(e_too_few_args)
# try:
# if sys.argv[3] == '-f':
# force = True
# else:
# e_invalid_argument = "The only option available is -f. It is used to force the script to continue when the Alchemy API limit is exceeded."
# raise Exception(e_invalid_argument)
# except IndexError:
# force = False
# Read the config file for config variables
config = ConfigParser.RawConfigParser()
config.read('config.cfg')
mongo_url = config.get('Mongo', 'db_url')
# Connect to the Mongo database using MongoClient
client = pymongo.MongoClient(mongo_url)
db = client.get_default_database()
# Access/create the collection based on the command line argument
tweets = db[keyword]
#Generate the alchemyapi variable
alchemyapi = AlchemyAPI()
# To accommodate for hashtags the user can substitute a . for the # in the command line. Lines 30 & 31 return it to a hashtag for the search.
if keyword[0] is ".":
keyword = keyword.replace('.', '#')
# Lines 33-42 ensure that the query is not doing duplicate work.
# First, it counts to see how many documents exist in the collection
db_count = tweets.count()
# If there are documents in the collection, the collection is queried, tweet objects are sorted by date, and the tweet_id of the most recent tweet is retrieved and later set as the "since_id"
if db_count is not 0:
latest_id = tweets.find( {}, { 'object.tweet_id':1 } ).sort("startedAtTime").limit(1)
latest_id_str = latest_id[db_count-1]['object']['tweet_id']
latest_id_int = int(latest_id_str)
print 'Count of documents in the ' + keyword + ' collection is not 0. It is ' + str(db_count) + '. Mongo is now identifying the latest tweet ID to append as a parameter to the API call.'
# If ther are no documents in the collection, no queries are done, and the since_id is left out of the API call.
else:
print 'The Mongo collection ' + keyword + ' is empty. The script will now collect all tweets.'
# create a TwitterSearchOrder object
tso = TwitterSearchOrder()
# let's define all words we would like to have a look for
tso.set_keywords([keyword])
# Select language
tso.set_language('en')
# Include Entity information
tso.set_include_entities(True)
if db_count is not 0:
tso.set_since_id(latest_id_int)
print 'Since the document count in the ' + keyword + ' collection is above 0, the since_id uses the parameter of the latest tweet so that only new tweets are collected.'
else:
print 'No documents exist in the ' + keyword + ' collection right now so the since_id parameter will be empty and all tweets will be collected.'
# Create a TwitterSearch object with our secret tokens
ts = TwitterSearch(
consumer_key = config.get('Twitter', 'consumer_key'),
consumer_secret = config.get('Twitter', 'consumer_secret'),
access_token = config.get('Twitter', 'access_token'),
access_token_secret = config.get('Twitter', 'access_token_secret')
)
# Perform the search
twitter_search = ts.search_tweets_iterable(tso)
# Start the insert count variable
db_inserts = 0
# this is where the fun actually starts :)
try:
for tweet in twitter_search:
if db_inserts < count:
mentions_list = []
hashtags_list = []
# Create the caliper_tweet object
caliper_tweet = {
"context": "http://purl.imsglobal.org/ctx/caliper/v1/MessagingEvent",
"type": "MessagingEvent",
"startedAtTime": "",
## Can be used to query Twitter API for user information
"actor": "",
"verb": "tweetSent",
"object": {
"type": "MessagingEvent",
"tweet_id": "",
"tweet_uri": "",
"subtype": "tweet",
## "to" should be calculated by checking in_reply_to_user_id_str is null. If it's not null, then it should be concatenated to "uri:twitter/user/" and stored in "object"['to']
"to": "",
"author": {
"author_uri": "",
"author_alias": "",
"author_name": "",
},
"text": "",
"sentiment": {
"type": "",
"score": "",
"color": ""
},
"parent": "",
## "mentions" is an array of the caliper IDs from the user_mentions objects array
"user_mentions": [],
## "hashtags" is an array of the hashtag texts included in the tweet entities
"hashtags": []
}
}
# Set the re-usable variables
tweet_text = tweet['text']
## AlchemyAPI Sentiment Analysis
tweet_sentiment = ''
response = alchemyapi.sentiment('text', tweet_text)
if 'docSentiment' in response.keys():
if 'score' in response['docSentiment']:
tweet_sentiment_score = response['docSentiment']['score']
tweet_sentiment_score = float(tweet_sentiment_score)
tweet_sentiment_score = round(tweet_sentiment_score, 2)
else:
tweet_sentiment_score = 0
tweet_sentiment_type = response['docSentiment']['type']
tweet_sentiment_score_a = abs(tweet_sentiment_score)
if (tweet_sentiment_score) > 0:
tweet_sentiment_color = "rgba(0,255,0," + str(tweet_sentiment_score_a) + ")"
else:
tweet_sentiment_color = "rgba(255,0,0," + str(tweet_sentiment_score_a) + ")"
elif force == True:
print 'Force option set to true. The tweet_sentiment object will be set with API Limit Exceeded values.'
tweet_sentiment_type = 'API Limit Exceeded'
tweet_sentiment_score = 0
tweet_sentiment_color = 'rgba(0,0,0,0)'
else:
e_alchemy_api_limit = 'Alchemy API daily limit exceeded. Retry search with force=True to continue'
raise Exception(e_alchemy_api_limit)
ds = tweet['created_at']
tweet_date = parse(ds)
caliper_tweet['startedAtTime'] = tweet_date
caliper_tweet['actor'] = 'student:' + tweet['user']['screen_name']
caliper_tweet['object']['tweet_uri'] = 'https://twitter.com/' + tweet['user']['screen_name'] + '/status/' + tweet['id_str']
caliper_tweet['object']['tweet_id'] = tweet['id_str']
if tweet['in_reply_to_user_id_str'] is None:
caliper_tweet['object']['to'] = 'NoReply'
caliper_tweet['object']['parent'] = 'NoReply'
else:
caliper_tweet['object']['to'] = 'https://twitter.com/intent/user?user_id=' + tweet['in_reply_to_user_id_str']
if tweet['in_reply_to_status_id_str'] is None:
caliper_tweet['object']['parent'] = 'None'
else:
caliper_tweet['object']['parent'] = 'https://twitter.com/' + tweet['user']['screen_name'] + '/status/' + tweet['in_reply_to_status_id_str']
caliper_tweet['object']['author']['author_uri'] = 'https://twitter.com/intent/user?user_id=' + tweet['user']['id_str']
caliper_tweet['object']['author']['author_alias'] = tweet['user']['screen_name']
caliper_tweet['object']['author']['author_name'] = tweet['user']['name']
caliper_tweet['object']['text'] = unicode(tweet['text'])
caliper_tweet['object']['sentiment']['type'] = tweet_sentiment_type
caliper_tweet['object']['sentiment']['score'] = tweet_sentiment_score
caliper_tweet['object']['sentiment']['color'] = tweet_sentiment_color
for x in list(tweet['entities']['hashtags']):
hashtag = x['text']
hashtags_list.append(hashtag)
for x in list(tweet['entities']['user_mentions']):
mention = x['id_str']
mentions_list.append(mention)
caliper_tweet['object']['user_mentions'] = mentions_list
caliper_tweet['object']['hashtags'] = hashtags_list
tweets.insert(caliper_tweet)
db_inserts = db_inserts + 1
else:
raise StopIteration
except StopIteration:
print str(db_inserts) + " inserts made in the " + keyword + " collection."
def Tweets():
try:
max_feeds = 10
tso = TwitterSearchOrder() # create a TwitterSearchOrder object
tso.set_language('en')
tso.set_include_entities(
False) # and don't give us all those entity information
tso.set_until(new_date)
tso.arguments.update({'tweet_mode': 'extended'})
tso.arguments.update({'truncated': 'False'})
ts = TwitterSearch(consumer_key='',
consumer_secret='',
access_token='',
access_token_secret='',
proxy='http://proxy_address')
for c in range(len(MainDF)):
count = 0
#kw=[MainDF['twitter'][c]]
#for h in MainDF['hashtag'][c]:
# kw.append(h)
tso.set_keywords(MainDF['hashtag'][c])
tweets_list = []
tuo = TwitterUserOrder(MainDF['twitter'][c])
# tuo.set_language('en')
tuo.set_include_entities(
False) # and don't give us all those entity information
# tuo.set_until(days_ago)
# tuo.set_count(15)
tuo.arguments.update({'tweet_mode': 'extended'})
tuo.arguments.update({'truncated': 'False'})
#for tweet in ts.search_tweets_iterable(tso):
# print(tweet)
# tweets_list.append([tweet['user']['screen_name'],tweet['full_text']])
for tweet in ts.search_tweets_iterable(tso):
if 'retweeted_status' in tweet:
None
#tweets_list.append([tweet['user']['screen_name'],tweet['retweeted_status']['full_text'],'Retweet of ' + tweet['retweeted_status']['user']['screen_name']])
else:
links = Find(tweet['full_text'])
links = ', '.join(link for link in links)
#print(tweet)
tweets_list.append([
MainDF['company'][c], tweet['user']['screen_name'],
tweet['full_text'], tweet['created_at'], links
])
for tweet in ts.search_tweets_iterable(tuo):
if tweet['lang'] != 'en':
#print(tweet)
None
else:
# print(tweet)
links = Find(tweet['full_text'])
links = ', '.join(link for link in links)
tweets_list.append([
MainDF['company'][c], tweet['user']['screen_name'],
tweet['full_text'], tweet['created_at'], links
])
count = count + 1
if count == max_feeds:
break
if tweets_list != []:
tweets_datasets[MainDF['company'][c]] = pd.DataFrame(
tweets_list)
tweets_datasets[MainDF['company'][c]].columns = [
'Company', 'Source/User', 'Title/Tweet', 'Date', 'Link'
]
tweets_datasets[MainDF['company'][c]].insert(
0, 'Category', 'Twitter')
for i in range(
len(tweets_datasets[MainDF['company'][c]]['Date'])):
tweets_datasets[MainDF['company'][c]]['Date'][i] = parse(
tweets_datasets[MainDF['company'][c]]['Date'][i])
tweets_datasets[
MainDF['company'][c]]['Date'][i] = tweets_datasets[
MainDF['company'][c]]['Date'][i].date()
#print(datasets[companies_names[count]])
tw_current_companies.append(MainDF['company'][c])
else:
None
#tweets_list.append()
#print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'] ) )
except TwitterSearchException as e: # take care of all those ugly errors if there are some
print(e)
import sys
import urllib2
from collections import defaultdict
from TwitterSearch import TwitterSearchOrder
import indicoio
from pprint import pprint
from twitter_demo.twitter import TwitterClient
import twitter_demo.settings as settings
indicoio.config.api_key = "2e559e40a6ebea9dddb39e753363f6e1"
query_string = sys.argv[1:]
query = TwitterSearchOrder()
query.set_keywords(query_string)
query.set_language('en')
query.set_include_entities(False)
results = TwitterClient.search_tweets(query)
#print results['content']['statuses'][0].keys()
#tweets = [tweet['text'] for tweet in results['content']['statuses']]
tweets = []
for tweet in results['content']['statuses']:
data = {'tweet_id': tweet['id_str'], 'tweet_text': tweet['text']}
req = urllib2.Request('http://45.33.90.42:5000/insert_data')
req.add_header('Content-Type', 'application/json')
response = urllib2.urlopen(req, json.dumps(data))
tweets.append(data)
def get_tweets():
#run: ./manage.py shell -c="from tweets.utils import get_tweets; get_tweets()"
#keywords = ["arrombada", "arrombado", "asno", "acefalo","babaca", "babuino", "baitola", "biba", "bicha", "bixa", "bixinha", "bobo", "boceta", "boquete", "borra", "bosta", "buceta", "boceta", "bundao", "burro", "cacete", "cadela", "cagar", "cala", "cale", "caralho", "caralio", "chupe", "come", "corno", "cu", "cusao", "cuzao", "desgracado", "disgracado", "egua", "enraba", "fdp", "fiderapariga", "fidumaegua", "filhodaputa", "filhodeumaputa", "foda", "fodase", "foder", "fuder", "fudeu", "fudido", "gay", "grelo", "idiota", "inferno", "jegue", "louco", "macaco", "mamar", "marica", "merda", "mijao", "otario", "pariu", "pau", "peidar", "pica", "pinto", "piriguete", "piroca", "piru", "porra", "puta", "quinto", "rapariga", "retardado", "rola", "siririca", "tesuda", "tomar", "vagabundo", "vaite", "veado", "velha", "viado", "xereca"]
#keywords = ['foda se', 'foda-se', 'vai te foder', 'vai se foder', 'vai te fuder', 'vai se foder', 'vai tomar no cú', 'vai tomar no cu', 'vai toma no cú', 'vai toma no cu', 'vai tomar no rabo', 'vai toma no rabo', 'vai dar o cú', 'vai dar o cu', 'vai dar o rabo', 'filho da puta', 'filho da égua', 'filho da puta', 'filho da égua', 'filho de uma puta', 'filho de uma égua', 'filho duma puta', 'filho duma égua']
#keywords = ['grande', 'bom', 'novo', 'pequeno', 'próprio', 'velho', 'cheio', 'branco', 'longo', 'único', 'alto', 'certo', 'só', 'possível', 'claro', 'capaz', 'estranho', 'negro', 'enorme', 'escuro', 'seguinte', 'mau', 'diferente', 'preciso', 'difícil', 'antigo', 'bonito', 'simples', 'forte', 'pobre', 'sério', 'belo', 'feliz', 'junto', 'vermelho', 'humano', 'inteiro', 'triste', 'importante', 'meio', 'sentado', 'fácil', 'verdadeiro', 'frio', 'vazio', 'baixo', 'terrível', 'próximo', 'livre', 'profundo', 'jovem', 'preto', 'impossível', 'vivo', 'largo', 'nu', 'necessário', 'azul', 'natural', 'quente', 'completo', 'verde', 'pesado', 'inglês', 'especial', 'rápido', 'igual', 'comprido', 'principal', 'breve', 'rico', 'seco', 'fino', 'geral', 'curto', 'chamado', 'pálido', 'leve', 'anterior', 'perfeito', 'grosso', 'direito', 'calado', 'interessante', 'amarelo', 'sujo', 'pronto', 'imenso', 'cansado', 'duro', 'doente', 'puro', 'presente', 'comum', 'brilhante', 'público', 'magro', 'grave', 'cego', 'deitado', 'vago', 'americano', 'alegre', 'real', 'raro', 'particular', 'calmo', 'ansioso', 'social', 'nervoso', 'má', 'silencioso', 'lento', 'semelhante', 'evidente', 'doce', 'curioso', 'normal', 'morto', 'horrível', 'familiar', 'lindo', 'caro', 'justo', 'distante', 'maravilhoso', 'inútil', 'dourado', 'gordo', 'preocupado', 'suave', 'agradável', 'pessoal', 'fraco', 'seguro', 'satisfeito', 'infeliz', 'sombrio', 'feminino', 'súbito', 'estreito', 'íntimo', 'sexual', 'interior', 'esquerdo', 'simpático', 'errado', 'contente', 'secreto', 'fresco', 'violento', 'solitário', 'político', 'moderno', 'extraordinário', 'vulgar', 'literário', 'ligeiro', 'interessado', 'eterno', 'habitual', 'falso', 'castanho', 'parado', 'inteligente', 'firme', 'misterioso', 'delicado', 'cinzento', 'intenso', 'doméstico', 'parecido', 'incapaz', 'profissional', 'disposto', 'absurdo', 'aberto', 'furioso', 'digno', 'grávido', 'amigo', 'quieto', 'físico', 'final', 'famoso', 'elegante', 'diverso', 'redondo', 'louco', 'local', 'imediato', 'provável', 'irritado', 'absoluto', 'respectivo', 'macio', 'santo', 'prestes', 'feio', 'desesperado', 'colorido', 'ridículo', 'precioso', 'imóvel', 'engraçado', 'útil', 'oposto', 'militar', 'determinado', 'britânico', 'vasto', 'religioso', 'ocupado', 'inquieto', 'impaciente', 'confuso', 'complicado', 'estúpido', 'perigoso', 'frágil', 'fechado', 'espesso', 'decidido', 'recente', 'central', 'alheio', 'agudo', 'surdo', 'minúsculo', 'húmido', 'francês', 'deserto', 'barato', 'sincero', 'sagrado', 'moral', 'luminoso', 'indiferente', 'doido', 'comercial', 'óbvio', 'obscuro', 'magnífico', 'excelente', 'inocente', 'total', 'selvagem', 'louro', 'cor-de-rosa', 'católico', 'amável', 'abafado', 'prático', 'pleno', 'infantil', 'liso', 'espantado', 'vão', 'perdido', 'grato', 'espanhol', 'divertido', 'desagradável', 'delicioso', 'consciente', 'assustado', 'português', 'mágico', 'mental', 'inferior', 'generoso', 'encantador', 'atento', 'invisível', 'industrial', 'exterior', 'estrangeiro', 'derradeiro', 'cruel', 'rígido', 'poderoso', 'modesto', 'italiano', 'gelado', 'farto', 'trémulo', 'oculto', 'nobre', 'infinito', 'inesperado', 'franco', 'encantado', 'amoroso', 'úmido', 'visível', 'severo', 'original', 'orgulhoso', 'mortal', 'esplêndido', 'coitado', 'subtil', 'constante', 'surpreendente', 'solene', 'médio', 'mudo', 'futuro', 'eléctrico', 'distinto', 'demasiado', 'culpado', 'confortável', 'banal', 'regular', 'razoável', 'molhado', 'esquisito', 'amplo', 'universitário', 'moreno', 'masculino', 'honesto', 'elevado', 'divino', 'discreto', 'confiante', 'aflito', 'zangado', 'tranquilo', 'surpreendido', 'preparado', 'miúdo', 'maldito', 'judeu', 'interno', 'insignificante', 'incrível', 'excitado', 'exacto', 'esperto', 'descalço', 'casado', 'ardente', 'típico', 'transparente', 'sólido', 'mero', 'humilde', 'doloroso', 'aéreo', 'autêntico', 'urgente', 'tenso', 'surpreso', 'sereno', 'sensível', 'querido', 'fatal', 'embaraçado', 'ruim', 'legal', 'fundo', 'contrário', 'chato', 'ausente', 'tímido', 'sinistro', 'privado', 'pensativo', 'melancólico', 'envergonhado', 'denso', 'definitivo', 'considerável', 'atrasado', 'assustador', 'artístico', 'tremendo', 'ruivo', 'mole', 'miserável', 'irritante', 'desconfiado', 'adequado', 'áspero', 'trêmulo', 'responsável', 'nítido', 'monstruoso', 'manso', 'fantástico', 'essencial', 'crescente', 'científico', 'bêbado', 'amargo', 'aliviado', 'afastado', 'telefónico', 'romântico', 'espiritual', 'cristão', 'brusco', 'apressado', 'remoto', 'insuportável', 'inicial', 'grosseiro', 'dramático', 'decente', 'admirável', 'acordado', 'sossegado', 'solteiro', 'singular', 'rouco', 'nocturno', 'loiro', 'judaico', 'irlandês', 'indignado', 'histórico', 'gracioso', 'admirado', 'supremo', 'solto', 'internacional', 'europeu', 'deprimido', 'brasileiro', 'atraente', 'animado', 'viúvo', 'ténue', 'terno', 'tamanho', 'ruidoso', 'relativo', 'penetrante', 'ordinário', 'numeroso', 'japonês', 'inúmero', 'interminável', 'ilustre', 'hesitante', 'gentil', 'feroz', 'barbudo', 'apaixonado', 'académico', 'aborrecido', 'tranqüilo', 'tradicional', 'popular', 'perturbado', 'oficial', 'limpo', 'inevitável', 'grisalho', 'extremo', 'desajeitado', 'complexo', 'artificial', 'abatido', 'virgem', 'valioso', 'sensato', 'risonho', 'piedoso', 'morno', 'metálico', 'incerto', 'idêntico', 'grego', 'gigantesco', 'fiel', 'excitante', 'esguio', 'célebre', 'apertado', 'vizinho', 'vacilante', 'universal', 'saudável', 'repleto', 'perpétuo', 'pardo', 'ofegante', 'mútuo', 'lustroso', 'isolado', 'inchado', 'gasto', 'formal', 'fascinado', 'exausto', 'escasso', 'dito', 'distraído', 'desportivo', 'contínuo', 'competente', 'clássico', 'civil', 'adulto', 'adormecido', 'técnico', 'suspeito', 'sensual', 'rigoroso', 'notável', 'irregular', 'idoso', 'excessivo', 'exato', 'espantoso', 'escolar', 'erótico', 'enganado', 'bastante', 'agitado', 'actual', 'absorto', 'teatral', 'roxo', 'robusto', 'reluzente', 'nacional', 'médico', 'moço', 'legítimo', 'idiota', 'frequente', 'conveniente', 'conhecido', 'chocado', 'brando', 'apinhado', 'tonto', 'sentimental', 'rubro', 'rosado', 'primitivo', 'ocasional', 'lateral', 'intelectual', 'horrorizado', 'devido', 'desgraçado', 'demorado', 'corrente', 'automático', 'triunfante', 'soturno', 'sorridente', 'radical', 'policial', 'perturbador', 'perplexo', 'perfumado', 'ocidental', 'monótono', 'lúgubre', 'intacto', 'indiano', 'impressionado', 'hebraico', 'exagerado', 'escarlate', 'devoto', 'constrangido', 'carregado', 'bizarro', 'aparente', 'variado', 'trágico', 'maduro', 'irónico', 'injusto', 'indeciso', 'ideal', 'heróico', 'favorável', 'débil', 'cultural', 'corajoso', 'concentrado', 'comovido', 'clandestino', 'ameaçador', 'alvo', 'vibrante', 'turístico', 'temporário', 'simbólico', 'sarcástico', 'respeitável', 'repentino', 'quadrado', 'preferível', 'ousado', 'oriental', 'negativo', 'lívido', 'límpido', 'insensível', 'grandioso', 'fatigado', 'existente', 'escondido', 'característico', 'alemão', 'sábio', 'sonoro', 'significativo', 'secundário', 'rude', 'repugnante', 'recto', 'prateado', 'podre', 'murcho', 'medieval', 'longínquo', 'intrigado', 'imperioso', 'hábil', 'formoso', 'fascinante', 'extenso', 'estridente', 'específico', 'embaraçoso', 'diário', 'disponível', 'cômico', 'cru', 'correcto', 'convencido', 'careca', 'agressivo', 'africano', 'vagaroso', 'sórdido', 'suado', 'satisfatório', 'prudente', 'obstante', 'natal', 'mesquinho', 'mecânico', 'leal', 'impressionante', 'implacável', 'horroroso', 'frouxo', 'formidável', 'fixo', 'enfadonho', 'emocional', 'económico', 'disperso', 'desvairado', 'desolado', 'dado', 'cintilante', 'afável', 'ágil', 'vistoso', 'traseiro', 'tolo', 'tardio', 'são', 'russo', 'receoso', 'prolongado', 'prodigioso', 'privilegiado', 'odioso', 'meigo', 'lógico', 'irresistível', 'intolerável', 'infernal', 'indispensável', 'habituado', 'fundamental', 'extra', 'espetado', 'escusado', 'egoísta', 'duvidoso', 'descuidado', 'convencional', 'conseguinte', 'confidencial', 'condicionado', 'cheiroso', 'cauteloso', 'atual', 'anual', 'aceitável', 'vário', 'vaidoso', 'trivial', 'tosco', 'rosa', 'ritual', 'respeitoso', 'reservado', 'propício', 'postiço', 'positivo', 'pequenino', 'pavoroso', 'patético', 'paciente', 'ondulante', 'nojento', 'musical', 'minucioso', 'maluco', 'improvável', 'glorioso', 'gigante', 'extravagante', 'exclusivo', 'esbranquiçado', 'enérgico', 'enrugado', 'direto', 'crítico', 'corpulento', 'básico', 'brutal', 'atarefado', 'arrumado', 'animal', 'ávido', 'vitoriano', 'tranquilizador', 'sonolento', 'reduzido', 'radioso', 'psicológico', 'primário', 'preferido', 'múltiplo', 'municipal', 'matinal', 'liberal', 'jovial', 'inofensivo', 'inexplicável', 'inclinado', 'iminente', 'flexível', 'financeiro', 'febril', 'entusiasmado', 'eletrônico', 'eficiente', 'directo', 'diplomático', 'diabólico', 'crucial', 'circular', 'bem-vindo', 'avermelhado', 'aterrorizado', 'aterrado', 'assírio', 'assente', 'apagado', 'ameno', 'amarelado', 'abundante', 'abjecto', 'vulnerável', 'volumoso', 'vertical', 'tácito', 'salgado', 'rasgado', 'perverso', 'permanente', 'peculiar', 'noturno', 'mórbido', 'momentâneo', 'memorável', 'maternal', 'lúcido', 'individual', 'indefinido', 'imponente', 'imperial', 'impecável', 'impassível', 'imaginário', 'hediondo', 'genial', 'fúnebre', 'exigente', 'festivo']
#keywords = ["abbo", "abo", "abortion", "abuse", "addict", "addicts", "adult", "africa", "african", "alla", "allah", "alligatorbait", "amateur", "american", "anal", "analannie", "analsex", "angie", "angry", "anus", "arab", "arabs", "areola", "argie", "aroused", "arse", "arsehole", "asian", "ass", "assassin", "assassinate", "assassination", "assault", "assbagger", "assblaster", "assclown", "asscowboy", "asses", "assfuck", "assfucker", "asshat", "asshole", "assholes", "asshore", "assjockey", "asskiss", "asskisser", "assklown", "asslick", "asslicker", "asslover", "assman", "assmonkey", "assmunch", "assmuncher", "asspacker", "asspirate", "asspuppies", "assranger", "asswhore", "asswipe", "athletesfoot", "attack", "australian", "babe", "babies", "backdoor", "backdoorman", "backseat", "badfuck", "balllicker", "balls", "ballsack", "banging", "baptist", "barelylegal", "barf", "barface", "barfface", "bast", "bastard ", "bazongas", "bazooms", "beaner", "beast", "beastality", "beastial", "beastiality", "beatoff", "beat-off", "beatyourmeat", "beaver", "bestial", "bestiality", "bi", "biatch", "bible", "bicurious", "bigass", "bigbastard", "bigbutt", "bigger", "bisexual", "bi-sexual", "bitch", "bitcher", "bitches", "bitchez", "bitchin", "bitching", "bitchslap", "bitchy", "biteme", "black", "blackman", "blackout", "blacks", "blind", "blow", "b*****b", "boang", "bogan", "bohunk", "bollick", "bollock", "bomb", "bombers", "bombing", "bombs", "bomd", "bondage", "boner", "bong", "boob", "boobies", "boobs", "booby", "boody", "boom", "boong", "boonga", "boonie", "booty", "bootycall", "bountybar", "bra", "brea5t", "breast", "breastjob", "breastlover", "breastman", "brothel", "bugger", "buggered", "buggery", "bullcrap", "bulldike", "bulldyke", "bullshit", "bumblefuck", "bumfuck", "bunga", "bunghole", "buried", "burn", "butchbabes", "butchdike", "butchdyke", "butt", "buttbang", "butt-bang", "buttface", "buttfuck", "butt-f**k", "buttfucker", "butt-f****r", "buttfuckers", "butt-fuckers", "butthead", "buttman", "buttmunch", "buttmuncher", "buttpirate", "b******g", "buttstain", "byatch", "cacker", "cameljockey", "cameltoe", "canadian", "cancer", "carpetmuncher", "carruth", "catholic", "catholics", "cemetery", "chav", "cherrypopper", "chickslick", "children's", "chin", "chinaman", "chinamen", "chinese", "chink", "chinky", "choad", "chode", "christ", "christian", "church", "cigarette", "cigs", "clamdigger", "clamdiver", "c**t", "clitoris", "clogwog", "cocaine", "c**k", "cockblock", "cockblocker", "cockcowboy", "cockfight", "cockhead", "cockknob", "cocklicker", "cocklover", "cocknob", "cockqueen", "cockrider", "cocksman", "cocksmith", "cocksmoker", "cocksucer", "cocksuck ", "cocksucked ", "c********r", "cocksucking", "cocktail", "cocktease", "cocky", "cohee", "coitus", "color", "colored", "coloured", "commie", "communist", "condom", "conservative", "conspiracy", "coolie", "cooly", "coon", "coondog", "copulate", "cornhole", "corruption", "cra5h", "crabs", "crack", "crackpipe", "crackwhore", "crack-w***e", "crap", "crapola", "crapper", "crappy", "crash", "creamy", "crime", "crimes", "criminal", "criminals", "crotch", "crotchjockey", "crotchmonkey", "crotchrot", "cum", "cumbubble", "cumfest", "cumjockey", "cumm", "cummer", "cumming", "cumquat", "cumqueen", "cumshot", "cunilingus", "cunillingus", "cunn", "cunnilingus", "cunntt", "c**t", "cunteyed", "cuntfuck", "cuntfucker", "cuntlick ", "cuntlicker ", "cuntlicking ", "cuntsucker", "cybersex", "cyberslimer", "dago", "dahmer", "dammit", "damn", "damnation", "damnit", "darkie", "darky", "datnigga", "dead", "deapthroat", "death", "deepthroat", "defecate", "dego", "demon", "deposit", "desire", "destroy", "deth", "devil", "devilworshipper", "dick", "dickbrain", "dickforbrains", "dickhead", "dickless", "dicklick", "dicklicker", "dickman", "dickwad", "dickweed", "diddle", "die", "died", "dies", "dike", "d***o", "dingleberry", "dink", "dipshit", "dipstick", "dirty", "disease", "diseases", "disturbed", "dive", "dix", "dixiedike", "dixiedyke", "doggiestyle", "doggystyle", "dong", "doodoo", "doo-doo", "doom", "dope", "dragqueen", "dragqween", "dripdick", "drug", "drunk", "drunken", "dumb", "dumbass", "dumbbitch", "dumbfuck", "dyefly", "dyke", "easyslut", "eatballs", "eatme", "eatpussy", "ecstacy", "e*******e", "ejaculated", "ejaculating ", "e*********n", "enema", "enemy", "erect", "erection", "ero", "escort", "ethiopian", "ethnic", "european", "evl", "excrement", "execute", "executed", "execution", "executioner", "explosion", "facefucker", "faeces", "f*g", "fagging", "f****t", "fagot", "failed", "failure", "fairies", "fairy", "faith", "fannyfucker", "fart", "farted ", "farting ", "farty ", "fastfuck", "fat", "fatah", "fatass", "fatfuck", "fatfucker", "fatso", "fckcum", "fear", "feces", "felatio ", "felch", "felcher", "f******g", "f******o", "feltch", "feltcher", "feltching", "fetish", "fight", "filipina", "filipino", "fingerfood", "fingerfuck ", "fingerfucked ", "fingerfucker ", "fingerfuckers", "fingerfucking ", "fire", "firing", "fister", "fistfuck", "fistfucked ", "fistfucker ", "fistfucking ", "fisting", "flange", "flasher", "flatulence", "floo", "flydie", "flydye", "fok", "fondle", "footaction", "footfuck", "footfucker", "footlicker", "footstar", "fore", "foreskin", "forni", "fornicate", "foursome", "fourtwenty", "fraud", "freakfuck", "freakyfucker", "freefuck", "fu", "fubar", "fuc", "fucck", "f**k", "f***a", "fuckable", "fuckbag", "fuckbuddy", "f****d", "fuckedup", "f****r", "fuckers", "fuckface", "fuckfest", "fuckfreak", "fuckfriend", "fuckhead", "fuckher", "f****n", "fuckina", "f*****g", "fuckingbitch", "fuckinnuts", "fuckinright", "fuckit", "fuckknob", "f****e ", "fuckmehard", "fuckmonkey", "fuckoff", "fuckpig", "f***s", "fucktard", "fuckwhore", "fuckyou", "f*********r", "fugly", "fuk", "fuks", "funeral", "funfuck", "fungus", "fuuck", "g******g", "gangbanged ", "gangbanger", "gangsta", "gatorbait", "gay", "gaymuthafuckinwhore", "gaysex ", "geez", "geezer", "geni", "genital", "german", "getiton", "gin", "ginzo", "gipp", "girls", "givehead", "glazeddonut", "gob", "god", "godammit", "goddamit", "goddammit", "goddamn", "goddamned", "goddamnes", "goddamnit", "goddamnmuthafucker", "goldenshower", "gonorrehea", "gonzagas", "gook", "gotohell", "goy", "goyim", "greaseball", "gringo", "groe", "gross", "grostulation", "gubba", "gummer", "gun", "gyp", "gypo", "gypp", "gyppie", "gyppo", "gyppy", "hamas", "handjob", "hapa", "harder", "hardon", "harem", "headfuck", "headlights", "hebe", "heeb", "hell", "henhouse", "heroin", "herpes", "heterosexual", "hijack", "hijacker", "hijacking", "hillbillies", "hindoo", "hiscock", "hitler", "hitlerism", "hitlerist", "hiv", "ho", "hobo", "hodgie", "hoes", "hole", "holestuffer", "homicide", "h**o", "homobangers", "homosexual", "honger", "honk", "honkers", "honkey", "honky", "hook", "hooker", "hookers", "hooters", "hore", "hork", "horn", "horney", "horniest", "horny", "horseshit", "hosejob", "hoser", "hostage", "hotdamn", "hotpussy", "hottotrot", "hummer", "husky", "hussy", "hustler", "hymen", "hymie", "iblowu", "idiot", "ikey", "illegal", "incest", "insest", "intercourse", "interracial", "intheass", "inthebuff", "israel", "israeli", "israel's", "italiano", "itch", "jackass", "jackoff", "jackshit", "jacktheripper", "jade", "jap", "japanese", "japcrap", "jebus", "jeez", "jerkoff", "jesus", "jesuschrist", "jew", "jewish", "jiga", "jigaboo", "jigg", "jigga", "jiggabo", "jigger ", "jiggy", "jihad", "jijjiboo", "jimfish", "jism", "j*z ", "jizim", "jizjuice", "jizm ", "j**z", "jizzim", "jizzum", "joint", "juggalo", "jugs", "junglebunny", "kaffer", "kaffir", "kaffre", "kafir", "kanake", "kid", "kigger", "kike", "kill", "killed", "killer", "killing", "kills", "kink", "kinky", "kissass", "kkk", "knife", "knockers", "kock", "kondum", "koon", "kotex", "krap", "krappy", "kraut", "kum", "kumbubble", "kumbullbe", "kummer", "kumming", "kumquat", "kums", "kunilingus", "kunnilingus", "kunt", "ky", "kyke", "lactate", "laid", "lapdance", "latin", "lesbain", "lesbayn", "lesbian", "lesbin", "lesbo", "lez", "lezbe", "lezbefriends", "lezbo", "lezz", "lezzo", "liberal", "libido", "licker", "lickme", "lies", "limey", "limpdick", "limy", "lingerie", "liquor", "livesex", "loadedgun", "lolita", "looser", "loser", "lotion", "lovebone", "lovegoo", "lovegun", "lovejuice", "lovemuscle", "lovepistol", "loverocket", "lowlife", "lsd", "lubejob", "lucifer", "luckycammeltoe", "lugan", "lynch", "macaca", "mad", "mafia", "magicwand", "mams", "manhater", "manpaste", "marijuana", "mastabate", "mastabater", "masterbate", "masterblaster", "mastrabator", "m********e", "masturbating", "mattressprincess", "meatbeatter", "meatrack", "meth", "mexican", "mgger", "mggor", "mickeyfinn", "mideast", "milf", "minority", "mockey", "mockie", "mocky", "mofo", "moky", "moles", "molest", "molestation", "molester", "molestor", "moneyshot", "mooncricket", "mormon", "moron", "moslem", "mosshead", "mothafuck", "m********a", "mothafuckaz", "mothafucked ", "mothafucker", "mothafuckin", "mothafucking ", "mothafuckings", "motherfuck", "motherfucked", "m**********r", "motherfuckin", "motherfucking", "motherfuckings", "motherlovebone", "muff", "muffdive", "muffdiver", "muffindiver", "mufflikcer", "mulatto", "muncher", "munt", "murder", "murderer", "muslim", "naked", "narcotic", "nasty", "nastybitch", "nastyho", "nastyslut", "nastywhore", "nazi", "necro", "negro", "negroes", "negroid", "negro's", "nig", "niger", "nigerian", "nigerians", "nigg", "n***a", "niggah", "niggaracci", "niggard", "niggarded", "niggarding", "niggardliness", "niggardliness's", "niggardly", "niggards", "niggard's", "niggaz", "nigger", "niggerhead", "niggerhole", "niggers", "nigger's", "niggle", "niggled", "niggles", "niggling", "nigglings", "niggor", "niggur", "niglet", "nignog", "nigr", "nigra", "nigre", "nip", "nipple", "nipplering", "nittit", "nlgger", "nlggor", "nofuckingway", "nook", "nookey", "nookie", "noonan", "nooner", "nude", "nudger", "nuke", "nutfucker", "nymph", "ontherag", "oral", "orga", "orgasim ", "o****m", "orgies", "orgy", "osama", "paki", "palesimian", "palestinian", "pansies", "pansy", "panti", "panties", "payo", "pearlnecklace", "peck", "pecker", "peckerwood", "pee", "peehole", "pee-pee", "peepshow", "peepshpw", "pendy", "penetration", "peni5", "penile", "penis", "penises", "penthouse", "period", "perv", "phonesex", "phuk", "phuked", "phuking", "phukked", "phukking", "phungky", "phuq", "pi55", "picaninny", "piccaninny", "pickaninny", "piker", "pikey", "piky", "pimp", "pimped", "pimper", "pimpjuic", "pimpjuice", "pimpsimp", "pindick", "piss", "pissed", "pisser", "pisses ", "pisshead", "pissin ", "pissing", "pissoff ", "pistol", "pixie", "pixy", "playboy", "playgirl", "pocha", "pocho", "pocketpool", "pohm", "polack", "pom", "pommie", "pommy", "poo", "poon", "poontang", "poop", "pooper", "pooperscooper", "pooping", "poorwhitetrash", "popimp", "porchmonkey", "p**n", "pornflick", "pornking", "porno", "pornography", "pornprincess", "pot", "poverty", "premature", "pric", "prick", "prickhead", "primetime", "propaganda", "pros", "prostitute", "protestant", "pu55i", "pu55y", "pube", "pubic", "pubiclice", "pud", "pudboy", "pudd", "puddboy", "puke", "puntang", "purinapricness", "puss", "pussie", "pussies", "pussy", "pussycat", "pussyeater", "pussyfucker", "pussylicker", "pussylips", "pussylover", "pussypounder", "pusy", "quashie", "queef", "queer", "quickie", "quim", "ra8s", "rabbi", "racial", "racist", "radical", "radicals", "raghead", "randy", "rape", "raped", "raper", "rapist", "rearend", "rearentry", "rectum", "redlight", "redneck", "reefer", "reestie", "refugee", "reject", "remains", "rentafuck", "republican", "rere", "retard", "retarded", "ribbed", "rigger", "rimjob", "rimming", "roach", "robber", "roundeye", "rump", "russki", "russkie", "sadis", "sadom", "samckdaddy", "sandm", "sandnigger", "satan", "scag", "scallywag", "scat", "schlong", "screw", "screwyou", "scrotum", "scum", "s***n", "seppo", "servant", "sex", "sexed", "sexfarm", "sexhound", "sexhouse", "sexing", "sexkitten", "sexpot", "sexslave", "sextogo", "sextoy", "sextoys", "sexual", "sexually", "sexwhore", "sexy", "sexymoma", "sexy-slim", "shag", "shaggin", "shagging", "shat", "shav", "shawtypimp", "sheeney", "shhit", "shinola", "shit", "shitcan", "shitdick", "s***e", "shiteater", "shited", "shitface", "shitfaced", "shitfit", "shitforbrains", "shitfuck", "shitfucker", "shitfull", "shithapens", "shithappens", "s******d", "shithouse", "shiting", "shitlist", "shitola", "shitoutofluck", "shits", "shitstain", "s*****d", "shitter", "s******g", "shitty ", "shoot", "shooting", "shortfuck", "showtime", "sick", "sissy", "sixsixsix", "sixtynine", "sixtyniner", "skank", "skankbitch", "skankfuck", "skankwhore", "skanky", "skankybitch", "skankywhore", "skinflute", "skum", "skumbag", "slant", "slanteye", "slapper", "slaughter", "slav", "slave", "slavedriver", "sleezebag", "sleezeball", "slideitin", "slime", "slimeball", "slimebucket", "slopehead", "slopey", "slopy", "s**t", "s***s", "slutt", "slutting", "slutty", "slutwear", "slutwhore", "smack", "smackthemonkey", "smut", "snatch", "snatchpatch", "snigger", "sniggered", "sniggering", "sniggers", "snigger's", "sniper", "snot", "snowback", "snownigger", "sob", "sodom", "sodomise", "sodomite", "sodomize", "sodomy", "sonofabitch", "sonofbitch", "sooty", "sos", "soviet", "spaghettibender", "spaghettinigger", "spank", "spankthemonkey", "sperm", "spermacide", "spermbag", "spermhearder", "spermherder", "spic", "spick", "spig", "spigotty", "spik", "spit", "spitter", "splittail", "spooge", "spreadeagle", "spunk", "spunky", "squaw", "stagg", "stiffy", "strapon", "stringer", "stripclub", "stroke", "stroking", "stupid", "stupidfuck", "stupidfucker", "suck", "suckdick", "sucker", "suckme", "suckmyass", "suckmydick", "suckmytit", "suckoff", "suicide", "swallow", "swallower", "swalow", "swastika", "sweetness", "syphilis", "taboo", "taff", "tampon", "tang", "tantra", "tarbaby", "tard", "teat", "terror", "terrorist", "teste", "testicle", "testicles", "thicklips", "thirdeye", "thirdleg", "threesome", "threeway", "timbernigger", "tinkle", "tit", "titbitnipply", "titfuck", "titfucker", "titfuckin", "titjob", "titlicker", "titlover", "t**s", "tittie", "titties", "titty", "tnt", "toilet", "tongethruster", "tongue", "tonguethrust", "tonguetramp", "tortur", "torture", "tosser", "towelhead", "trailertrash", "tramp", "trannie", "tranny", "transexual", "transsexual", "transvestite", "triplex", "trisexual", "trojan", "trots", "tuckahoe", "tunneloflove", "turd", "turnon", "twat", "twink", "twinkie", "twobitwhore", "uck", "uk", "unfuckable", "upskirt", "uptheass", "upthebutt", "urinary", "urinate", "urine", "usama", "uterus", "v****a", "vaginal", "vatican", "vibr", "vibrater", "vibrator", "vietcong", "violence", "virgin", "virginbreaker", "vomit", "vulva", "wab", "wank", "wanker", "wanking", "waysted", "weapon", "weenie", "weewee", "welcher", "welfare", "wetb", "wetback", "wetspot", "whacker", "whash", "whigger", "whiskey", "whiskeydick", "whiskydick", "whit", "whitenigger", "whites", "whitetrash", "whitey", "whiz", "whop", "w***e", "whorefucker", "whorehouse", "wigger", "willie", "williewanker", "willy", "wn", "wog", "women's", "wop", "wtf", "wuss", "wuzzie", "xtc", "xxx", "yankee", "yellowman", "zigabo", "zipperhead"]
#keywords = ['adorable', 'beautiful', 'clean', 'drab', 'elegant', 'fancy', 'glamorous', 'handsome', 'long', 'magnificent', 'old-fashioned', 'plain', 'quaint', 'sparkling', 'ugliest', 'unsightly', 'wide-eyed', 'red', 'orange', 'yellow', 'green', 'blue', 'purple', 'gray', 'black', 'white', 'alive', 'better', 'careful', 'clever', 'dead', 'easy', 'famous', 'gifted', 'helpful', 'important', 'inexpensive', 'mushy', 'odd', 'powerful', 'rich', 'shy', 'tender', 'uninterested', 'vast', 'wrong', 'angry', 'bewildered', 'clumsy', 'defeated', 'embarrassed', 'fierce', 'grumpy', 'helpless', 'itchy', 'jealous', 'lazy', 'mysterious', 'nervous', 'obnoxious', 'panicky', 'repulsive', 'scary', 'thoughtless', 'uptight', 'worried', 'agreeable', 'brave', 'calm', 'delightful', 'eager', 'faithful', 'gentle', 'happy', 'jolly', 'kind', 'lively', 'nice', 'obedient', 'proud', 'relieved', 'silly', 'thankful', 'victorious', 'witty', 'zealous', 'broad', 'chubby', 'crooked', 'curved', 'deep', 'flat', 'high', 'hollow', 'low', 'narrow', 'round', 'shallow', 'skinny', 'square', 'steep', 'straight', 'wide', 'big', 'colossal', 'fat', 'gigantic', 'great', 'huge', 'immense', 'large', 'little', 'mammoth', 'massive', 'miniature', 'petite', 'puny', 'scrawny', 'short', 'small', 'tall', 'teeny', 'teeny-tiny', 'tiny', 'cooing', 'deafening', 'faint', 'hissing', 'loud', 'melodic', 'noisy', 'purring', 'quiet', 'raspy', 'screeching', 'thundering', 'voiceless', 'whispering', 'ancient', 'brief', 'early', 'fast', 'late', 'long', 'modern', 'old', 'old-fashioned', 'quick', 'rapid', 'short', 'slow', 'swift', 'young', 'bitter', 'delicious', 'fresh', 'greasy', 'juicy', 'hot', 'icy', 'loose', 'melted', 'nutritious', 'prickly', 'rainy', 'rotten', 'salty', 'sticky', 'strong', 'sweet', 'tart', 'tasteless', 'uneven', 'weak', 'wet', 'wooden', 'yummy', 'boiling', 'breeze', 'broken', 'bumpy', 'chilly', 'cold', 'cool', 'creepy', 'crooked', 'cuddly', 'curly', 'damaged', 'damp', 'dirty', 'dry', 'dusty', 'filthy', 'flaky', 'fluffy', 'freezing', 'hot', 'warm', 'wet', 'abundant', 'empty', 'few', 'full', 'heavy', 'light', 'many', 'numerous', 'sparse', 'substantial']
#keywords = ['Du Hurensohn', 'Arschfotze', 'Arschgeige', 'Arschgesicht', 'Arschloch', 'Bloede Kuh', 'Bulle', 'Das ist mir scheißegal', 'Das war zu einfach', 'Deine Mutter schwitzt beim Kacken', 'Deine Mutter', 'Depp', 'Die Möpse', 'Dreckige Hure', 'Drecksau', 'Du Bastard', 'Du Drecksack', 'Du Fickfehler', 'Du Muschi', 'Du Schweinehund', 'Du Weichei', 'Du alte Kackbratze', 'Du arschgefickter Hurensohn', 'Du bist ein volltrottel', 'Du hässliger Ziegenficker', 'Du kannst mich mal', 'Du verdammtes Arschloch', 'Dumme Schlampe', 'Dummes Huhn', 'Dummkopf', 'Fick deine Mutter', 'Fick dich Arschloch', 'Fick dich ins Knie', 'Fick dich', 'Ficker', 'Fotze', 'Geh Staub fressen', 'Geh zum Teufel', 'Geh kacken', 'Gottverdammt ', 'Halt deine Fresse', 'Halt die Fresse', 'Hirnlose Ochse', 'Huhrensohn', 'Hurensohn', 'Ich Liebe Dich', 'Ich ficke deine Schwester', 'Ich hasse dich', 'Ich will dich ficken', 'Ich will ficken', 'Kackbratze', 'LMS', 'Lutsch mein Schwanz', 'Leck mich am Arsch', 'Luder', 'Lutsch meine Eier', 'Mieser Stricher', 'Mutterficker', 'Nutle', 'Nuttensohn', 'Onanieren', 'Scheissen', 'Scheiße', 'Scheißhaus', 'Schlampe', 'Schwanzlutscher', 'Schweinepriester', 'Schwuchtel', 'Schwul', 'Sheisse', 'Trottel', 'Veganer', 'Verdammte Scheiße', 'Verpiss dich', 'Wichser', 'Wixer', 'Zeig mir deine Pflaume', 'Zicke', 'Zickig', 'blutige Sau', 'bumsen', 'das Arschloch', 'der Arsch', 'der Schwanz', 'der Teufel', 'der abschaum', 'der abschaum der menschlichen gesellschaft', 'der drecksack', 'der dreckskerl', 'der schwanz', 'der schwanzlutscher', 'die Fotze', 'die Hure', 'die Scheiße', 'die Schlampe', 'die Titten', 'du blöde stinkfotze', 'du dumme kuh', 'du verdammter Arschficker', 'dumme Kuh', 'duncauf', 'fahr zur holle', 'fick dich', 'fickdich', 'homofuerst', 'ich fick deine mutter', 'ich will dich ficken', 'missgeburt', 'muschi lecker', 'verdammt du hurensohn', 'verpiss dich', 'voegeln', 'vögeln', 'wichser', 'zur Holle mit dir']
#keywords += ['Phrase', 'abenteuerlich', 'abhängig', 'abwesend', 'aggressiv', 'ahnungslos', 'aktiv', 'allein', 'altmodisch', 'anpassungsfähig', 'anständig', 'ärgerlich', 'arm', 'arrogant', 'attraktiv', 'ätzend', 'aufgeklärt', 'aufgeregt', 'aufgeschlossen', 'aufrichtig', 'ausgeflippt', 'begabt', 'begeistert', 'bekannt', 'berühmt', 'beliebt', 'populär', 'bequem', 'bescheiden', 'bescheuert', 'bezaubernd', 'billig', 'blöd', 'boshaft', 'brav', 'charmant', 'cool', 'dankbar', 'dick', 'dumm', 'doof', 'dünn', 'egozentrisch', 'ehrgeizig', 'ehrlich', 'eifersüchtig', 'einfach', 'eingebildet', 'einmalig', 'einsam', 'einverstanden', 'ekelhaft', 'eklig', 'elegant', 'empfindlich', 'engagiert', 'engstirnig', 'erfolgreich', 'ernst', 'erstklassig', 'fair', 'falsch', 'fantastisch', 'faszinierend', 'faul', 'feig', 'fein', 'fest', 'fett', 'fit', 'fleißig', 'fortgeschritten', 'frech', 'frei', 'freundlich', 'froh', 'fröhlich', 'fürsorglich', 'gastfreundlich', 'gebildet', 'geduldig', 'gefährlich', 'gefühlvoll', 'geistreich', 'geizig', 'gemein', 'gemütlich', 'genial', 'gerecht', 'geschätzt', 'gescheit', 'geschickt', 'geschlossen', 'geschwätzig', 'gesellig', 'gesund', 'gierig', 'glaubwürdig', 'glücklich', 'grob', 'grosszügig', 'groß', 'grüntig', 'gut', 'gut angezogen', 'gut gelaunt', 'gut informiert', 'halb', 'halsstarrig', 'hart', 'hartnäckig', 'hässlich', 'heiß', 'heiter', 'hell', 'hemmungslos', 'herrlich', 'herzlos', 'hilfreich', 'hinterlistig', 'hoch', 'hochmütig', 'hochnäsig', 'höflich', 'hübsch', 'hungrig', 'idealistisch', 'intelligent', 'interessant', 'intolerant', 'jung', 'kalt', 'kindisch', 'klasse', 'toll', 'super', 'klein', 'kleinlich', 'klug', 'komisch', 'kompliziert', 'konsequent', 'konservativ', 'kontaktfreudig', 'kräftig', 'krank', 'kreativ', 'kritisch', 'krumm', 'labil', 'lang', 'langsam', 'langweilig', 'lässig', 'launisch', 'laut', 'lebendig', 'leicht', 'leichtsinnig', 'leidenschaftlich', 'leise', 'liberal', 'lieb', 'liebenswürdig', 'lustig', 'melancholisch', 'merkwürdig', 'miserabel', 'misstrauisch', 'modern', 'modisch', 'mollig', 'moralisch', 'munter', 'musikalisch', 'mutig', 'nachlässig', 'nah', 'naß', 'neidisch', 'nervös', 'nett', 'neu', 'neugierig', 'niedergeschlagen', 'niedlich', 'niedrig', 'normal', 'oberflächlich', 'offen', 'optimistisch', 'ordentlich', 'passiv', 'parteiisch', 'peinlich', 'pessimistisch', 'praktisch', 'pünktlich', 'radikal', 'raffiniert', 'rauh', 'rebellisch', 'recht', 'rechthaberisch', 'redlich', 'reich', 'reif', 'religiös', 'richtig', 'riesig', 'romantisch', 'rücksichtslos', 'rücksichtsvoll', 'ruhig', 'sauber', 'sauer', 'schick', 'schlampig', 'schlau', 'schlecht', 'schlimm', 'schmutzig', 'schnell', 'schön', 'schüchtern', 'schwach', 'schwer', 'schwierig', 'schwerfällig', 'schwermütig', 'selbstlos', 'selbstsicher', 'selbstsüchtig', 'seltsam', 'sensibel', 'sicher', '"sorgenlos', 'sorgfältig', 'spät', 'spontan', 'sportlich', 'spöttisch', 'stark', 'stolz (auf)', 'streitsüchtig', 'süß', 'sympathisch', 'taktlos', 'taktvoll', 'temperamentvoll', 'teuer', 'tot', 'traurig', 'treu', 'typisch', 'übergeschnappt', 'umweltbewusst', 'unabhängig', 'unbeholfen', 'unbekümmert', 'unberechenbar', 'unbeugsam', 'unerfarhren', 'ungehorsam', 'ungeschickt', 'unhöflich', 'unwiderstehlich', 'verantwortlich', 'verbissen', 'verdrießlich', 'verlässlich', 'verlegen', 'vernünftig', 'verrückt', 'vertrauenswürdig', 'verwirrt', 'verwöhnt', 'verzweifelt', 'vorsichtig', 'wahnsinnig', 'warm', 'warmherzig', 'wichtig', 'widerlich', 'wild', 'winzig', 'witzig', 'wunderbar', 'wunderschön', 'zerstreut', 'zufällig', 'zufrieden', 'zusammen', 'zuverlässig']
#keywords += ['ehrgeizig', 'Amerikaner', 'ärgerlich', 'schlecht', 'schön', 'groß', 'blondine', 'langweilig', 'tapfer', 'unbesonnen', 'vorsichtig', 'bestimmt', 'charmant', 'fröhlich', 'Chinesisch', 'eingebildet', 'herkömmlich', 'feigling', 'Nüsse', 'grausam', 'schwierig', 'unangenehm', 'langweilig', 'leicht', 'Englisch', 'unecht', 'Fett', 'ein wenig', 'Französisch', 'häufig', 'freundlich', 'amüsant', 'komisch', 'General', 'großzügig', 'Deutsch', 'gut', 'hübsch', 'fleißig', 'hoch', 'ehrlich', 'intelligent', 'interessant', 'Art', 'entspannend', 'faul', 'klein', 'kurz', 'niedrig', 'bescheiden', 'launisch', 'naiv', 'engstirnig', 'neu', 'nett', 'alt', 'vollkommen', 'Persönlicher', 'fromm', 'höflich', 'schlecht', 'möglich', 'ziemlich', 'stolz', 'schnell', 'realistisch', 'neu', 'zuverlässig', 'reich', 'jämmerlich', 'egoistisch', 'empfindlich', 'schüchtern', 'stumm', 'dünn', 'schlank', 'langsam', 'klein', 'Spanisch', 'streng', 'stark', 'störrisch', 'gesprächig', 'vertrauenswürdig', 'hässlich', 'verschieden', 'schwach', 'unheimlich', 'weiß', 'jung']
lang = 'de'
#lang = 'en'
#lang = 'pt'
time = datetime.now().time()
print(time)
error = []
x = 0
try:
tso = TwitterSearchOrder() # create a TwitterSearchOrder object
tso.set_language(lang) # we want to see German tweets only
tso.set_include_entities(False) # and don't give us all those entity information
# it's about time to create a TwitterSearch object with our secret tokens
ts1 = TwitterSearch(
consumer_key = 'kTs2YF5jFUZwRwnfprYMmtHc4',
consumer_secret = 'ivn8IIaf9EByQgZ4wgvAU2nERl4J3uiuqziRWTg71ZnwOqKt3S',
access_token = '1323543967-Ws0vHHW5CC2MlGHYDvY8oFsrLEisLucl4kJ8PlO',
access_token_secret = 'WEQZDmz50jFmg81fjUcRQWUyPOCIUcxiWGez2UF1skk45',
)
ts2 = TwitterSearch(
consumer_key = 'KTlBBJPbcphOToiXMvN0ZWAkO',
consumer_secret = 'h5tOgnm8dt9c1lU21GdsSKPGNuBw1Zhhmlluo6bPiAucfDW0Fk',
access_token = '1323543967-mmSfNBFyMtvLArmsaTdwiWsfunq4OHCwWUc472y',
access_token_secret = 'jxEtyXlf5Gmh6S6awRgQ97TXsQxt5NUpxS4r8Ips1I4FH',
)
ts3 = TwitterSearch(
consumer_key = 'edGj6kTiDU1NOtdSyHHJ6yoD4',
consumer_secret = 'sGZmEV7xq4yx2ixwbhuBq6v1aCgRwuBF3IqIqhfYNLhWMokOML',
access_token = '1323543967-JcSeOiXMxND46nMj3mkx43J8SB4A3GBR24Pkfx2',
access_token_secret = 'zr19f231ZuV8D8gpo8asjnfEXI4gX0xWQ6AXk0kyMvimu',
)
ts4 = TwitterSearch(
consumer_key = 'j8l5JUzvtn9FB8SBuOdtKsifE',
consumer_secret = 'nKAEJnzgNn8G85bisr3wosZNKz61GnYZ3Nj2h8f6nLUP95K2L6',
access_token = '1323543967-8EX0CGfWERxnmAbh3efPPaqpM0Dz1r6ZGaHTpRY',
access_token_secret = '2Iga4V9dyvbR8AB3UmXd3NAqBinbJcMlu8PzNQhmoKS8O',
)
tss = [ts1, ts2, ts3, ts4]
except TwitterSearchException as e: # take care of all those ugly errors if there are some
print(e)
return
latest_id = tweets.find( {}, { 'object.tweet_id':1 } ).sort("startedAtTime").limit(1)
latest_id_str = latest_id[db_count-1]['object']['tweet_id']
latest_id_int = int(latest_id_str)
print 'Count of documents in the ' + keyword + ' collection is not 0. It is ' + str(db_count) + '. Mongo is now identifying the latest tweet ID to append as a parameter to the API call.'
# If ther are no documents in the collection, no queries are done, and the since_id is left out of the API call.
else:
print 'The Mongo collection ' + keyword + ' is empty. The script will now collect all tweets.'
# create a TwitterSearchOrder object
tso = TwitterSearchOrder()
# let's define all words we would like to have a look for
tso.set_keywords([keyword])
# Select language
tso.set_language('en')
# Include Entity information
tso.set_include_entities(True)
if db_count is not 0:
tso.set_since_id(latest_id_int)
print 'Since the document count in the ' + keyword + ' collection is above 0, the since_id uses the parameter of the latest tweet so that only new tweets are collected.'
else:
print 'No documents exist in the ' + keyword + ' collection right now so the since_id parameter will be empty and all tweets will be collected.'
# Create a TwitterSearch object with our secret tokens
ts = TwitterSearch(
consumer_key = config.get('Twitter', 'consumer_key'),
consumer_secret = config.get('Twitter', 'consumer_secret'),
def Tweets():
try:
max_feeds = 10
tso = TwitterSearchOrder() # create a TwitterSearchOrder object
tso.set_language('en')
tso.set_include_entities(
False) # and don't give us all those entity information
tso.set_until(new_date)
tso.arguments.update({'tweet_mode': 'extended'})
tso.arguments.update({'truncated': 'False'})
ts = TwitterSearch(
consumer_key='DMHjSht5U0UqNUsAWpZH9DXok',
consumer_secret=
'olCjsx8LltiHxEiPHWafExoibDuu4eZT48udXTeSYcQbLQ3juB',
access_token='1170976252213125121-ftEg9MzF9siFHUmcUkV6zzT7mQV9Db',
access_token_secret='eNA62T8Ig40Iz1wmKf6baDGHqY3Wh9kxzu9oaOQdGE9h8',
)
for c in range(len(MainDF)):
count = 0
#kw=[MainDF['twitter'][c]]
#for h in MainDF['hashtag'][c]:
# kw.append(h)
tso.set_keywords(MainDF['hashtag'][c])
tweets_list = []
tuo = TwitterUserOrder(MainDF['twitter'][c])
# tuo.set_language('en')
tuo.set_include_entities(
False) # and don't give us all those entity information
# tuo.set_until(days_ago)
# tuo.set_count(15)
tuo.arguments.update({'tweet_mode': 'extended'})
tuo.arguments.update({'truncated': 'False'})
#for tweet in ts.search_tweets_iterable(tso):
# print(tweet)
# tweets_list.append([tweet['user']['screen_name'],tweet['full_text']])
for tweet in ts.search_tweets_iterable(tso):
if 'retweeted_status' in tweet:
None
#tweets_list.append([tweet['user']['screen_name'],tweet['retweeted_status']['full_text'],'Retweet of ' + tweet['retweeted_status']['user']['screen_name']])
else:
links = Find(tweet['full_text'])
links = ', '.join(link for link in links)
#print(tweet)
tweets_list.append([
MainDF['company'][c], tweet['user']['screen_name'],
tweet['full_text'], tweet['created_at'], links
])
for tweet in ts.search_tweets_iterable(tuo):
if tweet['lang'] != 'en':
#print(tweet)
None
else:
# print(tweet)
links = Find(tweet['full_text'])
links = ', '.join(link for link in links)
tweets_list.append([
MainDF['company'][c], tweet['user']['screen_name'],
tweet['full_text'], tweet['created_at'], links
])
count = count + 1
if count == max_feeds:
break
if tweets_list != []:
tweets_datasets[MainDF['company'][c]] = pd.DataFrame(
tweets_list)
tweets_datasets[MainDF['company'][c]].columns = [
'Company', 'Source/User', 'Title/Tweet', 'Date', 'Link'
]
tweets_datasets[MainDF['company'][c]].insert(
0, 'Category', 'Twitter')
for i in range(
len(tweets_datasets[MainDF['company'][c]]['Date'])):
tweets_datasets[MainDF['company'][c]]['Date'][i] = parse(
tweets_datasets[MainDF['company'][c]]['Date'][i])
tweets_datasets[
MainDF['company'][c]]['Date'][i] = tweets_datasets[
MainDF['company'][c]]['Date'][i].date()
#print(datasets[companies_names[count]])
tw_current_companies.append(MainDF['company'][c])
else:
None
#tweets_list.append()
#print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'] ) )
except TwitterSearchException as e: # take care of all those ugly errors if there are some
print(e)