def tweets_user(user):
limit = request.args.get('limit')
if limit is None:
limit = DEFAULT_LIMIT
twitter_api = TwitterApi()
json_response = twitter_api.getTweetsByUsername(user, limit)
summary = twitter_api.extractTweetInfo(json_response)
return json.dumps(summary)
def test_createTweetSearchUrl(self):
twitter_api = TwitterApi()
actual = twitter_api.createTweetSearchUrl('influencer', '5')
expected = 'https://api.twitter.com/1.1/search/tweets.json?q=%23influencer&count=5'
self.assertEqual(expected, actual)
def test_createAuthHeader(self):
twitter_api = TwitterApi()
actual = twitter_api.createAuthHeader()
expected = {"Authorization": "Bearer bearer_token"}
self.assertEqual(expected, actual)
def test_createUserTweetsUrl(self):
twitter_api = TwitterApi()
actual = twitter_api.createUserTweetsUrl('nasa', '3')
expected = 'https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=nasa&count=3'
self.assertEqual(expected, actual)
def tweets_hashtag(hashtag):
limit = request.args.get('limit')
if limit is None:
limit = DEFAULT_LIMIT
twitter_api = TwitterApi()
json_response = twitter_api.getTweetsByHashtag(hashtag, limit)
tweets = json_response['statuses']
summary = twitter_api.extractTweetInfo(tweets)
return json.dumps(summary)
def __init__(self, thread_name, followings_queue, profiles_queue,
visited_followings_queue, visited_profiles_queue):
threading.Thread.__init__(self, name=thread_name)
self.api = TwitterApi()
self.db = Database()
self.followings_queue = followings_queue
self.profiles_queue = profiles_queue
self.visited_followings_queue = visited_followings_queue
self.visited_profiles_queue = visited_profiles_queue
def test_convertTwitterDate(self):
twitter_api = TwitterApi()
date_twitter = 'Sun Sep 27 15:50:28 +0000 2020'
actual = twitter_api.convertTwitterDate(date_twitter)
expected = '15:50 PM - 27 Sep 2020'
self.assertEqual(expected, actual)
def test_extractHashtags(self):
twitter_api = TwitterApi()
hashtags = [{'text': 'three'}, {'text': 'seagrass'}]
actual = twitter_api.extractHashtags(hashtags)
expected = ['three', 'seagrass']
self.assertEqual(expected, actual)
def test_getTweetsByUsername_obtainResultFromApiSuccessfully(
self, mocked_header, mocked_url, mocked_get):
twitter_api = TwitterApi()
mocked_url.return_value = "api_url"
mocked_header.return_value = "header"
json_response = "json_response"
mocked_get.return_value.status_code = 200
mocked_get.return_value.text = json_response
twitter_api.getTweetsByUsername('hashtag', '3')
mocked_get.assert_called_with("api_url", headers="header")
def test_getTweetsByHashtag_handleExceptionOnApiBadRequest(
self, mocked_header, mocked_url, mocked_get):
twitter_api = TwitterApi()
mocked_url.return_value = "api_url"
mocked_header.return_value = "header"
json_response = "json_response"
mocked_get.return_value.status_code = 400
mocked_get.return_value.text = json_response
with self.assertRaises(Exception):
twitter_api.getTweetsByHashtag('hashtag', '3')
mocked_get.assert_called_with("api_url", headers="header")
class ProfileThread(threading.Thread):
def __init__(self, thread_name, profiles_queue, visited_profiles_queue):
threading.Thread.__init__(self, name=thread_name)
self.api = TwitterApi()
self.db = Database()
self.profiles_queue = profiles_queue
self.visited_profiles_queue = visited_profiles_queue
def run(self):
while self.profiles_queue.qsize() >= 0:
if self.profiles_queue.qsize() == 0:
time.sleep(120)
continue
uid = self.profiles_queue.get()
if self.db.is_existed(uid, "profile"):
self.profiles_queue.task_done()
print "skip ", uid
continue
print "Profiles...I am alive", uid
user_profile = self.api.get_user_profile(uid)
if user_profile == None:
self.db.record_failure(failed_proile=uid)
self.profiles_queue.task_done()
else:
self.db.insert_profile(user_profile)
self.visited_profiles_queue.put(uid)
#self.db.update_profile_progress(self.profiles_queue, self.visited_profiles_queue)
self.profiles_queue.task_done()
print "%s Profiles Finished:\t\t %d" % (
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
self.visited_profiles_queue.qsize())
print "%s Profiles Left:\t\t %d" % (
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
self.profiles_queue.qsize())
time.sleep(6)
class FollowingThread(threading.Thread):
def __init__(self, thread_name, followings_queue, profiles_queue,
visited_followings_queue, visited_profiles_queue):
threading.Thread.__init__(self, name=thread_name)
self.api = TwitterApi()
self.db = Database()
self.followings_queue = followings_queue
self.profiles_queue = profiles_queue
self.visited_followings_queue = visited_followings_queue
self.visited_profiles_queue = visited_profiles_queue
def run(self):
while self.followings_queue.qsize() > 0:
uid = self.followings_queue.get()
if self.db.is_existed(uid, "following_ids"):
self.followings_queue.task_done()
print "skip ", uid
continue
if type(uid) is types.IntType:
print "Followings...I am alive ", uid
followings = self.api.get_user_followings(uid=uid)
else:
followings = self.api.get_user_followings(sname=uid)
# download followings ids
if followings == None:
self.db.record_failure(failed_following=uid)
# time.sleep(61)
self.followings_queue.task_done()
continue
self.db.insert_following(uid, followings)
self.visited_followings_queue.put(uid)
# add ids to task queues
followings_for_profiles_queue = self.exclude_processed_profiles(
followings)
followings_for_followings_queue = self.exclude_processed_followings(
followings)
for id in followings_for_profiles_queue:
self.profiles_queue.put(id)
for id in followings_for_followings_queue:
self.followings_queue.put(id)
#self.db.update_following_progress(self.followings_queue, self.visited_followings_queue)
self.followings_queue.task_done()
print "%s Followings Finished:\t %d" % (
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
self.visited_followings_queue.qsize())
print "%s Followings Left:\t\t %d" % (
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
self.followings_queue.qsize())
# time.sleep(61)
def exclude_processed_profiles(self, followings):
profiles_set = set(self.visited_profiles_queue.queue).union(
set(self.profiles_queue.queue))
return set(followings).difference(profiles_set)
def exclude_processed_followings(self, followings):
followings_set = set(self.visited_followings_queue.queue).union(
set(self.followings_queue.queue))
return set(followings).difference(followings_set)
def __init__(self):
self.twitter_api = TwitterApi(
consumer_key=CONFIG.get('consumer_key'),
consumer_secret=CONFIG.get('consumer_secret'),
oauth_token=CONFIG.get('oauth_token'),
oauth_secret=CONFIG.get('oauth_secret'))
def test_extractTweetInfo(self):
tweets = [{
"created_at": "Thu Apr 06 15:28:43 +0000 2017",
"text": "tweet_text_first",
"entities": {
"hashtags": [{
"text": "first"
}, {
"text": "second"
}],
},
"user": {
"id": 111,
"name": "name_one",
"screen_name": "screen_name_one",
},
"retweet_count": 10,
"favorite_count": 11,
}, {
"created_at": "Fri Apr 16 15:28:43 +0000 2017",
"text": "tweet_text_second",
"entities": {
"hashtags": [{
"text": "one"
}, {
"text": "two"
}],
},
"user": {
"id": 222,
"name": "name_two",
"screen_name": "screen_name_two",
},
"retweet_count": 20,
"favorite_count": 21,
}]
twitter_api = TwitterApi()
actual = twitter_api.extractTweetInfo(tweets)
expected = [
{
'account': {
'fullname': 'name_one',
'href': '/screen_name_one',
'id': 111,
},
'date': '15:28 PM - 06 Apr 2017',
'hashtags': ['first', 'second'],
'likes': 11,
'retweets': 10,
'text': 'tweet_text_first',
},
{
'account': {
'fullname': 'name_two',
'href': '/screen_name_two',
'id': 222,
},
'date': '15:28 PM - 16 Apr 2017',
'hashtags': ['one', 'two'],
'likes': 21,
'retweets': 20,
'text': 'tweet_text_second',
},
]
self.assertEqual(expected, actual)
def __init__(self):
self.api = TwitterApi()
class Scorer(object):
def __init__(self):
self.api = TwitterApi()
def score_document(self, doc):
"""
doc:
created_at
url
title
keywords
"""
max_id = None
last_created_date = doc.created_at
score = 0
while last_created_date >= doc.created_at:
results = self.api.get_keyword_tweets(doc.keywords,
max_id)['statuses']
print 'got {} tweets'.format(len(results))
print 'last date', last_created_date
for tweet in results:
score += self.compute_score(doc, tweet)
max_id = int(tweet['id'])
last_created_date = datetime_parse(tweet['created_at'])
if last_created_date < doc.created_at:
return score
if len(results) == 0:
return score
max_id -= 1
time.sleep(4) # rate limit 450 per 30 minutes
return score
def compute_score(self, doc, tweet):
"""
score = (num keywords in text) *
(1 + handle_followers / 100000) *
(1 + favorites / 1000) *
(1 + retweets / 100000) *
(1 + 0.2 * (doc_url in tweet_urls)) *
(1 + 0.5 * (title in tweet_text))
"""
tweet_text = tweet['text']
tweet_urls = set(url['expanded_url']
for url in tweet['entities']['urls'])
favorites = int(tweet['favorite_count'])
retweets = int(tweet['retweet_count'])
handle_followers = int(tweet['user']['followers_count'])
keyword_occurrences = 0
for word in doc.keywords:
keyword_occurrences += len(
[o for o in re.finditer(word, tweet_text)])
score = \
(keyword_occurrences) * \
(1 + handle_followers / 100000) * \
(1 + favorites / 1000) * \
(1 + retweets / 100000) * \
(1 + 0.2 * (doc.url in tweet_urls)) * \
(1 + 0.5 * (1 if re.finditer(doc.title, tweet['text']) else 0))
return score
# -*- coding: utf-8 -*-
import os
import sys
from settings import Settings
from twitter_api import TwitterApi
curdir = os.path.dirname(os.path.realpath(__file__))
s = Settings(os.path.join(curdir, "settings.ini"))
twitter_api = TwitterApi(
s.get_twitter_api_key(),
s.get_twitter_api_secret(),
s.get_twitter_access_token(),
s.get_twitter_access_token_secret(),
)
n = int(sys.argv[1])
timeline = twitter_api.twitter_api.user_timeline(count=n)
for t in timeline:
twitter_api.twitter_api.destroy_status(t.id)
print("{0} tweets removed.".format(n))
import os
import telebot
from twitter_api import TwitterApi
from twitter_downloader import TwitterDownloader
from url_helper import UrlHelper
SUPPORTED_DOMAINS = {'twitter': 'twitter.com'}
bot = telebot.TeleBot(os.getenv('TELEGRAM_KEY'))
url_helper = UrlHelper(SUPPORTED_DOMAINS.values())
twitter_downloader = TwitterDownloader(
TwitterApi(os.getenv('TWITTER_API_KEY'), os.getenv('TWITTER_API_SECRET')),
url_helper)
@bot.message_handler(content_types=['text'])
def handle_message(message):
if not url_helper.is_url_valid(message.text):
return bot.send_message(message.from_user.id, "Send me some url!")
if not url_helper.is_supported_domain(message.text):
supported_domains_formatted = ', '.join(SUPPORTED_DOMAINS.values())
return bot.send_message(
message.from_user.id,
'This website is not supported :( I only support {}'.format(
supported_domains_formatted))
send_video(message)
def send_video(message):