def tweets_user(user): limit = request.args.get('limit') if limit is None: limit = DEFAULT_LIMIT twitter_api = TwitterApi() json_response = twitter_api.getTweetsByUsername(user, limit) summary = twitter_api.extractTweetInfo(json_response) return json.dumps(summary)
def test_createTweetSearchUrl(self): twitter_api = TwitterApi() actual = twitter_api.createTweetSearchUrl('influencer', '5') expected = 'https://api.twitter.com/1.1/search/tweets.json?q=%23influencer&count=5' self.assertEqual(expected, actual)
def test_createAuthHeader(self): twitter_api = TwitterApi() actual = twitter_api.createAuthHeader() expected = {"Authorization": "Bearer bearer_token"} self.assertEqual(expected, actual)
def test_createUserTweetsUrl(self): twitter_api = TwitterApi() actual = twitter_api.createUserTweetsUrl('nasa', '3') expected = 'https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=nasa&count=3' self.assertEqual(expected, actual)
def tweets_hashtag(hashtag): limit = request.args.get('limit') if limit is None: limit = DEFAULT_LIMIT twitter_api = TwitterApi() json_response = twitter_api.getTweetsByHashtag(hashtag, limit) tweets = json_response['statuses'] summary = twitter_api.extractTweetInfo(tweets) return json.dumps(summary)
def __init__(self, thread_name, followings_queue, profiles_queue, visited_followings_queue, visited_profiles_queue): threading.Thread.__init__(self, name=thread_name) self.api = TwitterApi() self.db = Database() self.followings_queue = followings_queue self.profiles_queue = profiles_queue self.visited_followings_queue = visited_followings_queue self.visited_profiles_queue = visited_profiles_queue
def test_convertTwitterDate(self): twitter_api = TwitterApi() date_twitter = 'Sun Sep 27 15:50:28 +0000 2020' actual = twitter_api.convertTwitterDate(date_twitter) expected = '15:50 PM - 27 Sep 2020' self.assertEqual(expected, actual)
def test_extractHashtags(self): twitter_api = TwitterApi() hashtags = [{'text': 'three'}, {'text': 'seagrass'}] actual = twitter_api.extractHashtags(hashtags) expected = ['three', 'seagrass'] self.assertEqual(expected, actual)
def test_getTweetsByUsername_obtainResultFromApiSuccessfully( self, mocked_header, mocked_url, mocked_get): twitter_api = TwitterApi() mocked_url.return_value = "api_url" mocked_header.return_value = "header" json_response = "json_response" mocked_get.return_value.status_code = 200 mocked_get.return_value.text = json_response twitter_api.getTweetsByUsername('hashtag', '3') mocked_get.assert_called_with("api_url", headers="header")
def test_getTweetsByHashtag_handleExceptionOnApiBadRequest( self, mocked_header, mocked_url, mocked_get): twitter_api = TwitterApi() mocked_url.return_value = "api_url" mocked_header.return_value = "header" json_response = "json_response" mocked_get.return_value.status_code = 400 mocked_get.return_value.text = json_response with self.assertRaises(Exception): twitter_api.getTweetsByHashtag('hashtag', '3') mocked_get.assert_called_with("api_url", headers="header")
class ProfileThread(threading.Thread): def __init__(self, thread_name, profiles_queue, visited_profiles_queue): threading.Thread.__init__(self, name=thread_name) self.api = TwitterApi() self.db = Database() self.profiles_queue = profiles_queue self.visited_profiles_queue = visited_profiles_queue def run(self): while self.profiles_queue.qsize() >= 0: if self.profiles_queue.qsize() == 0: time.sleep(120) continue uid = self.profiles_queue.get() if self.db.is_existed(uid, "profile"): self.profiles_queue.task_done() print "skip ", uid continue print "Profiles...I am alive", uid user_profile = self.api.get_user_profile(uid) if user_profile == None: self.db.record_failure(failed_proile=uid) self.profiles_queue.task_done() else: self.db.insert_profile(user_profile) self.visited_profiles_queue.put(uid) #self.db.update_profile_progress(self.profiles_queue, self.visited_profiles_queue) self.profiles_queue.task_done() print "%s Profiles Finished:\t\t %d" % ( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.visited_profiles_queue.qsize()) print "%s Profiles Left:\t\t %d" % ( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.profiles_queue.qsize()) time.sleep(6)
class FollowingThread(threading.Thread): def __init__(self, thread_name, followings_queue, profiles_queue, visited_followings_queue, visited_profiles_queue): threading.Thread.__init__(self, name=thread_name) self.api = TwitterApi() self.db = Database() self.followings_queue = followings_queue self.profiles_queue = profiles_queue self.visited_followings_queue = visited_followings_queue self.visited_profiles_queue = visited_profiles_queue def run(self): while self.followings_queue.qsize() > 0: uid = self.followings_queue.get() if self.db.is_existed(uid, "following_ids"): self.followings_queue.task_done() print "skip ", uid continue if type(uid) is types.IntType: print "Followings...I am alive ", uid followings = self.api.get_user_followings(uid=uid) else: followings = self.api.get_user_followings(sname=uid) # download followings ids if followings == None: self.db.record_failure(failed_following=uid) # time.sleep(61) self.followings_queue.task_done() continue self.db.insert_following(uid, followings) self.visited_followings_queue.put(uid) # add ids to task queues followings_for_profiles_queue = self.exclude_processed_profiles( followings) followings_for_followings_queue = self.exclude_processed_followings( followings) for id in followings_for_profiles_queue: self.profiles_queue.put(id) for id in followings_for_followings_queue: self.followings_queue.put(id) #self.db.update_following_progress(self.followings_queue, self.visited_followings_queue) self.followings_queue.task_done() print "%s Followings Finished:\t %d" % ( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.visited_followings_queue.qsize()) print "%s Followings Left:\t\t %d" % ( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.followings_queue.qsize()) # time.sleep(61) def exclude_processed_profiles(self, followings): profiles_set = set(self.visited_profiles_queue.queue).union( set(self.profiles_queue.queue)) return set(followings).difference(profiles_set) def exclude_processed_followings(self, followings): followings_set = set(self.visited_followings_queue.queue).union( set(self.followings_queue.queue)) return set(followings).difference(followings_set)
def __init__(self): self.twitter_api = TwitterApi( consumer_key=CONFIG.get('consumer_key'), consumer_secret=CONFIG.get('consumer_secret'), oauth_token=CONFIG.get('oauth_token'), oauth_secret=CONFIG.get('oauth_secret'))
def test_extractTweetInfo(self): tweets = [{ "created_at": "Thu Apr 06 15:28:43 +0000 2017", "text": "tweet_text_first", "entities": { "hashtags": [{ "text": "first" }, { "text": "second" }], }, "user": { "id": 111, "name": "name_one", "screen_name": "screen_name_one", }, "retweet_count": 10, "favorite_count": 11, }, { "created_at": "Fri Apr 16 15:28:43 +0000 2017", "text": "tweet_text_second", "entities": { "hashtags": [{ "text": "one" }, { "text": "two" }], }, "user": { "id": 222, "name": "name_two", "screen_name": "screen_name_two", }, "retweet_count": 20, "favorite_count": 21, }] twitter_api = TwitterApi() actual = twitter_api.extractTweetInfo(tweets) expected = [ { 'account': { 'fullname': 'name_one', 'href': '/screen_name_one', 'id': 111, }, 'date': '15:28 PM - 06 Apr 2017', 'hashtags': ['first', 'second'], 'likes': 11, 'retweets': 10, 'text': 'tweet_text_first', }, { 'account': { 'fullname': 'name_two', 'href': '/screen_name_two', 'id': 222, }, 'date': '15:28 PM - 16 Apr 2017', 'hashtags': ['one', 'two'], 'likes': 21, 'retweets': 20, 'text': 'tweet_text_second', }, ] self.assertEqual(expected, actual)
def __init__(self): self.api = TwitterApi()
class Scorer(object): def __init__(self): self.api = TwitterApi() def score_document(self, doc): """ doc: created_at url title keywords """ max_id = None last_created_date = doc.created_at score = 0 while last_created_date >= doc.created_at: results = self.api.get_keyword_tweets(doc.keywords, max_id)['statuses'] print 'got {} tweets'.format(len(results)) print 'last date', last_created_date for tweet in results: score += self.compute_score(doc, tweet) max_id = int(tweet['id']) last_created_date = datetime_parse(tweet['created_at']) if last_created_date < doc.created_at: return score if len(results) == 0: return score max_id -= 1 time.sleep(4) # rate limit 450 per 30 minutes return score def compute_score(self, doc, tweet): """ score = (num keywords in text) * (1 + handle_followers / 100000) * (1 + favorites / 1000) * (1 + retweets / 100000) * (1 + 0.2 * (doc_url in tweet_urls)) * (1 + 0.5 * (title in tweet_text)) """ tweet_text = tweet['text'] tweet_urls = set(url['expanded_url'] for url in tweet['entities']['urls']) favorites = int(tweet['favorite_count']) retweets = int(tweet['retweet_count']) handle_followers = int(tweet['user']['followers_count']) keyword_occurrences = 0 for word in doc.keywords: keyword_occurrences += len( [o for o in re.finditer(word, tweet_text)]) score = \ (keyword_occurrences) * \ (1 + handle_followers / 100000) * \ (1 + favorites / 1000) * \ (1 + retweets / 100000) * \ (1 + 0.2 * (doc.url in tweet_urls)) * \ (1 + 0.5 * (1 if re.finditer(doc.title, tweet['text']) else 0)) return score
# -*- coding: utf-8 -*- import os import sys from settings import Settings from twitter_api import TwitterApi curdir = os.path.dirname(os.path.realpath(__file__)) s = Settings(os.path.join(curdir, "settings.ini")) twitter_api = TwitterApi( s.get_twitter_api_key(), s.get_twitter_api_secret(), s.get_twitter_access_token(), s.get_twitter_access_token_secret(), ) n = int(sys.argv[1]) timeline = twitter_api.twitter_api.user_timeline(count=n) for t in timeline: twitter_api.twitter_api.destroy_status(t.id) print("{0} tweets removed.".format(n))
import os import telebot from twitter_api import TwitterApi from twitter_downloader import TwitterDownloader from url_helper import UrlHelper SUPPORTED_DOMAINS = {'twitter': 'twitter.com'} bot = telebot.TeleBot(os.getenv('TELEGRAM_KEY')) url_helper = UrlHelper(SUPPORTED_DOMAINS.values()) twitter_downloader = TwitterDownloader( TwitterApi(os.getenv('TWITTER_API_KEY'), os.getenv('TWITTER_API_SECRET')), url_helper) @bot.message_handler(content_types=['text']) def handle_message(message): if not url_helper.is_url_valid(message.text): return bot.send_message(message.from_user.id, "Send me some url!") if not url_helper.is_supported_domain(message.text): supported_domains_formatted = ', '.join(SUPPORTED_DOMAINS.values()) return bot.send_message( message.from_user.id, 'This website is not supported :( I only support {}'.format( supported_domains_formatted)) send_video(message) def send_video(message):