def resolve_screen_name_for_export(tw_context, screen_name_to_export): oauth_token = tw_context['oauth_final_token'] oauth_token_secret = tw_context['oauth_final_token_secret'] twitter = Twython(app_key=APP_KEY, app_secret=APP_SECRET, oauth_token=oauth_token, oauth_token_secret=oauth_token_secret) try: u = twitter.show_user(screen_name=screen_name_to_export) # test actual access to the list of friends of the user to export data for twitter.get_friends_list( screen_name=screen_name_to_export, skip_status=True, include_user_entities=False, count=5) except TwythonError as te: msg = resolve_twitter_error(te) msg = f"{screen_name_to_export}: {msg}" return False, None, msg resolved_screen_name = u['screen_name'] friends_count = u['friends_count'] if friends_count == 0: return False, None, f"Profile {screen_name_to_export} doesn't have any friends to export" elif friends_count > MAX_FRIENDS: err_msg_for_user = f"Profile {screen_name_to_export} has {friends_count}, we support a maximum of {MAX_FRIENDS}" return False, None, err_msg_for_user else: return True, resolved_screen_name, None
class TwitterCrawlerFollow: users_expanded=[] def __init__(self, file_name): self.output_file=open(file_name, "w") self.twitter_connection=Twython(con_key, access_token=acc_token) def start_crawl(self, search_term, node_limit): data={} while True: try: data = self.twitter_connection.search(q=search_term, lang="en", result_type="recent", count=1) break except TwythonError as e: print e if str(429) in e.msg: time.sleep(900) self.twitter_connection = Twython(con_key, access_token=acc_token) def expand_user(self, user, depth, max_depth): friends=self.twitter_connection.get_friends_list(user_id=user, count=200) for friend in friends: if depth<max_depth: if user not in self.users_expanded: self.users_expanded.append(friend) self.expand_user(friend, depth+1, max_depth) else: return
def fetch_friends(self, user): """ fetches the friends from twitter using the information on django-social-auth models user is an instance of UserSocialAuth Returns: collection of friend objects fetched from Twitter """ # now fetch the twitter friends using `twython` auth_data = self._auth_data(user) tw = Twython(**auth_data) cursor = -1 friends = [] while True: data = tw.get_friends_list(cursor=cursor) friends += data.get('users', []) next_cursor = data.get('next_cursor', 0) prev_cursor = data.get('prev_cursor', 0) if not next_cursor or next_cursor == prev_cursor: break else: cursor = next_cursor return friends
def twitter_users(request): try: user = Twython(app_key=CONSUMER_KEY, app_secret=CONSUMER_SECRET, oauth_token=ACCESS_TOKEN, oauth_token_secret=ACCESS_TOKEN_SECRET ) search_results = user.get_friends_list(count=10) users = search_results['users'] user_names = [usr['name'] for usr in users] return render(request, 'users.html', {'data': user_names}) except: pass
def getFollowing(screenName, userCount = 200,skipStatus = True): twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) results = twitter.get_friends_list(screen_name=screenName, count=userCount, skip_status=skipStatus) if userCount <= 5: for result in results['users']: print 'ID: ', result['id'] print 'Screen Handle: ', result['screen_name'] print '===================//===================' print ' ' else: fp = codecs.open('.\Results\\'+currentTime()+' (FOLLOWING).txt', 'w', "utf-8") for result in results['users']: fp.write('ID: '+str(result['id'])+'\r\n') fp.write('Screen Handle: '+result['screen_name']+'\r\n') fp.write('===================//===================\r\n')
def get_following(log, id): log.debug(" Getting people %s is following", id) check_twitter_config() logging.captureWarnings(True) old_level = log.getEffectiveLevel() log.setLevel(logging.ERROR) twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) log.setLevel(old_level) cursor = -1 max_loops = 15 while cursor != 0: try: log.setLevel(logging.ERROR) following = twitter.get_friends_list(screen_name=id, cursor=cursor, count=200) log.setLevel(old_level) except TwythonAuthError, e: log.exception(" Problem trying to get people following") twitter_auth_issue(e) raise except:
def get_following(log, id): log.debug(" Getting people %s is following", id) check_twitter_config() logging.captureWarnings(True) old_level = log.getEffectiveLevel() log.setLevel(logging.ERROR) twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) log.setLevel(old_level) cursor = -1 max_loops = 15 while cursor != 0: try: log.setLevel(logging.ERROR) following = twitter.get_friends_list(screen_name=id, cursor=cursor, count=200) log.setLevel(old_level) except TwythonAuthError as e: log.exception(" Problem trying to get people following") twitter_auth_issue(e) raise except: raise for u in following["users"]: yield u["screen_name"] cursor = following["next_cursor"] if cursor: s = random.randint(55, 65) log.debug(" Sleeping %ds to avoid rate limit. Cursor: %s", s, cursor) time.sleep(s) else: log.debug(" Normal query end") max_loops -= 1 if max_loops <= 0: log.debug(" Killing search due to max loops") break log.setLevel(old_level)
def fetch_friends(self, user): """ fetches the friends from twitter Returns: collection of friend objects fetched from Twitter """ auth_data = self._auth_data(user) tw = Twython(**auth_data) cursor = -1 friends = [] while True: data = tw.get_friends_list(cursor=cursor) friends += data.get('users', []) next_cursor = data.get('next_cursor', 0) prev_cursor = data.get('prev_cursor', 0) if not next_cursor or next_cursor == prev_cursor: break else: cursor = next_cursor return friends
def get(self, request, *args, **kwargs): APP_KEY = 'Yf8q3TPgL59StYj8KRlsAMmLu' APP_SECRET = 'KuMfl4Kvu0Y5W8qVYbusiu6L5x78wK3jpDJV1WAPfcjEy6zAvA' OAUTH_TOKEN = "198514629-mTKORfs9NmA29kfbnxrU9bCzyc8NvrzFDP9XLiTw" OAUTH_TOKEN_SECRET = 'r9u7SJPrc7u7Vy9pz66uMrfcBSzL8he6HRHprJgEC6bGQ' twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) try: search_term = request.GET.get('srch-term', 'twitter') search_results = twitter.search(q=search_term, count=10) except TwythonError as e: print e my_updates = twitter.get_friends_list(count=500) my_updates = my_updates['users'] the_type = type(my_updates) return render( request, "catalog/twitter.html", { 'search_results': search_results['statuses'], 'my_updates': my_updates, 'the_type': the_type })
def get(self, request, *args, **kwargs): APP_KEY = 'Yf8q3TPgL59StYj8KRlsAMmLu' APP_SECRET = 'KuMfl4Kvu0Y5W8qVYbusiu6L5x78wK3jpDJV1WAPfcjEy6zAvA' OAUTH_TOKEN = "198514629-mTKORfs9NmA29kfbnxrU9bCzyc8NvrzFDP9XLiTw" OAUTH_TOKEN_SECRET = 'r9u7SJPrc7u7Vy9pz66uMrfcBSzL8he6HRHprJgEC6bGQ' twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) try: search_term = request.GET.get('srch-term', 'twitter') search_results = twitter.search(q=search_term, count=10) except TwythonError as e: print e my_updates = twitter.get_friends_list(count=500) my_updates = my_updates['users'] the_type = type(my_updates) return render(request, "catalog/twitter.html", { 'search_results': search_results['statuses'], 'my_updates': my_updates, 'the_type': the_type })
class TwythonAPITestCase(unittest.TestCase): def setUp(self): self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret) # Timelines def test_get_mentions_timeline(self): '''Test returning mentions timeline for authenticated user succeeds''' self.api.get_mentions_timeline() def test_get_user_timeline(self): '''Test returning timeline for authenticated user and random user succeeds''' self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): '''Test returning a protected user timeline who you are following succeeds''' self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): '''Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError''' self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_get_home_timeline(self): '''Test returning home timeline for authenticated user succeeds''' self.api.get_home_timeline() # Tweets def test_get_retweets(self): '''Test getting retweets of a specific tweet succeeds''' self.api.get_retweets(id=test_tweet_id) def test_show_status(self): '''Test returning a single status details succeeds''' self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): '''Test updating and deleting a status succeeds''' status = self.api.update_status(status='Test post just to get deleted :(') self.api.destroy_status(id=status['id_str']) def test_retweet(self): '''Test retweeting a status succeeds''' retweet = self.api.retweet(id='99530515043983360') self.api.destroy_status(id=retweet['id_str']) def test_retweet_twice(self): '''Test that trying to retweet a tweet twice raises a TwythonError''' retweet = self.api.retweet(id='99530515043983360') self.assertRaises(TwythonError, self.api.retweet, id='99530515043983360') # Then clean up self.api.destroy_status(id=retweet['id_str']) def test_get_oembed_tweet(self): '''Test getting info to embed tweet on Third Party site succeeds''' self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): '''Test getting ids for people who retweeted a tweet succeeds''' self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): '''Test searching tweets succeeds''' self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): '''Test getting the authenticated users direct messages succeeds''' self.api.get_direct_messages() def test_get_sent_messages(self): '''Test getting the authenticated users direct messages they've sent succeeds''' self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): '''Test sending, getting, then destory a direct message succeeds''' message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d!') self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): '''Test sending a direct message to someone who doesn't follow you fails''' self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man!') # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): '''Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds''' self.api.get_user_ids_of_blocked_retweets(stringify_ids='true') def test_get_friends_ids(self): '''Test returning ids of users the authenticated user and then a random user is following succeeds''' self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): '''Test returning ids of users the authenticated user and then a random user are followed by succeeds''' self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): '''Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds''' self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): '''Test returning incoming friendship ids succeeds''' self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): '''Test returning outgoing friendship ids succeeds''' self.api.get_outgoing_friendship_ids() def test_create_friendship(self): '''Test creating a friendship succeeds''' self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): '''Test destroying a friendship succeeds''' self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): '''Test updating friendships succeeds''' self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets='false') def test_show_friendships(self): '''Test showing specific friendship succeeds''' self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): '''Test getting list of users authenticated user then random user is following succeeds''' self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): '''Test getting list of users authenticated user then random user are followed by succeeds''' self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): '''Test getting the authenticated user account settings succeeds''' self.api.get_account_settings() def test_verify_credentials(self): '''Test representation of the authenticated user call succeeds''' self.api.verify_credentials() def test_update_account_settings(self): '''Test updating a user account settings succeeds''' self.api.update_account_settings(lang='en') def test_update_delivery_service(self): '''Test updating delivery settings fails because we don't have a mobile number on the account''' self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): '''Test updating profile succeeds''' self.api.update_profile(include_entities='true') def test_update_profile_colors(self): '''Test updating profile colors succeeds''' self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): '''Test listing users who are blocked by the authenticated user succeeds''' self.api.list_blocks() def test_list_block_ids(self): '''Test listing user ids who are blocked by the authenticated user succeeds''' self.api.list_block_ids() def test_create_block(self): '''Test blocking a user succeeds''' self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): '''Test unblocking a user succeeds''' self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): '''Test listing a number of user objects succeeds''' self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): '''Test showing one user works''' self.api.show_user(screen_name='twitter') def test_search_users(self): '''Test that searching for users succeeds''' self.api.search_users(q='Twitter API') def test_get_contributees(self): '''Test returning list of accounts the specified user can contribute to succeeds''' self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): '''Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account''' self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): '''Test removing profile banner succeeds''' self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): '''Test getting list of profile banner sizes fails because we have not uploaded a profile banner''' self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): '''Test getting user suggestions by slug succeeds''' self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): '''Test getting user suggestions succeeds''' self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): '''Test getting status of suggested users succeeds''' self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): '''Test getting list of favorites for the authenticated user succeeds''' self.api.get_favorites() def test_create_and_destroy_favorite(self): '''Test creating and destroying a favorite on a tweet succeeds''' self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): '''Test show lists for specified user''' self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): '''Test timeline of tweets authored by members of the specified list succeeds''' self.api.get_list_statuses(list_id=test_list_id) def test_create_update_destroy_list_add_remove_list_members(self): '''Test create a list, adding and removing members then deleting the list succeeds''' the_list = self.api.create_list(name='Stuff') list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed') # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name='johncena,xbox') self.api.delete_list_members(list_id=list_id, screen_name='johncena,xbox') # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_memberships(self): '''Test list of lists the authenticated user is a member of succeeds''' self.api.get_list_memberships() def test_get_list_subscribers(self): '''Test list of subscribers of a specific list succeeds''' self.api.get_list_subscribers(list_id=test_list_id) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): '''Test subscribing, is a list sub and unsubbing to list succeeds''' self.api.subscribe_to_list(list_id=test_list_id) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(list_id=test_list_id, screen_name=screen_name) self.api.unsubscribe_from_list(list_id=test_list_id) def test_is_list_member(self): '''Test returning if specified user is member of a list succeeds''' # Returns 404 if not list member self.api.is_list_member(list_id=test_list_id, screen_name='jack') def test_get_list_members(self): '''Test listing members of the specified list succeeds''' self.api.get_list_members(list_id=test_list_id) def test_get_specific_list(self): '''Test getting specific list succeeds''' self.api.get_specific_list(list_id=test_list_id) def test_get_list_subscriptions(self): '''Test collection of the lists the specified user is subscribed to succeeds''' self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): '''Test collection of lists the specified user owns succeeds''' self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): '''Test getting list of saved searches for authenticated user succeeds''' self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): '''Test getting list of saved searches for authenticated user succeeds''' saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): '''Test getting info about a geo location succeeds''' self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): '''Test reversing geocode succeeds''' self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): '''Test search for places that can be attached to a statuses/update succeeds''' self.api.search_geo(query='Toronto') def test_get_similar_places(self): '''Test locates places near the given coordinates which are similar in name succeeds''' self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): '''Test getting the top 10 trending topics for a specific WOEID succeeds''' self.api.get_place_trends(id=1) def test_get_available_trends(self): '''Test returning locations that Twitter has trending topic information for succeeds''' self.api.get_available_trends() def test_get_closest_trends(self): '''Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds''' self.api.get_closest_trends(lat='37', long='-122')
tweets = twitter.get_home_timeline(count=1) for tweet in tweets: print(tweet['text']) print print "My Settings" print "===========" settings = twitter.get_account_settings() #print settings print "I am @%s" % settings['screen_name'] print print "My friends" print "==========" list = twitter.get_friends_list() for friend in list['users']: print "I follow user @%s, name %s, id %d" % ( friend['screen_name'], friend['name'], friend['id']) print print "My followers" print "============" list = twitter.get_followers_list() for friend in list['users']: print "User @%s, name %s, id %d is following me" % ( friend['screen_name'], friend['name'], friend['id']) #print friend
azi = datetime.datetime.now().strftime("%Y-%m-%d") ieri = datetime.datetime.now() - datetime.timedelta(days=1) ieri.strftime("%Y-%m-%d") datestamp = datetime.datetime.now().strftime("%Y-%m-%d") raport = "raport-friends-" + username + "-" + str(azi) + ".html" tweetsdata = {} index = 1 next_cursor = -1 save = open(raport, 'w', encoding="utf-8") save.write( "<!DOCTYPE html> <html lang=\"en\"> <head> <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>" ) while (next_cursor): get_friends = twitter.get_friends_list(screen_name=username, count=200, cursor=next_cursor) for friend in get_friends["users"]: location = friend["location"] location = location.replace("'", " ") nume = friend["name"] nume = nume.replace("'", " ") try: time.sleep(sleeptime) get_tweets = twitter.get_user_timeline( screen_name=friend['screen_name'], count=1, include_rts=True) for x in get_tweets: x['text'] = Twython.html_for_tweet(x) if (verbose > 0): print( str(index) + " - <a href=\"https://twitter.com/" +
class TwitterClient(): def __init__(self, file_name='cred.txt'): with open(file_name) as f: lines = f.readlines() self.APP_KEY = lines[0].strip() self.APP_SECRET = lines[1].strip() if len(lines)>3: self.USER_TOKEN = lines[2].strip() self.USER_SECRET = lines[3].strip() else: self.USER_TOKEN = None self.USER_SECRET = None self.reset() def reset(self): self.twitter = Twython(self.APP_KEY, self.APP_SECRET, self.USER_TOKEN, self.USER_SECRET) def get_timeline(self, screen_name=None, user_id=None): all_tweets = [] results = [] next_max_id = None print('Extracting timeline of: @{}'.format(screen_name or user_id)) while next_max_id==None or (results!=None and len(results) != 0): try: results = self.try_call(lambda: self.twitter.get_user_timeline(screen_name=screen_name, user_id=user_id, count=200, max_id=next_max_id, tweet_mode='extended')) except: #something went wrong: internet or auth break if results: next_max_id = results[-1]['id'] - 1 all_tweets+=results else: break print('Total tweets: {}'.format(len(all_tweets))) timeline_df = pd.DataFrame(all_tweets) if len(timeline_df)>0: timeline_df['created_at']=pd.to_datetime(timeline_df['created_at']) return timeline_df def get_likes(self, screen_name=None, user_id=None): all_tweets = [] results = [] next_max_id = None print('Extracting likes of: @{}'.format(screen_name or user_id)) while next_max_id==None or (results!=None and len(results) != 0): try: results = self.try_call(lambda: self.twitter.get_favorites(screen_name=screen_name, user_id=user_id, count=200, max_id=next_max_id, tweet_mode='extended')) except: #something went wrong: internet or auth break if results: next_max_id = results[-1]['id'] - 1 all_tweets+=results else: break print('Total likes: {}'.format(len(all_tweets))) likes_df = pd.DataFrame(all_tweets) if len(likes_df)>0: likes_df['created_at']=pd.to_datetime(likes_df['created_at']) return likes_df def get_friends_list(self, screen_name=None, user_id=None): all_friends = [] next_cursor = -1 response = None print('Extracting friends of: @{}'.format(screen_name or user_id)) while next_cursor!=0 and (next_cursor==-1 or (response!=None and len(response['users']) != 0)): print(next_cursor, response and len(response['users']), len(all_friends)) time.sleep(5) try: response = self.try_call(lambda: self.twitter.get_friends_list(screen_name=screen_name, user_id=user_id, count=200, cursor=next_cursor)) except: #something went wrong: internet or auth break if response: next_cursor = response['next_cursor'] all_friends+=response['users'] else: break print('Total friends: {}'.format(len(all_friends))) friends_df = pd.DataFrame(all_friends) return friends_df def get_followers_list(self, screen_name=None, user_id=None): all_friends = [] next_cursor = -1 response = None print('Extracting followers of: @{}'.format(screen_name or user_id)) while next_cursor!=0 and (next_cursor==-1 or (response!=None and len(response['users']) != 0)): print(next_cursor, response and len(response['users']), len(all_friends)) time.sleep(5) try: response = self.try_call(lambda: self.twitter.get_followers_list(screen_name=screen_name, user_id=user_id, count=200, cursor=next_cursor)) except: #something went wrong: internet or auth break if response: next_cursor = response['next_cursor'] all_friends+=response['users'] else: break print('Total followers: {}'.format(len(all_friends))) friends_df = pd.DataFrame(all_friends) return friends_df def show_users(self, screen_names): dict_user_details = {} for screen_name in screen_names: dict_user_details[screen_name] = self.try_call(lambda: self.twitter.show_user(screen_name=screen_name, include_entities=False)) return dict_user_details def show_users_ids(self, ids): dict_user_details = {} for id_u in ids: dict_user_details[id_u] = self.try_call(lambda: self.twitter.show_user(user_id=id_u, include_entities=False)) return dict_user_details def try_call(self, call, deep=1, throw_errors = False): try: response = call() return response except TwythonRateLimitError as e: time_to_wait = int(self.twitter.get_lastfunction_header('x-rate-limit-reset')) - int(time.time()) + 10 print('Rate limit exceded. {}th try. Waiting {} seconds: {}'.format(deep, time_to_wait, time.strftime('%X', time.localtime(time.time()+time_to_wait)))) time.sleep(time_to_wait) return self.try_call(call, deep=deep+1) except TwythonAuthError as e: print('Auth error') print(str(e)) if throw_errors: raise return except TwythonError as e: if deep>=6: return print('No internet. Waiting {} seconds'.format(10)) print(str(e)) if throw_errors: raise #time.sleep(10) return #self.try_call(call, deep=deep+1)
oauth_token_secret=OAUTH_TOKEN_SECRET, ) # https://twython.readthedocs.io/en/latest/api.html # get the screen names I follow # can only make 15 requests in a 15-minute window (1 per minute) # https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-list i_follow = [] cursor_if = -1 sleep_seconds = 60 counter = 1 while True: logger.info(f"Query {counter}, I follow cursor: {cursor_if}") result = twitter.get_friends_list(screen_name=MY_SCREEN_NAME, count=200, skip_status="true", cursor=cursor_if) if len(result["users"]) == 0: break else: counter += 1 user_list = [user["screen_name"] for user in result["users"]] i_follow.extend(user_list) cursor_if = result["next_cursor"] logger.info( f"Added {len(user_list)} users who I follow (total: {len(i_follow)})") # # find the screen names with notifications turned on # user_list_notif = [ # user["screen_name"] for user in ifollow["users"] if user["notifications"] # ] # logger.info(
friendslen.append(len(friends)) followerslen.append(len(followers)) # Calculate time for Twython start = time.time() twython_consumerkey = '2IiwQDV5c4OeUEA8PI0aqvg5g' twython_consumersecret = 'RqEDv68Brt3JKVetWAeNFUObP5VeeRloBbQ0V9eqRFCerbas1I' twitter = Twython(twython_consumerkey, twython_consumersecret, oauth_version=2) twython_accesstoken = twitter.obtain_access_token() twitter = Twython(twython_consumerkey, access_token=twython_accesstoken) friends = twitter.get_friends_list(screen_name=u, count=n) followers = twitter.get_followers_list(screen_name=u, count=n) end = time.time() twittersec.append(end - start) friendslen.append(len(friends)) followerslen.append(len(followers)) # Calculating time for TweetPony start = time.time() api = tweetpony.API( consumer_key="s08vLRIM5VnqCUqFVYTaV3ET9", consumer_secret="q4N5vB5KQrakhjAcWAk8dKOeTn4896cURoAg33A37UjGG2jJxf",
class Tweety(object): """ Twitter client using Twython """ def __init__(self, app_key=TWITTER_APP_KEY, app_secret=TWITTER_APP_SECRET, oauth_token=TWITTER_OAUTH_TOKEN, oauth_token_secret=TWITTER_OAUTH_SECRET): try: self.twitter = Twython( app_key, app_secret, oauth_token, oauth_token_secret) self.cache = RedisCache({ 'server': REDIS_SERVER, 'port': REDIS_PORT, 'database': REDIS_DATABASE, 'key_prefix': 'domus-twitter' }) except TwythonAuthError: raise Exception("Unable to connect to twitter") def __get_friends(self): """ Using twitter get_friends and redis, gets a list of screen names :return:a list of twitter users """ results = self.cache.get('twitter_friends') if not results: try: results = [a['screen_name'] for a in self.twitter.get_friends_list()['users']] self.cache.store('twitter_friends',json.dumps(results), expires=120) except (TwythonError, TwythonRateLimitError): raise Exception('Unable to get followers list') else: results = json.loads(results) return results def tweet(self, message,to_friends=False): """ Writtes a twit :param message: what to tweet :param to_friends: send to all friends? :return: """ try: if to_friends: for a_friend in self.__get_friends(): mention = "@{} ".format(a_friend) available_chars = 140 - len(mention) self.twitter.update_status( status=(mention+message)[:available_chars] + ((mention+message)[(available_chars-2):] and '..')) else: self.twitter.update_status( status=message[:140] + (message[138:] and '..')) except (TwythonError, TwythonRateLimitError): raise Exception("Unable to post update") def dm(self, user, message): try: self.twitter.send_direct_message(screen_name=user, text=message) except TwythonError: raise Exception("Unable to send dm to {}".format(user)) def get_dms(self): """ Gets a list of dms. Stores the last id seen so the next request is for the new messages only. :return: a dict of the form {tweet_id:{sender:screen_name,text:the_message}} """ results = {} dms = [] last_id = self.cache.get('twitter_last_dm') if last_id: dms = self.twitter.get_direct_messages(count=100,since_id = last_id) else: dms = self.twitter.get_direct_messages(count=100) if dms: last_id = 0 for a_dm in dms: results[a_dm['id']] = {'from':a_dm['sender_screen_name'],'text': a_dm['text']} last_id = a_dm['id'] if a_dm['id'] > last_id else last_id self.cache.store('twitter_last_dm', last_id) return results def get_mentions(self): """ Gets a list of mentions. Stores the last id seen so the next request is for the new messages only. :return: a dict of the form {tweet_id:{sender:screen_name,text:the_message}} """ results = {} mentions = [] last_id = self.cache.get('twitter_last_mention') if last_id: mentions = self.twitter.get_mentions_timeline(count=100,since_id = last_id) else: mentions = self.twitter.get_mentions_timeline(count=100) if mentions: last_id = 0 for a_mention in mentions: results[a_mention['id']] = {'from':a_mention['user']['screen_name'],'text': a_mention['text']} last_id = a_mention['id'] if a_mention['id'] > last_id else last_id self.cache.store('twitter_last_mention', last_id) return results
class TwythonEndpointsTestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) # Timelines @unittest.skip('skipping non-updated test') def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() @unittest.skip('skipping non-updated test') def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) @unittest.skip('skipping non-updated test') def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() @unittest.skip('skipping non-updated test') def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets @unittest.skip('skipping non-updated test') def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) @unittest.skip('skipping non-updated test') def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') @unittest.skip('skipping non-updated test') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search @unittest.skip('skipping non-updated test') def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages @unittest.skip('skipping non-updated test') def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() @unittest.skip('skipping non-updated test') def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() @unittest.skip('skipping non-updated test') def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) @unittest.skip('skipping non-updated test') def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time())) # Friends & Followers @unittest.skip('skipping non-updated test') def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) @unittest.skip('skipping non-updated test') def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') @unittest.skip('skipping non-updated test') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() @unittest.skip('skipping non-updated test') def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() @unittest.skip('skipping non-updated test') def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) @unittest.skip('skipping non-updated test') def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users @unittest.skip('skipping non-updated test') def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() @unittest.skip('skipping non-updated test') def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() @unittest.skip('skipping non-updated test') def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') @unittest.skip('skipping non-updated test') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') @unittest.skip('skipping non-updated test') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') @unittest.skip('skipping non-updated test') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') @unittest.skip('skipping non-updated test') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() @unittest.skip('skipping non-updated test') def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() @unittest.skip('skipping non-updated test') def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') @unittest.skip('skipping non-updated test') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') @unittest.skip('skipping non-updated test') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') @unittest.skip('skipping non-updated test') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) @unittest.skip('skipping non-updated test') def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() @unittest.skip('skipping non-updated test') def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users @unittest.skip('skipping non-updated test') def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') @unittest.skip('skipping non-updated test') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() @unittest.skip('skipping non-updated test') def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites @unittest.skip('skipping non-updated test') def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() @unittest.skip('skipping non-updated test') def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists @unittest.skip('skipping non-updated test') def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) @unittest.skip('skipping non-updated test') def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() @unittest.skip('skipping non-updated test') def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') @unittest.skip('skipping non-updated test') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches @unittest.skip('skipping non-updated test') def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() @unittest.skip('skipping non-updated test') def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo @unittest.skip('skipping non-updated test') def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') @unittest.skip('skipping non-updated test') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') @unittest.skip('skipping non-updated test') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') @unittest.skip('skipping non-updated test') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends @unittest.skip('skipping non-updated test') def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) @unittest.skip('skipping non-updated test') def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() @unittest.skip('skipping non-updated test') def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help @unittest.skip('skipping non-updated test') def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() @unittest.skip('skipping non-updated test') def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() @unittest.skip('skipping non-updated test') def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() @unittest.skip('skipping non-updated test') def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() @unittest.skip('skipping non-updated test') def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
class TwythonAPITestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_get(self): """Test Twython generic GET request works""" self.api.get('account/verify_credentials') def test_post(self): """Test Twython generic POST request works, with a full url and with just an endpoint""" update_url = 'https://api.twitter.com/1.1/statuses/update.json' status = self.api.post(update_url, params={'status': 'I love Twython!'}) self.api.post('statuses/destroy/%s' % status['id_str']) def test_get_lastfunction_header(self): """Test getting last specific header of the last API call works""" self.api.get('statuses/home_timeline') self.api.get_lastfunction_header('x-rate-limit-remaining') def test_get_lastfunction_header_not_present(self): """Test getting specific header that does not exist from the last call returns None""" self.api.get('statuses/home_timeline') header = self.api.get_lastfunction_header('does-not-exist') self.assertEqual(header, None) def test_get_lastfunction_header_no_last_api_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') def test_search_gen(self): """Test looping through the generator results works, at least once that is""" search = self.api.search_gen('twitter', count=1) counter = 0 while counter < 2: counter += 1 result = next(search) new_id_str = int(result['id_str']) if counter == 1: prev_id_str = new_id_str time.sleep(1) # Give time for another tweet to come into search if counter == 2: self.assertTrue(new_id_str > prev_id_str) def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR exapanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text) def test_raise_error_on_bad_ssl_cert(self): """Test TwythonError is raised by a RequestException when an actual HTTP happens""" self.assertRaises(TwythonError, self.api.get, 'https://example.com') # Timelines def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get deleted :(') self.api.destroy_status(id=status['id_str']) def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man!') # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(list_id=test_list_id) def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff') list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed') screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(list_id=test_list_id) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(list_id=test_list_id) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(list_id=test_list_id, screen_name=screen_name) self.api.unsubscribe_from_list(list_id=test_list_id) def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(list_id=test_list_id, screen_name='jack') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(list_id=test_list_id) def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(list_id=test_list_id) def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
tweets = twitter.get_home_timeline(count=1) for tweet in tweets: print(tweet['text']) print print "My Settings" print "===========" settings = twitter.get_account_settings() #print settings print "I am @%s" % settings['screen_name'] print print "My friends" print "==========" list = twitter.get_friends_list() for friend in list['users']: print "I follow user @%s, name %s, id %d" % (friend['screen_name'], friend['name'], friend['id']) print print "My followers" print "============" list = twitter.get_followers_list() for friend in list['users']: print "User @%s, name %s, id %d is following me" % ( friend['screen_name'], friend['name'], friend['id']) #print friend
class Crawler(object): """ Crawler class """ # initial def __init__(self): # when load normal json, disable print("...") # own cursor reader def cursorReader(self, query, count, max_id, language, geocode, result_type, until): n = 2 # the deep of loop for checking cursor for i in range(0, n): print("searching with max_id parameter: " + str(max_id)) result = self._tw.search(q=query, max_id=max_id, count=count, lang=language, geocode=geocode, result_type=result_type, until=until, tweet_mode="extended") # print(result["search_metadata"]) metadata = result["search_metadata"].get("next_results") if metadata: max_id = furl(metadata).args['max_id'] print("\n======================================\n") print("found new max_id: " + str(max_id)) return max_id break else: print("can't find max_id in metadata... retrying..") print("searching with since_id parameter: " + str(max_id)) result = self._tw.search(q=query, since_id=max_id, count=count, lang=language, geocode=geocode, result_type=result_type, until=until, tweet_mode="extended") max_id = result['statuses'][-1].get("id") print("retying using tweet id: " + str(max_id)) time.sleep(random.randint(5, 10)) # last check if i == n - 1: result = self._tw.search(q=query, count=count, lang=language, geocode=geocode, result_type=result_type, until=until, tweet_mode="extended") max_id = result['statuses'][0].get("id") print("udahan mz, kaga nemu apa-apa... pake ini aje.." + str(max_id) + " atulah.. :(") return max_id break else: pass # search handling def searchHandler(self, query, count, querytype, language, geocode, result_type, until): sequence = 0 max_id = 0 while True: print("current sequence: " + str(sequence)) randomizer = random.randint(0, len(tokens) - 1) keys = keyRotator().rotator(sequence=randomizer) consumer_key, consumer_key_secret, access_token, access_token_secret = keys print("using key: " + str(consumer_key) + "\n") try: self._tw_auth = Twython(consumer_key, consumer_key_secret, oauth_version=2) OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token() self._tw = Twython(consumer_key, access_token=OAUTH2_ACCESS_TOKEN) except TwythonError as e: print("error on Twython!") result = self._tw.search(q=query, max_id=max_id, count=count, lang=language, geocode=geocode, result_type=result_type, until=until, tweet_mode="extended") # check latest tweet created_at = result["statuses"][-1].get("created_at") unix_time = time.mktime( time.strptime(created_at, "%a %b %d %H:%M:%S +0000 %Y") ) + 25200 # quickhack Jakarta timezone when = datetime.utcfromtimestamp(unix_time).strftime( '%Y-%m-%d %H:%M:%S') print("latest tweet created at: " + str(when)) # preprocessing data print("preprocessing data...") self.preprocessingData(result, query, sequence, querytype) print("producing csv...") self.produceCsv(post_data, query, querytype) print("current API rate limit: " + str( self._tw.get_lastfunction_header('x-rate-limit-remaining'))) # metadata finder metadata = result["search_metadata"].get("next_results") if metadata: max_id_finder = self.cursorReader(query=query, count=count, max_id=max_id, language=language, geocode=geocode, result_type=result_type, until=until) max_id = max_id_finder else: print("oops.. no max_id again.. \nlatest max_id: " + str(max_id)) break # adding new sequence sequence += 1 # print("sleeping...") # time.sleep(random.randint(1,5)) # stream handling def streamHandler(self, querytype, query, language, geocode, follow): randomizer = random.randint(0, len(tokens) - 1) keys = keyRotator().rotator(sequence=randomizer) consumer_key, consumer_key_secret, access_token, access_token_secret = keys print("using key: " + str(consumer_key) + "\n") try: self._tw_stream = Streamer(app_key=consumer_key, app_secret=consumer_key_secret, oauth_token=access_token, oauth_token_secret=access_token_secret, chunk_size=stream_stopper, retry_in=300) self._tw_stream.statuses.filter(track=query, lang=language, locations=geocode, follow=follow, tweet_mode="extended") except requests.ConnectionError: print("connection error..") sys.exit(0) if len(stream_data) == stream_stopper: for i in stream_data: self.jsonCruncher(i, querytype) stream_data[:] = [] self.produceCsv(post_data, query, querytype) print("\n==========================\n") else: pass # followers handling def followerListHandler(self, query, count, querytype): sequence = 0 next_cursor = -1 while True: print("current sequence: " + str(sequence)) print("current cursor: " + str(next_cursor)) randomizer = random.randint(0, len(tokens) - 1) keys = keyRotator().rotator(sequence=randomizer) consumer_key, consumer_key_secret, access_token, access_token_secret = keys print("using key: " + str(consumer_key)) try: self._tw_auth = Twython(consumer_key, consumer_key_secret, oauth_version=2) OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token() self._tw = Twython(consumer_key, access_token=OAUTH2_ACCESS_TOKEN) except TwythonError as e: print("error on Twython!") try: result = self._tw.get_followers_list(screen_name=query, cursor=next_cursor, count=count) except TwythonError as e: if e.error_code == 503: print( "Twitter API returned a 503 (Service Unavailable), Over capacity.." ) print("shuting down..") break self.preprocessingData(result, query, sequence, querytype) print("producing csv...") self.produceCsv(user_data, query, querytype) print("current API rate limit: " + str( self._tw.get_lastfunction_header('x-rate-limit-remaining'))) # metadata finder metadata = result.get("next_cursor") if metadata: next_cursor = metadata else: break # adding new sequence sequence += 1 print("sleeping...") time.sleep(random.randint(1, 5)) print("\n==========================\n") # following handling def followingListHandler(self, query, count, querytype): sequence = 0 next_cursor = -1 while True: print("current sequence: " + str(sequence)) print("current cursor: " + str(next_cursor)) randomizer = random.randint(0, len(tokens) - 1) keys = keyRotator().rotator(sequence=randomizer) consumer_key, consumer_key_secret, access_token, access_token_secret = keys print("using key: " + str(consumer_key)) try: self._tw_auth = Twython(consumer_key, consumer_key_secret, oauth_version=2) OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token() self._tw = Twython(consumer_key, access_token=OAUTH2_ACCESS_TOKEN) except TwythonError as e: print("error on Twython!") try: result = self._tw.get_friends_list(screen_name=query, cursor=next_cursor, count=count) except TwythonError as e: if e.error_code == 503: print( "Twitter API returned a 503 (Service Unavailable), Over capacity.." ) print("shuting down..") break self.preprocessingData(result, query, sequence, querytype) print("producing csv...") self.produceCsv(user_data, query, querytype) print("current API rate limit: " + str( self._tw.get_lastfunction_header('x-rate-limit-remaining'))) # metadata finder metadata = result.get("next_cursor") if metadata: next_cursor = metadata else: break # adding new sequence sequence += 1 print("sleeping...") time.sleep(random.randint(1, 5)) print("\n==========================\n") # user_timeline handling def userTimeline(self, query, count, querytype): sequence = 0 max_id = 0 while True: print("current sequence: " + str(sequence)) randomizer = random.randint(0, len(tokens) - 1) keys = keyRotator().rotator(sequence=randomizer) consumer_key, consumer_key_secret, access_token, access_token_secret = keys print("using key: " + str(consumer_key) + "\n") try: self._tw_auth = Twython(consumer_key, consumer_key_secret, oauth_version=2) OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token() self._tw = Twython(consumer_key, access_token=OAUTH2_ACCESS_TOKEN) except TwythonError as e: print("error on Twython!") try: if max_id == 0: result = self._tw.get_user_timeline(screen_name=query, count=count, tweet_mode="extended") else: result = self._tw.get_user_timeline(screen_name=query, max_id=max_id, count=count, tweet_mode="extended") result.pop(0) except TwythonError as e: print("error on Twython!") if e.error_code == 503: print( "Twitter API returned a 503 (Service Unavailable), Over capacity.." ) print("shuting down..") break # metadata finder metadata = result[-1].get("id") if metadata: max_id = metadata else: print("oops.. no max_id again.. \nlatest max_id: " + str(max_id)) break # check latest tweet created_at = result[-1].get("created_at") unix_time = time.mktime( time.strptime(created_at, "%a %b %d %H:%M:%S +0000 %Y") ) + 25200 # quickhack Jakarta timezone when = datetime.utcfromtimestamp(unix_time).strftime( '%Y-%m-%d %H:%M:%S') print("latest tweet created at: " + str(when)) # preprocessing data print("preprocessing data...") self.preprocessingData(result, query, sequence, querytype) print("producing csv...") self.produceCsv(post_data, query, querytype) print("current API rate limit: " + str( self._tw.get_lastfunction_header('x-rate-limit-remaining'))) # adding new sequence sequence += 1 # ========================== # MAIN ENGINE # ========================== def crawl(self, querytype, query, count, authentication, language, geocode, result_type, follow, until): if querytype == "search": # todos: loop moved here self.searchHandler(querytype=querytype, query=query, count=count, language=language, geocode=geocode, result_type=result_type, until=until) # DEBUG # result = self._tw.search(q=query, count=count, result_type="mixed", tweet_mode="extended") # print(result["statuses"]) # result = self._tw.show_status(id=815411895343587330) elif querytype == "stream": timeout = time.time() + 60 * 360 # 4 hours while True: if time.time() > timeout: break self.streamHandler(querytype=querytype, query=query, language=language, geocode=geocode, follow=follow) elif querytype == "followers": self.followerListHandler(query=query, count=count, querytype=querytype) elif querytype == "following": self.followingListHandler(query=query, count=count, querytype=querytype) elif querytype == "user_timeline": self.userTimeline(query=query, count=count, querytype=querytype) else: print("Your API type not found in this function..") # ========================== # END MAIN ENGINE # ========================== # preprocessing data def preprocessingData(self, data, query, sequence, querytype): self.saveRawJson(data, query, sequence, querytype) if querytype in {"followers", "following"}: raw_users = data['users'] for users in raw_users: self.jsonCruncher(users, querytype) elif querytype in {"search", "stream"}: raw_statuses = data['statuses'] for statuses in raw_statuses: self.jsonCruncher(statuses, querytype) elif querytype in {"user_timeline"}: raw_statuses = data for statuses in raw_statuses: self.jsonCruncher(statuses, querytype) # json specialist chruncher def jsonCruncher(self, data, querytype): temp_data = [] if querytype in {"followers", "following"}: temp_data.append(data.get("id")) # user_id temp_data.append(data.get("screen_name")) # screen_name temp_data.append(data.get("name")) # user_name temp_data.append(data.get("created_at")) # created_at temp_data.append(data.get("protected")) # is_protected temp_data.append(data.get("verified")) # is_verified temp_data.append(data.get("description")) # description if data["entities"].get("description"): # description urls user_description_urls_list = [] for i in data["entities"]["description"]["urls"]: user_description_urls_list.append( str(i.get("expanded_url"))) user_description_urls = ", ".join(user_description_urls_list) temp_data.append(user_description_urls) else: temp_data.append("None") if data["entities"].get("url"): # urls user_urls_list = [] for i in data["entities"]["url"]["urls"]: user_urls_list.append(str(i.get("expanded_url"))) user_urls = ", ".join(user_urls_list) temp_data.append(user_urls) else: temp_data.append("None") temp_data.append( data.get("profile_image_url")) # profile_image_url temp_data.append(data.get("profile_background_image_url") ) # profile_background_image_url temp_data.append(data.get("statuses_count")) # statuses_count temp_data.append(data.get("followers_count")) # followers_count temp_data.append(data.get("friends_count")) # friends_count temp_data.append(data.get("favourites_count")) # favourites_count temp_data.append(data.get("blocked_by")) # blocked_by temp_data.append(data.get("geo_enabled")) # geo_enabled temp_data.append(data.get("location")) # location temp_data.append(data.get("lang")) # lang temp_data.append(data.get("time_zone")) # time_zone temp_data.append(data.get("utc_offset")) # utc_offset # clean up temp_data_replaced = [ "None" if v is None else v for v in temp_data ] # send it to post_data user_data.append(temp_data_replaced) elif querytype in {"search", "stream", "user_timeline"}: # user temp_data.append(data["user"].get("id")) # user_id temp_data.append(data["user"].get("verified")) # user is verified temp_data.append( data["user"].get("protected")) # user is protected temp_data.append( data["user"].get("screen_name")) # user screen_name temp_data.append(data["user"].get("name")) # user name temp_data.append(data["user"].get("created_at")) # user created_at temp_data.append( data["user"].get("followers_count")) # user followers_count temp_data.append( data["user"].get("statuses_count")) # user statuses_count temp_data.append( data["user"].get("friends_count")) # user following_count temp_data.append(data["user"].get("location")) # user location temp_data.append(data["user"].get("utc_offset")) # user utc_offset temp_data.append(data["user"].get( "profile_image_url")) # user profile_image_url temp_data.append(data["user"].get( "profile_banner_url")) # user profile_banner_url temp_data.append(data["user"].get("profile_background_image_url") ) # user profile_background_image_url temp_data.append( data["user"].get("description")) # user description temp_data.append(data["user"].get("time_zone")) # user time_zone temp_data.append(data["user"].get("url")) # user url (profile_url) # tweet temp_data.append(data.get("id")) # tweet id (status_id) temp_data.append(data.get("created_at")) # tweet created_at temp_data.append(data.get("lang")) # tweet lang if data.get("full_text"): temp_data.append(' '.join( data.get("full_text").split())) # tweet text if 280 char else: temp_data.append(' '.join( data.get("full_text").split())) # tweet text if 140 char temp_data.append(data.get("source")) # tweet source temp_data.append( data.get("possibly_sensitive")) # tweet possibly_sensitive if data.get("coordinates"): longitude, latitude = data["coordinates"].get("coordinates") temp_data.append(str(longitude) + ", " + str(latitude)) # tweet coordinates joined temp_data.append(longitude) # tweet_coordinates_long temp_data.append(latitude) # tweet_coordinates_lat else: temp_data.append( "None") # set to none -- # tweet coordinates joined temp_data.append( "None") # set to none -- # tweet_coordinates_long temp_data.append( "None") # set to none -- # tweet_coordinates_at if data.get("geo"): longitude, latitude = data["geo"].get("coordinates") temp_data.append(str(longitude) + ", " + str(latitude)) # tweet geo joined temp_data.append(longitude) # tweet_geo_long temp_data.append(latitude) # tweet_geo_lat else: temp_data.append("None") # set to none -- # tweet geo joined temp_data.append("None") # set to none -- # tweet_geo_long temp_data.append("None") # set to none -- # tweet_geo_at if data.get('place'): temp_data.append( data['place'].get("country")) # tweet place country temp_data.append(data['place'].get("name")) # tweet place name else: temp_data.append("None") # set to none temp_data.append("None") # set to none temp_data.append(data.get("retweeted")) # tweet retweeted temp_data.append(data.get("favorited")) # tweet favorited temp_data.append(data.get("retweet_count")) # tweet retweet_count temp_data.append( data.get("favorite_count")) # tweet favorite_count temp_data.append( data.get("is_quote_status")) # tweet favorite_count if data.get("quoted_status_id"): temp_data.append( data.get("quoted_status_id")) # tweet qouted_status_id else: temp_data.append("None") # if not qouting, set to none temp_data.append(data.get( "in_reply_to_status_id")) # tweet in_reply_to_status_id temp_data.append(data.get( "in_reply_to_screen_name")) # tweet in_reply_to_screen_name temp_data.append( data.get("in_reply_to_user_id")) # tweet in_reply_to_user_id # tweet entities - symbols if data["entities"].get("symbols"): tweet_entities_symbols_list = [] for i in data["entities"].get("symbols"): tweet_entities_symbols_list.append( i.get("text")) # extract text to list tweet_entities_symbols = ", ".join( tweet_entities_symbols_list) # join it temp_data.append( tweet_entities_symbols) # append it to temp_data else: temp_data.append("None") # empty symbols -- set to none # tweet entities - user_mentions if data["entities"].get("user_mentions"): tweet_entities_user_mentions_id_list = [] tweet_entities_user_mentions_screen_name_list = [] tweet_entities_user_mentions_name_list = [] for i in data["entities"].get("user_mentions"): tweet_entities_user_mentions_id_list.append( str(i.get("id"))) tweet_entities_user_mentions_screen_name_list.append( i.get("screen_name")) tweet_entities_user_mentions_name_list.append( i.get("name")) tweet_entities_user_mentions_id = ", ".join( tweet_entities_user_mentions_id_list ) # extract id user_mentions tweet_entities_user_mentions_screen_name = ", ".join( tweet_entities_user_mentions_screen_name_list ) # extract screen_name user_mentions tweet_entities_user_mentions_name = ", ".join( tweet_entities_user_mentions_name_list ) # extract name user_mentions temp_data.append(tweet_entities_user_mentions_id ) # append entities_id to temp_data temp_data.append(tweet_entities_user_mentions_screen_name ) # append entities_screen_name to temp_data temp_data.append(tweet_entities_user_mentions_name ) # append entities_name to temp_data else: temp_data.append("None") # empty user_mentions_id set to none temp_data.append( "None") # empty user_mentions_screen_name set to none temp_data.append( "None") # empty user_mentions_name set to none # tweet entities - hashtag if data["entities"].get("hashtags"): tweet_entities_hashtags_list = [] for i in data["entities"].get("hashtags"): tweet_entities_hashtags_list.append( i.get("text")) # extract hashtags tweet_entities_hashtags = ", ".join( tweet_entities_hashtags_list) # join to new string temp_data.append( tweet_entities_hashtags) # append it to temp_data else: temp_data.append("None") # empty hashtags # tweet media if data.get("extended_entities" ): # check whenever extended_entities exist or not medias = data["extended_entities"]["media"] # temp list tweet_extended_entitites_media_type_list = [] tweet_extended_entitites_media_id_list = [] tweet_extended_entitites_media_monetizable_list = [] tweet_extended_entitites_media_urls_list = [] # tweet_extended_entities_media_video_duration_millis_list = [] # look up for media in medias: tweet_extended_entitites_media_type_list.append( media.get("type")) # extract media_type tweet_extended_entitites_media_id_list.append( str(media.get("id"))) # extract media_id tweet_extended_entitites_media_monetizable_list.append( str(media.get( "monetizable"))) # extract media_monetizable tweet_extended_entitites_media_urls_list.append( str(media.get("media_url"))) # extract media_url # joins tweet_extended_entitites_media_type = ", ".join( tweet_extended_entitites_media_type_list) tweet_extended_entitites_media_id = ", ".join( tweet_extended_entitites_media_id_list) tweet_extended_entitites_media_monetizable = ", ".join( tweet_extended_entitites_media_monetizable_list) tweet_extended_entitites_media_urls = ", ".join( tweet_extended_entitites_media_urls_list) # append temp_data.append( tweet_extended_entitites_media_type ) # append tweet_extended_entitites_media_type temp_data.append(tweet_extended_entitites_media_id ) # append tweet_extended_entitites_media_id temp_data.append( tweet_extended_entitites_media_monetizable ) # append tweet_extended_entitites_media_monetizable temp_data.append( tweet_extended_entitites_media_urls ) # append tweet_extended_entitites_media_urls else: temp_data.append( "None" ) # empty tweet_extended_entitites_media_type set to none temp_data.append( "None" ) # empty tweet_extended_entitites_media_id set to none temp_data.append( "None" ) # empty tweet_extended_entitites_media_monetizable set to none temp_data.append( "None" ) # empty tweet_extended_entitites_media_urls set to none # clean up temp_data_replaced = [ "None" if v is None else v for v in temp_data ] # send it to post_data post_data.append(temp_data_replaced) else: print("failed to crunch data..") # create csv def produceCsv(self, data, query, querytype): if querytype in {"search", "stream", "user_timeline"}: df = pd.DataFrame.from_records(data, columns=post_data_labels) elif querytype in {"following", "followers"}: df = pd.DataFrame.from_records(data, columns=user_data_labels) if query.startswith("#"): filename = "hashtag-" + query.replace("#", "") elif query.startswith("@"): filename = "user-" + query.replace("@", "") elif " " in query == True: print("ada spasi..") filename = str(query.replace(" ", "_")) elif " " in query: filename = str(query.replace(" ", "_").replace(",", "")) else: filename = "keyword-" + query filename = filename dir_path = './result' today = time.strftime("%Y_%m_%d") fullfilename = str(querytype) + "-" + filename + '-' + today + '.csv' filepath = os.path.join(dir_path, fullfilename) if os.path.isfile(filepath): # when result csv exist df.to_csv(filepath, mode="a", header=False, index=False, encoding="utf-8-sig", line_terminator="\r\n") print("appending to exisiting csv file at: " + str(filepath)) print("cleaning up temporary data...") post_data[:] = [] else: # when result csv NOT exist df.to_csv(filepath, index=False, encoding="utf-8-sig", line_terminator="\r\n") print("writing new csv file at: " + str(filepath)) print("cleaning up temporary data...") post_data[:] = [] # save raw json file def saveRawJson(self, data, query, sequence, querytype): today = time.strftime("%Y_%m_%d") if query.startswith("#"): filename = "hashtag-" + query.replace("#", "") elif query.startswith("@"): filename = "user-" + query.replace("@", "") elif " " in query: filename = str(query.replace(" ", "_").replace(",", "")) else: filename = "keyword-" + query dir_path = os.path.join('./data', filename) fullfilename = str( querytype) + "-" + filename + '-' + today + '-' + str( sequence) + ".json" file_path = os.path.join(dir_path, fullfilename) # check dirs if not os.path.exists(dir_path): os.makedirs(dir_path) else: pass # writer get ready file_writer = open(file_path, "w") file_appender = open(file_path, "a") raw_json = json.dumps(data) # writer on action if os.path.isfile(file_path): file_appender.write(raw_json.decode("utf-8")) else: file_writer.write(raw_json.decode("utf-8"))
REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" AUTHORIZE_URL = "https://api.twitter.com/oauth/authorize?oauth_token=" ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" CONSUMER_KEY = "K7ddHMoZcHXu0WyWdnp6wdQxf" CONSUMER_SECRET = "OcvSpsGJ5e1f9GvHbT1y5TXLzwETVkL9wkd1WDzL3FHgC7WDzk" ACCESS_KEY = "2539409241-FFZPA5lh2vZklUx3q5jASFesRVVXPxqfS6nY8QB" ACCESS_SECRET = "zzQVXih1Gy2cR1WdMtcMhmXbWpi1S45Jc7vlLlkHFWwm9" twitter = Twython(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET) user_Friends = twitter.get_friends_list(screen_name="Francis_Pruter", count=200, skip_status='true', include_user_entities='false') ctr = 1 fcountlist={} for friend in user_Friends['users']: fcountlist[friend['name']] = friend['friends_count'] ctr = ctr+1 fcountlist["ME"] = ctr-1 #sort list by friend count https://docs.python.org/2/library/collections.html#ordereddict-examples-and-recipes fcountlist = OrderedDict(sorted(fcountlist.items(), key=lambda t: t[1]))
class TwythonEndpointsTestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } # This is so we can hit coverage that Twython sets # User-Agent for us if none is supplied oauth2_client_args = { 'headers': {} } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) # Timelines @unittest.skip('skipping non-updated test') def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() @unittest.skip('skipping non-updated test') def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) @unittest.skip('skipping non-updated test') def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() @unittest.skip('skipping non-updated test') def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets @unittest.skip('skipping non-updated test') def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get \ deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) @unittest.skip('skipping non-updated test') def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') @unittest.skip('skipping non-updated test') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search @unittest.skip('skipping non-updated test') def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages @unittest.skip('skipping non-updated test') def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() @unittest.skip('skipping non-updated test') def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() @unittest.skip('skipping non-updated test') def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s\ ' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) @unittest.skip('skipping non-updated test') def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! \ %s' % int(time.time())) # Friends & Followers @unittest.skip('skipping non-updated test') def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) @unittest.skip('skipping non-updated test') def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') @unittest.skip('skipping non-updated test') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() @unittest.skip('skipping non-updated test') def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() @unittest.skip('skipping non-updated test') def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) @unittest.skip('skipping non-updated test') def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users @unittest.skip('skipping non-updated test') def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() @unittest.skip('skipping non-updated test') def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() @unittest.skip('skipping non-updated test') def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') @unittest.skip('skipping non-updated test') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') @unittest.skip('skipping non-updated test') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') @unittest.skip('skipping non-updated test') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') @unittest.skip('skipping non-updated test') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() @unittest.skip('skipping non-updated test') def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() @unittest.skip('skipping non-updated test') def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') @unittest.skip('skipping non-updated test') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') @unittest.skip('skipping non-updated test') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') @unittest.skip('skipping non-updated test') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) @unittest.skip('skipping non-updated test') def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() @unittest.skip('skipping non-updated test') def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) @unittest.skip('skipping non-updated test') def test_list_mutes(self): """Test listing users who are muted by the authenticated user succeeds""" self.api.list_mutes() @unittest.skip('skipping non-updated test') def test_list_mute_ids(self): """Test listing user ids who are muted by the authenticated user succeeds""" self.api.list_mute_ids() @unittest.skip('skipping non-updated test') def test_create_mute(self): """Test muting a user succeeds""" self.api.create_mute(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_mute(self): """Test muting a user succeeds""" self.api.destroy_mute(screen_name='justinbieber') # Suggested Users @unittest.skip('skipping non-updated test') def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') @unittest.skip('skipping non-updated test') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() @unittest.skip('skipping non-updated test') def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites @unittest.skip('skipping non-updated test') def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() @unittest.skip('skipping non-updated test') def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists @unittest.skip('skipping non-updated test') def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed \ %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) @unittest.skip('skipping non-updated test') def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() @unittest.skip('skipping non-updated test') def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') @unittest.skip('skipping non-updated test') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches @unittest.skip('skipping non-updated test') def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() @unittest.skip('skipping non-updated test') def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo @unittest.skip('skipping non-updated test') def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') @unittest.skip('skipping non-updated test') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') @unittest.skip('skipping non-updated test') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') @unittest.skip('skipping non-updated test') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends @unittest.skip('skipping non-updated test') def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) @unittest.skip('skipping non-updated test') def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() @unittest.skip('skipping non-updated test') def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help @unittest.skip('skipping non-updated test') def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() @unittest.skip('skipping non-updated test') def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() @unittest.skip('skipping non-updated test') def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() @unittest.skip('skipping non-updated test') def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() @unittest.skip('skipping non-updated test') def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) friends_file = unicodecsv.writer(open(FILE_LOC, "wb")) #Run basic program if(ADVANCED==0): #Write csv column headers friends_file.writerow(["User Name","Name","ID","Description","Location","URL"]) next_cursor=-1 p = 1 while(next_cursor != 0): print "Loading page", p , "of results from Twitter" #Loop/set variables for each user in friends list friends = twitter.get_friends_list(screen_name = USERNAME, count = 200, cursor = next_cursor) print "Writing page", p , "to csv file" for result in friends['users']: screen_name = result['screen_name'] name = result['name'] tw_id = result['id'] description = result['description'] location = result['location'] entities = result['entities'] expanded_url = None if('url' in entities): entities_url = entities['url'] entities_urls = entities_url['urls'] for entities_result in entities_urls: expanded_url = entities_result['expanded_url']
friends_file = unicodecsv.writer(open(FILE_LOC, "wb")) #Run basic program if (ADVANCED == 0): #Write csv column headers friends_file.writerow( ["User Name", "Name", "ID", "Description", "Location", "URL"]) next_cursor = -1 p = 1 while (next_cursor != 0): print "Loading page", p, "of results from Twitter" #Loop/set variables for each user in friends list friends = twitter.get_friends_list(screen_name=USERNAME, count=200, cursor=next_cursor) print "Writing page", p, "to csv file" for result in friends['users']: screen_name = result['screen_name'] name = result['name'] tw_id = result['id'] description = result['description'] location = result['location'] entities = result['entities'] expanded_url = None if ('url' in entities): entities_url = entities['url'] entities_urls = entities_url['urls'] for entities_result in entities_urls:
class TwythonAPITestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': { } # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual( constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_get(self): """Test Twython generic GET request works""" self.api.get('account/verify_credentials') def test_post(self): """Test Twython generic POST request works, with a full url and with just an endpoint""" update_url = 'https://api.twitter.com/1.1/statuses/update.json' status = self.api.post( update_url, params={'status': 'I love Twython! %s' % int(time.time())}) self.api.post('statuses/destroy/%s' % status['id_str']) def test_get_lastfunction_header(self): """Test getting last specific header of the last API call works""" self.api.get('statuses/home_timeline') self.api.get_lastfunction_header('x-rate-limit-remaining') def test_get_lastfunction_header_not_present(self): """Test getting specific header that does not exist from the last call returns None""" self.api.get('statuses/home_timeline') header = self.api.get_lastfunction_header('does-not-exist') self.assertEqual(header, None) def test_get_lastfunction_header_no_last_api_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') def test_cursor(self): """Test looping through the generator results works, at least once that is""" search = self.api.cursor(self.api.search, q='twitter', count=1) counter = 0 while counter < 2: counter += 1 result = next(search) new_id_str = int(result['id_str']) if counter == 1: prev_id_str = new_id_str time.sleep( 1) # Give time for another tweet to come into search if counter == 2: self.assertTrue(new_id_str > prev_id_str) def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR exapanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text) def test_raise_error_on_bad_ssl_cert(self): """Test TwythonError is raised by a RequestException when an actual HTTP happens""" self.assertRaises(TwythonError, self.api.get, 'https://example.com') # Timelines def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline( screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status( status='Test post just to get deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time())) # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
def add_alert(request): if not request.user.is_authenticated(): return HttpResponseRedirect('/') context = {} context.update(csrf(request)) user = request.user profile = TwitterProfile.objects.all().filter(user=user).get() alerts_count = len(GrainVacuum.objects.all().filter(profile=profile)) screen_name_list = None name_list = None if request.method == "POST": follow_user = request.POST['follow_user'] key_words = request.POST['key_words_list'] follow_user = follow_user.replace('@','') new_grain = GrainVacuum(profile=profile,user=user,follow_user=follow_user,key_words=key_words) new_grain.save() return HttpResponseRedirect('/dashboard/') else: # Update FriendList try: # Determine if friend_list exists try: friend_list = FriendList.objects.all().filter(twitter_profile=profile).get() except: # if it doesn't exist create a blank one new_friend_list = FriendList(twitter_profile=profile) new_friend_list.save() friend_list = FriendList.objects.all().filter(twitter_profile=profile).get() est = pytz.timezone('US/Eastern') # Eastern Time Zone last_checked = friend_list.last_checked now = datetime.datetime.now() tdelta = 0 if last_checked != None: last_checked = last_checked.astimezone(est).replace(tzinfo=None) tdelta = int((now - last_checked).total_seconds() / 60) print last_checked if tdelta > 15 or last_checked == None: APP_KEY = Config.objects.all().latest('id').twitter_key APP_SECRET = Config.objects.all().latest('id').twitter_secret # Get Authorization Credentials OAUTH_TOKEN = profile.oauth_token OAUTH_TOKEN_SECRET = profile.oauth_secret friend_list_update = FriendList.objects.all().filter(twitter_profile=profile) twitter = Twython(APP_KEY, APP_SECRET,OAUTH_TOKEN, OAUTH_TOKEN_SECRET) # Query Twitter Friends List API until none screen_name_list = "" name_list = "" next_cursor=-1 count = 0 while next_cursor: follow_list = twitter.get_friends_list(count=200,cursor=next_cursor) y = 0 for user in follow_list['users']: screen_name = user['screen_name'] name = user['name'] if y == 0: screen_name_list += screen_name name_list += name else: screen_name_list += "," + screen_name name_list += "," + name y += 1 print screen_name + ' ' + name count += 1 if count == 15 or next_cursor == 0: break next_cursor = follow_list["next_cursor"] friend_list_update.update(last_checked=now,name=name_list,handle=screen_name_list) else: screen_name_list = friend_list.handle name_list = friend_list.name except Exception as error: print error return render_to_response('add_alert.html', {'screen_name_list':screen_name_list,'name_list':name_list,'alerts_count':alerts_count}, context_instance=RequestContext(request))
class twitterbot: __twitter = None __db = None def __init__(self, file_path_bot=None, file_path_db=None): if file_path_bot is None: file_path_bot = 'twitter_aut.json' access = {'apiKey': '', 'apiSecret': '', 'accessToken': '', 'accessTokenSecret': ''} if os.path.isfile(file_path_bot): with open(file_path_bot, 'r') as f: access = json.load(f) else: with open(file_path_bot, 'w') as outfile: json.dump(access, outfile) self.__twitter = Twython(access['apiKey'], access['apiSecret'], access['accessToken'], access['accessTokenSecret']) self.__db = BotDb(file_path_db=file_path_db) def get_limits(self): limits = self.__twitter.get_application_rate_limit_status() return limits def tweet(self, msg): self.__twitter.update_status(status=msg) def __check_tags(self, tweet_tags, tags): for tweet_tag in tweet_tags: if tweet_tag in tags: return True return False def __check_tweets(self): try: tweets = self.__db.get_open_tweets() for tweet in tweets: self.tweet(tweet.message) self.__db.set_tweet_to_sended(tweet.id) except twython.exceptions.TwythonError: self.__db.set_tweet_to_sended(tweet.id) except: info = sys.exc_info() print(info) def loop(self): while True: self.__check_tweets() tags = self.__db.get_tags() followers = self.__twitter.get_friends_list()['users'] for follower in followers: screenName = follower["screen_name"] tweets = self.__twitter.get_user_timeline(screen_name=screenName, tweet_mode='extended') for status in tweets: id = status['id'] tweet_tags = [tag['text'].lower() for tag in status['entities']['hashtags']] if self.__check_tags(tweet_tags, tags): if self.__db.add_try_retweet(int(id)): try: self.__twitter.retweet(id=id) except: pass break time.sleep(120) def run(self): _thread.start_new(self.loop, ())