def resolve_screen_name_for_export(tw_context, screen_name_to_export):
oauth_token = tw_context['oauth_final_token']
oauth_token_secret = tw_context['oauth_final_token_secret']
twitter = Twython(app_key=APP_KEY,
app_secret=APP_SECRET,
oauth_token=oauth_token,
oauth_token_secret=oauth_token_secret)
try:
u = twitter.show_user(screen_name=screen_name_to_export)
# test actual access to the list of friends of the user to export data for
twitter.get_friends_list(
screen_name=screen_name_to_export,
skip_status=True,
include_user_entities=False,
count=5)
except TwythonError as te:
msg = resolve_twitter_error(te)
msg = f"{screen_name_to_export}: {msg}"
return False, None, msg
resolved_screen_name = u['screen_name']
friends_count = u['friends_count']
if friends_count == 0:
return False, None, f"Profile {screen_name_to_export} doesn't have any friends to export"
elif friends_count > MAX_FRIENDS:
err_msg_for_user = f"Profile {screen_name_to_export} has {friends_count}, we support a maximum of {MAX_FRIENDS}"
return False, None, err_msg_for_user
else:
return True, resolved_screen_name, None
class TwitterCrawlerFollow:
users_expanded=[]
def __init__(self, file_name):
self.output_file=open(file_name, "w")
self.twitter_connection=Twython(con_key, access_token=acc_token)
def start_crawl(self, search_term, node_limit):
data={}
while True:
try:
data = self.twitter_connection.search(q=search_term, lang="en", result_type="recent", count=1)
break
except TwythonError as e:
print e
if str(429) in e.msg:
time.sleep(900)
self.twitter_connection = Twython(con_key, access_token=acc_token)
def expand_user(self, user, depth, max_depth):
friends=self.twitter_connection.get_friends_list(user_id=user, count=200)
for friend in friends:
if depth<max_depth:
if user not in self.users_expanded:
self.users_expanded.append(friend)
self.expand_user(friend, depth+1, max_depth)
else:
return
def fetch_friends(self, user):
"""
fetches the friends from twitter using the
information on django-social-auth models
user is an instance of UserSocialAuth
Returns:
collection of friend objects fetched from Twitter
"""
# now fetch the twitter friends using `twython`
auth_data = self._auth_data(user)
tw = Twython(**auth_data)
cursor = -1
friends = []
while True:
data = tw.get_friends_list(cursor=cursor)
friends += data.get('users', [])
next_cursor = data.get('next_cursor', 0)
prev_cursor = data.get('prev_cursor', 0)
if not next_cursor or next_cursor == prev_cursor:
break
else:
cursor = next_cursor
return friends
def twitter_users(request):
try:
user = Twython(app_key=CONSUMER_KEY,
app_secret=CONSUMER_SECRET,
oauth_token=ACCESS_TOKEN,
oauth_token_secret=ACCESS_TOKEN_SECRET
)
search_results = user.get_friends_list(count=10)
users = search_results['users']
user_names = [usr['name'] for usr in users]
return render(request, 'users.html', {'data': user_names})
except:
pass
def getFollowing(screenName, userCount = 200,skipStatus = True):
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
results = twitter.get_friends_list(screen_name=screenName, count=userCount, skip_status=skipStatus)
if userCount <= 5:
for result in results['users']:
print 'ID: ', result['id']
print 'Screen Handle: ', result['screen_name']
print '===================//==================='
print ' '
else:
fp = codecs.open('.\Results\\'+currentTime()+' (FOLLOWING).txt', 'w', "utf-8")
for result in results['users']:
fp.write('ID: '+str(result['id'])+'\r\n')
fp.write('Screen Handle: '+result['screen_name']+'\r\n')
fp.write('===================//===================\r\n')
def get_following(log, id):
log.debug(" Getting people %s is following", id)
check_twitter_config()
logging.captureWarnings(True)
old_level = log.getEffectiveLevel()
log.setLevel(logging.ERROR)
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
log.setLevel(old_level)
cursor = -1
max_loops = 15
while cursor != 0:
try:
log.setLevel(logging.ERROR)
following = twitter.get_friends_list(screen_name=id, cursor=cursor, count=200)
log.setLevel(old_level)
except TwythonAuthError, e:
log.exception(" Problem trying to get people following")
twitter_auth_issue(e)
raise
except:
def get_following(log, id):
log.debug(" Getting people %s is following", id)
check_twitter_config()
logging.captureWarnings(True)
old_level = log.getEffectiveLevel()
log.setLevel(logging.ERROR)
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
log.setLevel(old_level)
cursor = -1
max_loops = 15
while cursor != 0:
try:
log.setLevel(logging.ERROR)
following = twitter.get_friends_list(screen_name=id,
cursor=cursor,
count=200)
log.setLevel(old_level)
except TwythonAuthError as e:
log.exception(" Problem trying to get people following")
twitter_auth_issue(e)
raise
except:
raise
for u in following["users"]:
yield u["screen_name"]
cursor = following["next_cursor"]
if cursor:
s = random.randint(55, 65)
log.debug(" Sleeping %ds to avoid rate limit. Cursor: %s", s,
cursor)
time.sleep(s)
else:
log.debug(" Normal query end")
max_loops -= 1
if max_loops <= 0:
log.debug(" Killing search due to max loops")
break
log.setLevel(old_level)
def fetch_friends(self, user):
"""
fetches the friends from twitter
Returns:
collection of friend objects fetched from Twitter
"""
auth_data = self._auth_data(user)
tw = Twython(**auth_data)
cursor = -1
friends = []
while True:
data = tw.get_friends_list(cursor=cursor)
friends += data.get('users', [])
next_cursor = data.get('next_cursor', 0)
prev_cursor = data.get('prev_cursor', 0)
if not next_cursor or next_cursor == prev_cursor:
break
else:
cursor = next_cursor
return friends
def get(self, request, *args, **kwargs):
APP_KEY = 'Yf8q3TPgL59StYj8KRlsAMmLu'
APP_SECRET = 'KuMfl4Kvu0Y5W8qVYbusiu6L5x78wK3jpDJV1WAPfcjEy6zAvA'
OAUTH_TOKEN = "198514629-mTKORfs9NmA29kfbnxrU9bCzyc8NvrzFDP9XLiTw"
OAUTH_TOKEN_SECRET = 'r9u7SJPrc7u7Vy9pz66uMrfcBSzL8he6HRHprJgEC6bGQ'
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
try:
search_term = request.GET.get('srch-term', 'twitter')
search_results = twitter.search(q=search_term, count=10)
except TwythonError as e:
print e
my_updates = twitter.get_friends_list(count=500)
my_updates = my_updates['users']
the_type = type(my_updates)
return render(
request, "catalog/twitter.html", {
'search_results': search_results['statuses'],
'my_updates': my_updates,
'the_type': the_type
})
def get(self, request, *args, **kwargs):
APP_KEY = 'Yf8q3TPgL59StYj8KRlsAMmLu'
APP_SECRET = 'KuMfl4Kvu0Y5W8qVYbusiu6L5x78wK3jpDJV1WAPfcjEy6zAvA'
OAUTH_TOKEN = "198514629-mTKORfs9NmA29kfbnxrU9bCzyc8NvrzFDP9XLiTw"
OAUTH_TOKEN_SECRET = 'r9u7SJPrc7u7Vy9pz66uMrfcBSzL8he6HRHprJgEC6bGQ'
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
try:
search_term = request.GET.get('srch-term', 'twitter')
search_results = twitter.search(q=search_term, count=10)
except TwythonError as e:
print e
my_updates = twitter.get_friends_list(count=500)
my_updates = my_updates['users']
the_type = type(my_updates)
return render(request, "catalog/twitter.html", {
'search_results': search_results['statuses'],
'my_updates': my_updates,
'the_type': the_type
})
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret)
# Timelines
def test_get_mentions_timeline(self):
'''Test returning mentions timeline for authenticated user succeeds'''
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
'''Test returning timeline for authenticated user and random user
succeeds'''
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
'''Test returning a protected user timeline who you are following
succeeds'''
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
'''Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError'''
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_get_home_timeline(self):
'''Test returning home timeline for authenticated user succeeds'''
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
'''Test getting retweets of a specific tweet succeeds'''
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
'''Test returning a single status details succeeds'''
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
'''Test updating and deleting a status succeeds'''
status = self.api.update_status(status='Test post just to get deleted :(')
self.api.destroy_status(id=status['id_str'])
def test_retweet(self):
'''Test retweeting a status succeeds'''
retweet = self.api.retweet(id='99530515043983360')
self.api.destroy_status(id=retweet['id_str'])
def test_retweet_twice(self):
'''Test that trying to retweet a tweet twice raises a TwythonError'''
retweet = self.api.retweet(id='99530515043983360')
self.assertRaises(TwythonError, self.api.retweet,
id='99530515043983360')
# Then clean up
self.api.destroy_status(id=retweet['id_str'])
def test_get_oembed_tweet(self):
'''Test getting info to embed tweet on Third Party site succeeds'''
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
'''Test getting ids for people who retweeted a tweet succeeds'''
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
'''Test searching tweets succeeds'''
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
'''Test getting the authenticated users direct messages succeeds'''
self.api.get_direct_messages()
def test_get_sent_messages(self):
'''Test getting the authenticated users direct messages they've
sent succeeds'''
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
'''Test sending, getting, then destory a direct message succeeds'''
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d!')
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
'''Test sending a direct message to someone who doesn't follow you
fails'''
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man!')
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
'''Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds'''
self.api.get_user_ids_of_blocked_retweets(stringify_ids='true')
def test_get_friends_ids(self):
'''Test returning ids of users the authenticated user and then a random
user is following succeeds'''
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
'''Test returning ids of users the authenticated user and then a random
user are followed by succeeds'''
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
'''Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds'''
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
'''Test returning incoming friendship ids succeeds'''
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
'''Test returning outgoing friendship ids succeeds'''
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
'''Test creating a friendship succeeds'''
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
'''Test destroying a friendship succeeds'''
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
'''Test updating friendships succeeds'''
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='false')
def test_show_friendships(self):
'''Test showing specific friendship succeeds'''
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
'''Test getting list of users authenticated user then random user is
following succeeds'''
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
'''Test getting list of users authenticated user then random user are
followed by succeeds'''
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
'''Test getting the authenticated user account settings succeeds'''
self.api.get_account_settings()
def test_verify_credentials(self):
'''Test representation of the authenticated user call succeeds'''
self.api.verify_credentials()
def test_update_account_settings(self):
'''Test updating a user account settings succeeds'''
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
'''Test updating delivery settings fails because we don't have
a mobile number on the account'''
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
def test_update_profile(self):
'''Test updating profile succeeds'''
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
'''Test updating profile colors succeeds'''
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
'''Test listing users who are blocked by the authenticated user
succeeds'''
self.api.list_blocks()
def test_list_block_ids(self):
'''Test listing user ids who are blocked by the authenticated user
succeeds'''
self.api.list_block_ids()
def test_create_block(self):
'''Test blocking a user succeeds'''
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
'''Test unblocking a user succeeds'''
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
'''Test listing a number of user objects succeeds'''
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
'''Test showing one user works'''
self.api.show_user(screen_name='twitter')
def test_search_users(self):
'''Test that searching for users succeeds'''
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
'''Test returning list of accounts the specified user can
contribute to succeeds'''
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
'''Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account'''
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
'''Test removing profile banner succeeds'''
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
'''Test getting list of profile banner sizes fails because
we have not uploaded a profile banner'''
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
'''Test getting user suggestions by slug succeeds'''
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
'''Test getting user suggestions succeeds'''
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
'''Test getting status of suggested users succeeds'''
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
'''Test getting list of favorites for the authenticated
user succeeds'''
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
'''Test creating and destroying a favorite on a tweet succeeds'''
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
'''Test show lists for specified user'''
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
'''Test timeline of tweets authored by members of the
specified list succeeds'''
self.api.get_list_statuses(list_id=test_list_id)
def test_create_update_destroy_list_add_remove_list_members(self):
'''Test create a list, adding and removing members then
deleting the list succeeds'''
the_list = self.api.create_list(name='Stuff')
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed')
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name='johncena,xbox')
self.api.delete_list_members(list_id=list_id,
screen_name='johncena,xbox')
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_memberships(self):
'''Test list of lists the authenticated user is a member of succeeds'''
self.api.get_list_memberships()
def test_get_list_subscribers(self):
'''Test list of subscribers of a specific list succeeds'''
self.api.get_list_subscribers(list_id=test_list_id)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
'''Test subscribing, is a list sub and unsubbing to list succeeds'''
self.api.subscribe_to_list(list_id=test_list_id)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(list_id=test_list_id,
screen_name=screen_name)
self.api.unsubscribe_from_list(list_id=test_list_id)
def test_is_list_member(self):
'''Test returning if specified user is member of a list succeeds'''
# Returns 404 if not list member
self.api.is_list_member(list_id=test_list_id, screen_name='jack')
def test_get_list_members(self):
'''Test listing members of the specified list succeeds'''
self.api.get_list_members(list_id=test_list_id)
def test_get_specific_list(self):
'''Test getting specific list succeeds'''
self.api.get_specific_list(list_id=test_list_id)
def test_get_list_subscriptions(self):
'''Test collection of the lists the specified user is
subscribed to succeeds'''
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
'''Test collection of lists the specified user owns succeeds'''
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
'''Test getting list of saved searches for authenticated
user succeeds'''
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
'''Test getting list of saved searches for authenticated
user succeeds'''
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
'''Test getting info about a geo location succeeds'''
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
'''Test reversing geocode succeeds'''
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
'''Test search for places that can be attached
to a statuses/update succeeds'''
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
'''Test locates places near the given coordinates which
are similar in name succeeds'''
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
'''Test getting the top 10 trending topics for a specific
WOEID succeeds'''
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
'''Test returning locations that Twitter has trending
topic information for succeeds'''
self.api.get_available_trends()
def test_get_closest_trends(self):
'''Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds'''
self.api.get_closest_trends(lat='37', long='-122')
tweets = twitter.get_home_timeline(count=1)
for tweet in tweets:
print(tweet['text'])
print
print "My Settings"
print "==========="
settings = twitter.get_account_settings()
#print settings
print "I am @%s" % settings['screen_name']
print
print "My friends"
print "=========="
list = twitter.get_friends_list()
for friend in list['users']:
print "I follow user @%s, name %s, id %d" % (
friend['screen_name'], friend['name'], friend['id'])
print
print "My followers"
print "============"
list = twitter.get_followers_list()
for friend in list['users']:
print "User @%s, name %s, id %d is following me" % (
friend['screen_name'], friend['name'], friend['id'])
#print friend
azi = datetime.datetime.now().strftime("%Y-%m-%d")
ieri = datetime.datetime.now() - datetime.timedelta(days=1)
ieri.strftime("%Y-%m-%d")
datestamp = datetime.datetime.now().strftime("%Y-%m-%d")
raport = "raport-friends-" + username + "-" + str(azi) + ".html"
tweetsdata = {}
index = 1
next_cursor = -1
save = open(raport, 'w', encoding="utf-8")
save.write(
"<!DOCTYPE html> <html lang=\"en\"> <head> <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
)
while (next_cursor):
get_friends = twitter.get_friends_list(screen_name=username,
count=200,
cursor=next_cursor)
for friend in get_friends["users"]:
location = friend["location"]
location = location.replace("'", " ")
nume = friend["name"]
nume = nume.replace("'", " ")
try:
time.sleep(sleeptime)
get_tweets = twitter.get_user_timeline(
screen_name=friend['screen_name'], count=1, include_rts=True)
for x in get_tweets:
x['text'] = Twython.html_for_tweet(x)
if (verbose > 0):
print(
str(index) + " - <a href=\"https://twitter.com/" +
class TwitterClient():
def __init__(self, file_name='cred.txt'):
with open(file_name) as f:
lines = f.readlines()
self.APP_KEY = lines[0].strip()
self.APP_SECRET = lines[1].strip()
if len(lines)>3:
self.USER_TOKEN = lines[2].strip()
self.USER_SECRET = lines[3].strip()
else:
self.USER_TOKEN = None
self.USER_SECRET = None
self.reset()
def reset(self):
self.twitter = Twython(self.APP_KEY, self.APP_SECRET, self.USER_TOKEN, self.USER_SECRET)
def get_timeline(self, screen_name=None, user_id=None):
all_tweets = []
results = []
next_max_id = None
print('Extracting timeline of: @{}'.format(screen_name or user_id))
while next_max_id==None or (results!=None and len(results) != 0):
try:
results = self.try_call(lambda: self.twitter.get_user_timeline(screen_name=screen_name, user_id=user_id, count=200, max_id=next_max_id, tweet_mode='extended'))
except:
#something went wrong: internet or auth
break
if results:
next_max_id = results[-1]['id'] - 1
all_tweets+=results
else:
break
print('Total tweets: {}'.format(len(all_tweets)))
timeline_df = pd.DataFrame(all_tweets)
if len(timeline_df)>0:
timeline_df['created_at']=pd.to_datetime(timeline_df['created_at'])
return timeline_df
def get_likes(self, screen_name=None, user_id=None):
all_tweets = []
results = []
next_max_id = None
print('Extracting likes of: @{}'.format(screen_name or user_id))
while next_max_id==None or (results!=None and len(results) != 0):
try:
results = self.try_call(lambda: self.twitter.get_favorites(screen_name=screen_name, user_id=user_id, count=200, max_id=next_max_id, tweet_mode='extended'))
except:
#something went wrong: internet or auth
break
if results:
next_max_id = results[-1]['id'] - 1
all_tweets+=results
else:
break
print('Total likes: {}'.format(len(all_tweets)))
likes_df = pd.DataFrame(all_tweets)
if len(likes_df)>0:
likes_df['created_at']=pd.to_datetime(likes_df['created_at'])
return likes_df
def get_friends_list(self, screen_name=None, user_id=None):
all_friends = []
next_cursor = -1
response = None
print('Extracting friends of: @{}'.format(screen_name or user_id))
while next_cursor!=0 and (next_cursor==-1 or (response!=None and len(response['users']) != 0)):
print(next_cursor, response and len(response['users']), len(all_friends))
time.sleep(5)
try:
response = self.try_call(lambda: self.twitter.get_friends_list(screen_name=screen_name, user_id=user_id, count=200, cursor=next_cursor))
except:
#something went wrong: internet or auth
break
if response:
next_cursor = response['next_cursor']
all_friends+=response['users']
else:
break
print('Total friends: {}'.format(len(all_friends)))
friends_df = pd.DataFrame(all_friends)
return friends_df
def get_followers_list(self, screen_name=None, user_id=None):
all_friends = []
next_cursor = -1
response = None
print('Extracting followers of: @{}'.format(screen_name or user_id))
while next_cursor!=0 and (next_cursor==-1 or (response!=None and len(response['users']) != 0)):
print(next_cursor, response and len(response['users']), len(all_friends))
time.sleep(5)
try:
response = self.try_call(lambda: self.twitter.get_followers_list(screen_name=screen_name, user_id=user_id, count=200, cursor=next_cursor))
except:
#something went wrong: internet or auth
break
if response:
next_cursor = response['next_cursor']
all_friends+=response['users']
else:
break
print('Total followers: {}'.format(len(all_friends)))
friends_df = pd.DataFrame(all_friends)
return friends_df
def show_users(self, screen_names):
dict_user_details = {}
for screen_name in screen_names:
dict_user_details[screen_name] = self.try_call(lambda: self.twitter.show_user(screen_name=screen_name, include_entities=False))
return dict_user_details
def show_users_ids(self, ids):
dict_user_details = {}
for id_u in ids:
dict_user_details[id_u] = self.try_call(lambda: self.twitter.show_user(user_id=id_u, include_entities=False))
return dict_user_details
def try_call(self, call, deep=1, throw_errors = False):
try:
response = call()
return response
except TwythonRateLimitError as e:
time_to_wait = int(self.twitter.get_lastfunction_header('x-rate-limit-reset')) - int(time.time()) + 10
print('Rate limit exceded. {}th try. Waiting {} seconds: {}'.format(deep, time_to_wait, time.strftime('%X', time.localtime(time.time()+time_to_wait))))
time.sleep(time_to_wait)
return self.try_call(call, deep=deep+1)
except TwythonAuthError as e:
print('Auth error')
print(str(e))
if throw_errors:
raise
return
except TwythonError as e:
if deep>=6:
return
print('No internet. Waiting {} seconds'.format(10))
print(str(e))
if throw_errors:
raise
#time.sleep(10)
return #self.try_call(call, deep=deep+1)
oauth_token_secret=OAUTH_TOKEN_SECRET,
)
# https://twython.readthedocs.io/en/latest/api.html
# get the screen names I follow
# can only make 15 requests in a 15-minute window (1 per minute)
# https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-list
i_follow = []
cursor_if = -1
sleep_seconds = 60
counter = 1
while True:
logger.info(f"Query {counter}, I follow cursor: {cursor_if}")
result = twitter.get_friends_list(screen_name=MY_SCREEN_NAME,
count=200,
skip_status="true",
cursor=cursor_if)
if len(result["users"]) == 0:
break
else:
counter += 1
user_list = [user["screen_name"] for user in result["users"]]
i_follow.extend(user_list)
cursor_if = result["next_cursor"]
logger.info(
f"Added {len(user_list)} users who I follow (total: {len(i_follow)})")
# # find the screen names with notifications turned on
# user_list_notif = [
# user["screen_name"] for user in ifollow["users"] if user["notifications"]
# ]
# logger.info(
friendslen.append(len(friends))
followerslen.append(len(followers))
# Calculate time for Twython
start = time.time()
twython_consumerkey = '2IiwQDV5c4OeUEA8PI0aqvg5g'
twython_consumersecret = 'RqEDv68Brt3JKVetWAeNFUObP5VeeRloBbQ0V9eqRFCerbas1I'
twitter = Twython(twython_consumerkey, twython_consumersecret, oauth_version=2)
twython_accesstoken = twitter.obtain_access_token()
twitter = Twython(twython_consumerkey, access_token=twython_accesstoken)
friends = twitter.get_friends_list(screen_name=u, count=n)
followers = twitter.get_followers_list(screen_name=u, count=n)
end = time.time()
twittersec.append(end - start)
friendslen.append(len(friends))
followerslen.append(len(followers))
# Calculating time for TweetPony
start = time.time()
api = tweetpony.API(
consumer_key="s08vLRIM5VnqCUqFVYTaV3ET9",
consumer_secret="q4N5vB5KQrakhjAcWAk8dKOeTn4896cURoAg33A37UjGG2jJxf",
class Tweety(object):
"""
Twitter client using Twython
"""
def __init__(self, app_key=TWITTER_APP_KEY, app_secret=TWITTER_APP_SECRET,
oauth_token=TWITTER_OAUTH_TOKEN, oauth_token_secret=TWITTER_OAUTH_SECRET):
try:
self.twitter = Twython(
app_key,
app_secret,
oauth_token,
oauth_token_secret)
self.cache = RedisCache({
'server': REDIS_SERVER,
'port': REDIS_PORT,
'database': REDIS_DATABASE,
'key_prefix': 'domus-twitter'
})
except TwythonAuthError:
raise Exception("Unable to connect to twitter")
def __get_friends(self):
"""
Using twitter get_friends and redis, gets a list of screen names
:return:a list of twitter users
"""
results = self.cache.get('twitter_friends')
if not results:
try:
results = [a['screen_name'] for a in self.twitter.get_friends_list()['users']]
self.cache.store('twitter_friends',json.dumps(results), expires=120)
except (TwythonError, TwythonRateLimitError):
raise Exception('Unable to get followers list')
else:
results = json.loads(results)
return results
def tweet(self, message,to_friends=False):
"""
Writtes a twit
:param message: what to tweet
:param to_friends: send to all friends?
:return:
"""
try:
if to_friends:
for a_friend in self.__get_friends():
mention = "@{} ".format(a_friend)
available_chars = 140 - len(mention)
self.twitter.update_status(
status=(mention+message)[:available_chars] + ((mention+message)[(available_chars-2):] and '..'))
else:
self.twitter.update_status(
status=message[:140] + (message[138:] and '..'))
except (TwythonError, TwythonRateLimitError):
raise Exception("Unable to post update")
def dm(self, user, message):
try:
self.twitter.send_direct_message(screen_name=user, text=message)
except TwythonError:
raise Exception("Unable to send dm to {}".format(user))
def get_dms(self):
"""
Gets a list of dms. Stores the last id seen so the next request is for the new messages only.
:return: a dict of the form {tweet_id:{sender:screen_name,text:the_message}}
"""
results = {}
dms = []
last_id = self.cache.get('twitter_last_dm')
if last_id:
dms = self.twitter.get_direct_messages(count=100,since_id = last_id)
else:
dms = self.twitter.get_direct_messages(count=100)
if dms:
last_id = 0
for a_dm in dms:
results[a_dm['id']] = {'from':a_dm['sender_screen_name'],'text': a_dm['text']}
last_id = a_dm['id'] if a_dm['id'] > last_id else last_id
self.cache.store('twitter_last_dm', last_id)
return results
def get_mentions(self):
"""
Gets a list of mentions. Stores the last id seen so the next request is for the new messages only.
:return: a dict of the form {tweet_id:{sender:screen_name,text:the_message}}
"""
results = {}
mentions = []
last_id = self.cache.get('twitter_last_mention')
if last_id:
mentions = self.twitter.get_mentions_timeline(count=100,since_id = last_id)
else:
mentions = self.twitter.get_mentions_timeline(count=100)
if mentions:
last_id = 0
for a_mention in mentions:
results[a_mention['id']] = {'from':a_mention['user']['screen_name'],'text': a_mention['text']}
last_id = a_mention['id'] if a_mention['id'] > last_id else last_id
self.cache.store('twitter_last_mention', last_id)
return results
class TwythonEndpointsTestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
# Timelines
@unittest.skip('skipping non-updated test')
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
@unittest.skip('skipping non-updated test')
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
@unittest.skip('skipping non-updated test')
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
@unittest.skip('skipping non-updated test')
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
@unittest.skip('skipping non-updated test')
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
@unittest.skip('skipping non-updated test')
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
@unittest.skip('skipping non-updated test')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
@unittest.skip('skipping non-updated test')
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
@unittest.skip('skipping non-updated test')
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
@unittest.skip('skipping non-updated test')
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
@unittest.skip('skipping non-updated test')
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
@unittest.skip('skipping non-updated test')
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time()))
# Friends & Followers
@unittest.skip('skipping non-updated test')
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
@unittest.skip('skipping non-updated test')
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
@unittest.skip('skipping non-updated test')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
@unittest.skip('skipping non-updated test')
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
@unittest.skip('skipping non-updated test')
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
@unittest.skip('skipping non-updated test')
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
@unittest.skip('skipping non-updated test')
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
@unittest.skip('skipping non-updated test')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
@unittest.skip('skipping non-updated test')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
@unittest.skip('skipping non-updated test')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
@unittest.skip('skipping non-updated test')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
@unittest.skip('skipping non-updated test')
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
@unittest.skip('skipping non-updated test')
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
@unittest.skip('skipping non-updated test')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
@unittest.skip('skipping non-updated test')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
@unittest.skip('skipping non-updated test')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
@unittest.skip('skipping non-updated test')
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
@unittest.skip('skipping non-updated test')
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
@unittest.skip('skipping non-updated test')
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
@unittest.skip('skipping non-updated test')
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
@unittest.skip('skipping non-updated test')
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
@unittest.skip('skipping non-updated test')
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
@unittest.skip('skipping non-updated test')
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
@unittest.skip('skipping non-updated test')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
@unittest.skip('skipping non-updated test')
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
@unittest.skip('skipping non-updated test')
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
@unittest.skip('skipping non-updated test')
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
@unittest.skip('skipping non-updated test')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
@unittest.skip('skipping non-updated test')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
@unittest.skip('skipping non-updated test')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
@unittest.skip('skipping non-updated test')
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
@unittest.skip('skipping non-updated test')
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
@unittest.skip('skipping non-updated test')
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
@unittest.skip('skipping non-updated test')
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
@unittest.skip('skipping non-updated test')
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
@unittest.skip('skipping non-updated test')
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
@unittest.skip('skipping non-updated test')
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
@unittest.skip('skipping non-updated test')
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
def test_construct_api_url(self):
"""Test constructing a Twitter API url works as we expect"""
url = 'https://api.twitter.com/1.1/search/tweets.json'
constructed_url = self.api.construct_api_url(url, q='#twitter')
self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')
def test_get(self):
"""Test Twython generic GET request works"""
self.api.get('account/verify_credentials')
def test_post(self):
"""Test Twython generic POST request works, with a full url and
with just an endpoint"""
update_url = 'https://api.twitter.com/1.1/statuses/update.json'
status = self.api.post(update_url, params={'status': 'I love Twython!'})
self.api.post('statuses/destroy/%s' % status['id_str'])
def test_get_lastfunction_header(self):
"""Test getting last specific header of the last API call works"""
self.api.get('statuses/home_timeline')
self.api.get_lastfunction_header('x-rate-limit-remaining')
def test_get_lastfunction_header_not_present(self):
"""Test getting specific header that does not exist from the last call returns None"""
self.api.get('statuses/home_timeline')
header = self.api.get_lastfunction_header('does-not-exist')
self.assertEqual(header, None)
def test_get_lastfunction_header_no_last_api_call(self):
"""Test attempting to get a header when no API call was made raises a TwythonError"""
self.assertRaises(TwythonError, self.api.get_lastfunction_header,
'no-api-call-was-made')
def test_search_gen(self):
"""Test looping through the generator results works, at least once that is"""
search = self.api.search_gen('twitter', count=1)
counter = 0
while counter < 2:
counter += 1
result = next(search)
new_id_str = int(result['id_str'])
if counter == 1:
prev_id_str = new_id_str
time.sleep(1) # Give time for another tweet to come into search
if counter == 2:
self.assertTrue(new_id_str > prev_id_str)
def test_encode(self):
"""Test encoding UTF-8 works"""
self.api.encode('Twython is awesome!')
def test_html_for_tweet(self):
"""Test HTML for Tweet returns what we want"""
tweet_text = self.api.html_for_tweet(test_tweet_object)
self.assertEqual(test_tweet_html, tweet_text)
def test_html_for_tweet_expanded_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object,
use_expanded_url=True)
# Make sure full url is in HTML
self.assertTrue('http://google.com' in tweet_text)
def test_html_for_tweet_short_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object, False)
# Make sure HTML doesn't contain the display OR exapanded url
self.assertTrue(not 'http://google.com' in tweet_text)
self.assertTrue(not 'google.com' in tweet_text)
def test_raise_error_on_bad_ssl_cert(self):
"""Test TwythonError is raised by a RequestException when an actual HTTP happens"""
self.assertRaises(TwythonError, self.api.get, 'https://example.com')
# Timelines
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get deleted :(')
self.api.destroy_status(id=status['id_str'])
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man!')
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(list_id=test_list_id)
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff')
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed')
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(list_id=test_list_id)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(list_id=test_list_id)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(list_id=test_list_id,
screen_name=screen_name)
self.api.unsubscribe_from_list(list_id=test_list_id)
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(list_id=test_list_id, screen_name='jack')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(list_id=test_list_id)
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(list_id=test_list_id)
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
tweets = twitter.get_home_timeline(count=1)
for tweet in tweets:
print(tweet['text'])
print
print "My Settings"
print "==========="
settings = twitter.get_account_settings()
#print settings
print "I am @%s" % settings['screen_name']
print
print "My friends"
print "=========="
list = twitter.get_friends_list()
for friend in list['users']:
print "I follow user @%s, name %s, id %d" % (friend['screen_name'],
friend['name'], friend['id'])
print
print "My followers"
print "============"
list = twitter.get_followers_list()
for friend in list['users']:
print "User @%s, name %s, id %d is following me" % (
friend['screen_name'], friend['name'], friend['id'])
#print friend
class Crawler(object):
"""
Crawler class
"""
# initial
def __init__(self):
# when load normal json, disable
print("...")
# own cursor reader
def cursorReader(self, query, count, max_id, language, geocode,
result_type, until):
n = 2 # the deep of loop for checking cursor
for i in range(0, n):
print("searching with max_id parameter: " + str(max_id))
result = self._tw.search(q=query,
max_id=max_id,
count=count,
lang=language,
geocode=geocode,
result_type=result_type,
until=until,
tweet_mode="extended")
# print(result["search_metadata"])
metadata = result["search_metadata"].get("next_results")
if metadata:
max_id = furl(metadata).args['max_id']
print("\n======================================\n")
print("found new max_id: " + str(max_id))
return max_id
break
else:
print("can't find max_id in metadata... retrying..")
print("searching with since_id parameter: " + str(max_id))
result = self._tw.search(q=query,
since_id=max_id,
count=count,
lang=language,
geocode=geocode,
result_type=result_type,
until=until,
tweet_mode="extended")
max_id = result['statuses'][-1].get("id")
print("retying using tweet id: " + str(max_id))
time.sleep(random.randint(5, 10))
# last check
if i == n - 1:
result = self._tw.search(q=query,
count=count,
lang=language,
geocode=geocode,
result_type=result_type,
until=until,
tweet_mode="extended")
max_id = result['statuses'][0].get("id")
print("udahan mz, kaga nemu apa-apa... pake ini aje.." +
str(max_id) + " atulah.. :(")
return max_id
break
else:
pass
# search handling
def searchHandler(self, query, count, querytype, language, geocode,
result_type, until):
sequence = 0
max_id = 0
while True:
print("current sequence: " + str(sequence))
randomizer = random.randint(0, len(tokens) - 1)
keys = keyRotator().rotator(sequence=randomizer)
consumer_key, consumer_key_secret, access_token, access_token_secret = keys
print("using key: " + str(consumer_key) + "\n")
try:
self._tw_auth = Twython(consumer_key,
consumer_key_secret,
oauth_version=2)
OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token()
self._tw = Twython(consumer_key,
access_token=OAUTH2_ACCESS_TOKEN)
except TwythonError as e:
print("error on Twython!")
result = self._tw.search(q=query,
max_id=max_id,
count=count,
lang=language,
geocode=geocode,
result_type=result_type,
until=until,
tweet_mode="extended")
# check latest tweet
created_at = result["statuses"][-1].get("created_at")
unix_time = time.mktime(
time.strptime(created_at, "%a %b %d %H:%M:%S +0000 %Y")
) + 25200 # quickhack Jakarta timezone
when = datetime.utcfromtimestamp(unix_time).strftime(
'%Y-%m-%d %H:%M:%S')
print("latest tweet created at: " + str(when))
# preprocessing data
print("preprocessing data...")
self.preprocessingData(result, query, sequence, querytype)
print("producing csv...")
self.produceCsv(post_data, query, querytype)
print("current API rate limit: " + str(
self._tw.get_lastfunction_header('x-rate-limit-remaining')))
# metadata finder
metadata = result["search_metadata"].get("next_results")
if metadata:
max_id_finder = self.cursorReader(query=query,
count=count,
max_id=max_id,
language=language,
geocode=geocode,
result_type=result_type,
until=until)
max_id = max_id_finder
else:
print("oops.. no max_id again.. \nlatest max_id: " +
str(max_id))
break
# adding new sequence
sequence += 1
# print("sleeping...")
# time.sleep(random.randint(1,5))
# stream handling
def streamHandler(self, querytype, query, language, geocode, follow):
randomizer = random.randint(0, len(tokens) - 1)
keys = keyRotator().rotator(sequence=randomizer)
consumer_key, consumer_key_secret, access_token, access_token_secret = keys
print("using key: " + str(consumer_key) + "\n")
try:
self._tw_stream = Streamer(app_key=consumer_key,
app_secret=consumer_key_secret,
oauth_token=access_token,
oauth_token_secret=access_token_secret,
chunk_size=stream_stopper,
retry_in=300)
self._tw_stream.statuses.filter(track=query,
lang=language,
locations=geocode,
follow=follow,
tweet_mode="extended")
except requests.ConnectionError:
print("connection error..")
sys.exit(0)
if len(stream_data) == stream_stopper:
for i in stream_data:
self.jsonCruncher(i, querytype)
stream_data[:] = []
self.produceCsv(post_data, query, querytype)
print("\n==========================\n")
else:
pass
# followers handling
def followerListHandler(self, query, count, querytype):
sequence = 0
next_cursor = -1
while True:
print("current sequence: " + str(sequence))
print("current cursor: " + str(next_cursor))
randomizer = random.randint(0, len(tokens) - 1)
keys = keyRotator().rotator(sequence=randomizer)
consumer_key, consumer_key_secret, access_token, access_token_secret = keys
print("using key: " + str(consumer_key))
try:
self._tw_auth = Twython(consumer_key,
consumer_key_secret,
oauth_version=2)
OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token()
self._tw = Twython(consumer_key,
access_token=OAUTH2_ACCESS_TOKEN)
except TwythonError as e:
print("error on Twython!")
try:
result = self._tw.get_followers_list(screen_name=query,
cursor=next_cursor,
count=count)
except TwythonError as e:
if e.error_code == 503:
print(
"Twitter API returned a 503 (Service Unavailable), Over capacity.."
)
print("shuting down..")
break
self.preprocessingData(result, query, sequence, querytype)
print("producing csv...")
self.produceCsv(user_data, query, querytype)
print("current API rate limit: " + str(
self._tw.get_lastfunction_header('x-rate-limit-remaining')))
# metadata finder
metadata = result.get("next_cursor")
if metadata:
next_cursor = metadata
else:
break
# adding new sequence
sequence += 1
print("sleeping...")
time.sleep(random.randint(1, 5))
print("\n==========================\n")
# following handling
def followingListHandler(self, query, count, querytype):
sequence = 0
next_cursor = -1
while True:
print("current sequence: " + str(sequence))
print("current cursor: " + str(next_cursor))
randomizer = random.randint(0, len(tokens) - 1)
keys = keyRotator().rotator(sequence=randomizer)
consumer_key, consumer_key_secret, access_token, access_token_secret = keys
print("using key: " + str(consumer_key))
try:
self._tw_auth = Twython(consumer_key,
consumer_key_secret,
oauth_version=2)
OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token()
self._tw = Twython(consumer_key,
access_token=OAUTH2_ACCESS_TOKEN)
except TwythonError as e:
print("error on Twython!")
try:
result = self._tw.get_friends_list(screen_name=query,
cursor=next_cursor,
count=count)
except TwythonError as e:
if e.error_code == 503:
print(
"Twitter API returned a 503 (Service Unavailable), Over capacity.."
)
print("shuting down..")
break
self.preprocessingData(result, query, sequence, querytype)
print("producing csv...")
self.produceCsv(user_data, query, querytype)
print("current API rate limit: " + str(
self._tw.get_lastfunction_header('x-rate-limit-remaining')))
# metadata finder
metadata = result.get("next_cursor")
if metadata:
next_cursor = metadata
else:
break
# adding new sequence
sequence += 1
print("sleeping...")
time.sleep(random.randint(1, 5))
print("\n==========================\n")
# user_timeline handling
def userTimeline(self, query, count, querytype):
sequence = 0
max_id = 0
while True:
print("current sequence: " + str(sequence))
randomizer = random.randint(0, len(tokens) - 1)
keys = keyRotator().rotator(sequence=randomizer)
consumer_key, consumer_key_secret, access_token, access_token_secret = keys
print("using key: " + str(consumer_key) + "\n")
try:
self._tw_auth = Twython(consumer_key,
consumer_key_secret,
oauth_version=2)
OAUTH2_ACCESS_TOKEN = self._tw_auth.obtain_access_token()
self._tw = Twython(consumer_key,
access_token=OAUTH2_ACCESS_TOKEN)
except TwythonError as e:
print("error on Twython!")
try:
if max_id == 0:
result = self._tw.get_user_timeline(screen_name=query,
count=count,
tweet_mode="extended")
else:
result = self._tw.get_user_timeline(screen_name=query,
max_id=max_id,
count=count,
tweet_mode="extended")
result.pop(0)
except TwythonError as e:
print("error on Twython!")
if e.error_code == 503:
print(
"Twitter API returned a 503 (Service Unavailable), Over capacity.."
)
print("shuting down..")
break
# metadata finder
metadata = result[-1].get("id")
if metadata:
max_id = metadata
else:
print("oops.. no max_id again.. \nlatest max_id: " +
str(max_id))
break
# check latest tweet
created_at = result[-1].get("created_at")
unix_time = time.mktime(
time.strptime(created_at, "%a %b %d %H:%M:%S +0000 %Y")
) + 25200 # quickhack Jakarta timezone
when = datetime.utcfromtimestamp(unix_time).strftime(
'%Y-%m-%d %H:%M:%S')
print("latest tweet created at: " + str(when))
# preprocessing data
print("preprocessing data...")
self.preprocessingData(result, query, sequence, querytype)
print("producing csv...")
self.produceCsv(post_data, query, querytype)
print("current API rate limit: " + str(
self._tw.get_lastfunction_header('x-rate-limit-remaining')))
# adding new sequence
sequence += 1
# ==========================
# MAIN ENGINE
# ==========================
def crawl(self, querytype, query, count, authentication, language, geocode,
result_type, follow, until):
if querytype == "search":
# todos: loop moved here
self.searchHandler(querytype=querytype,
query=query,
count=count,
language=language,
geocode=geocode,
result_type=result_type,
until=until)
# DEBUG
# result = self._tw.search(q=query, count=count, result_type="mixed", tweet_mode="extended")
# print(result["statuses"])
# result = self._tw.show_status(id=815411895343587330)
elif querytype == "stream":
timeout = time.time() + 60 * 360 # 4 hours
while True:
if time.time() > timeout:
break
self.streamHandler(querytype=querytype,
query=query,
language=language,
geocode=geocode,
follow=follow)
elif querytype == "followers":
self.followerListHandler(query=query,
count=count,
querytype=querytype)
elif querytype == "following":
self.followingListHandler(query=query,
count=count,
querytype=querytype)
elif querytype == "user_timeline":
self.userTimeline(query=query, count=count, querytype=querytype)
else:
print("Your API type not found in this function..")
# ==========================
# END MAIN ENGINE
# ==========================
# preprocessing data
def preprocessingData(self, data, query, sequence, querytype):
self.saveRawJson(data, query, sequence, querytype)
if querytype in {"followers", "following"}:
raw_users = data['users']
for users in raw_users:
self.jsonCruncher(users, querytype)
elif querytype in {"search", "stream"}:
raw_statuses = data['statuses']
for statuses in raw_statuses:
self.jsonCruncher(statuses, querytype)
elif querytype in {"user_timeline"}:
raw_statuses = data
for statuses in raw_statuses:
self.jsonCruncher(statuses, querytype)
# json specialist chruncher
def jsonCruncher(self, data, querytype):
temp_data = []
if querytype in {"followers", "following"}:
temp_data.append(data.get("id")) # user_id
temp_data.append(data.get("screen_name")) # screen_name
temp_data.append(data.get("name")) # user_name
temp_data.append(data.get("created_at")) # created_at
temp_data.append(data.get("protected")) # is_protected
temp_data.append(data.get("verified")) # is_verified
temp_data.append(data.get("description")) # description
if data["entities"].get("description"): # description urls
user_description_urls_list = []
for i in data["entities"]["description"]["urls"]:
user_description_urls_list.append(
str(i.get("expanded_url")))
user_description_urls = ", ".join(user_description_urls_list)
temp_data.append(user_description_urls)
else:
temp_data.append("None")
if data["entities"].get("url"): # urls
user_urls_list = []
for i in data["entities"]["url"]["urls"]:
user_urls_list.append(str(i.get("expanded_url")))
user_urls = ", ".join(user_urls_list)
temp_data.append(user_urls)
else:
temp_data.append("None")
temp_data.append(
data.get("profile_image_url")) # profile_image_url
temp_data.append(data.get("profile_background_image_url")
) # profile_background_image_url
temp_data.append(data.get("statuses_count")) # statuses_count
temp_data.append(data.get("followers_count")) # followers_count
temp_data.append(data.get("friends_count")) # friends_count
temp_data.append(data.get("favourites_count")) # favourites_count
temp_data.append(data.get("blocked_by")) # blocked_by
temp_data.append(data.get("geo_enabled")) # geo_enabled
temp_data.append(data.get("location")) # location
temp_data.append(data.get("lang")) # lang
temp_data.append(data.get("time_zone")) # time_zone
temp_data.append(data.get("utc_offset")) # utc_offset
# clean up
temp_data_replaced = [
"None" if v is None else v for v in temp_data
]
# send it to post_data
user_data.append(temp_data_replaced)
elif querytype in {"search", "stream", "user_timeline"}:
# user
temp_data.append(data["user"].get("id")) # user_id
temp_data.append(data["user"].get("verified")) # user is verified
temp_data.append(
data["user"].get("protected")) # user is protected
temp_data.append(
data["user"].get("screen_name")) # user screen_name
temp_data.append(data["user"].get("name")) # user name
temp_data.append(data["user"].get("created_at")) # user created_at
temp_data.append(
data["user"].get("followers_count")) # user followers_count
temp_data.append(
data["user"].get("statuses_count")) # user statuses_count
temp_data.append(
data["user"].get("friends_count")) # user following_count
temp_data.append(data["user"].get("location")) # user location
temp_data.append(data["user"].get("utc_offset")) # user utc_offset
temp_data.append(data["user"].get(
"profile_image_url")) # user profile_image_url
temp_data.append(data["user"].get(
"profile_banner_url")) # user profile_banner_url
temp_data.append(data["user"].get("profile_background_image_url")
) # user profile_background_image_url
temp_data.append(
data["user"].get("description")) # user description
temp_data.append(data["user"].get("time_zone")) # user time_zone
temp_data.append(data["user"].get("url")) # user url (profile_url)
# tweet
temp_data.append(data.get("id")) # tweet id (status_id)
temp_data.append(data.get("created_at")) # tweet created_at
temp_data.append(data.get("lang")) # tweet lang
if data.get("full_text"):
temp_data.append(' '.join(
data.get("full_text").split())) # tweet text if 280 char
else:
temp_data.append(' '.join(
data.get("full_text").split())) # tweet text if 140 char
temp_data.append(data.get("source")) # tweet source
temp_data.append(
data.get("possibly_sensitive")) # tweet possibly_sensitive
if data.get("coordinates"):
longitude, latitude = data["coordinates"].get("coordinates")
temp_data.append(str(longitude) + ", " +
str(latitude)) # tweet coordinates joined
temp_data.append(longitude) # tweet_coordinates_long
temp_data.append(latitude) # tweet_coordinates_lat
else:
temp_data.append(
"None") # set to none -- # tweet coordinates joined
temp_data.append(
"None") # set to none -- # tweet_coordinates_long
temp_data.append(
"None") # set to none -- # tweet_coordinates_at
if data.get("geo"):
longitude, latitude = data["geo"].get("coordinates")
temp_data.append(str(longitude) + ", " +
str(latitude)) # tweet geo joined
temp_data.append(longitude) # tweet_geo_long
temp_data.append(latitude) # tweet_geo_lat
else:
temp_data.append("None") # set to none -- # tweet geo joined
temp_data.append("None") # set to none -- # tweet_geo_long
temp_data.append("None") # set to none -- # tweet_geo_at
if data.get('place'):
temp_data.append(
data['place'].get("country")) # tweet place country
temp_data.append(data['place'].get("name")) # tweet place name
else:
temp_data.append("None") # set to none
temp_data.append("None") # set to none
temp_data.append(data.get("retweeted")) # tweet retweeted
temp_data.append(data.get("favorited")) # tweet favorited
temp_data.append(data.get("retweet_count")) # tweet retweet_count
temp_data.append(
data.get("favorite_count")) # tweet favorite_count
temp_data.append(
data.get("is_quote_status")) # tweet favorite_count
if data.get("quoted_status_id"):
temp_data.append(
data.get("quoted_status_id")) # tweet qouted_status_id
else:
temp_data.append("None") # if not qouting, set to none
temp_data.append(data.get(
"in_reply_to_status_id")) # tweet in_reply_to_status_id
temp_data.append(data.get(
"in_reply_to_screen_name")) # tweet in_reply_to_screen_name
temp_data.append(
data.get("in_reply_to_user_id")) # tweet in_reply_to_user_id
# tweet entities - symbols
if data["entities"].get("symbols"):
tweet_entities_symbols_list = []
for i in data["entities"].get("symbols"):
tweet_entities_symbols_list.append(
i.get("text")) # extract text to list
tweet_entities_symbols = ", ".join(
tweet_entities_symbols_list) # join it
temp_data.append(
tweet_entities_symbols) # append it to temp_data
else:
temp_data.append("None") # empty symbols -- set to none
# tweet entities - user_mentions
if data["entities"].get("user_mentions"):
tweet_entities_user_mentions_id_list = []
tweet_entities_user_mentions_screen_name_list = []
tweet_entities_user_mentions_name_list = []
for i in data["entities"].get("user_mentions"):
tweet_entities_user_mentions_id_list.append(
str(i.get("id")))
tweet_entities_user_mentions_screen_name_list.append(
i.get("screen_name"))
tweet_entities_user_mentions_name_list.append(
i.get("name"))
tweet_entities_user_mentions_id = ", ".join(
tweet_entities_user_mentions_id_list
) # extract id user_mentions
tweet_entities_user_mentions_screen_name = ", ".join(
tweet_entities_user_mentions_screen_name_list
) # extract screen_name user_mentions
tweet_entities_user_mentions_name = ", ".join(
tweet_entities_user_mentions_name_list
) # extract name user_mentions
temp_data.append(tweet_entities_user_mentions_id
) # append entities_id to temp_data
temp_data.append(tweet_entities_user_mentions_screen_name
) # append entities_screen_name to temp_data
temp_data.append(tweet_entities_user_mentions_name
) # append entities_name to temp_data
else:
temp_data.append("None") # empty user_mentions_id set to none
temp_data.append(
"None") # empty user_mentions_screen_name set to none
temp_data.append(
"None") # empty user_mentions_name set to none
# tweet entities - hashtag
if data["entities"].get("hashtags"):
tweet_entities_hashtags_list = []
for i in data["entities"].get("hashtags"):
tweet_entities_hashtags_list.append(
i.get("text")) # extract hashtags
tweet_entities_hashtags = ", ".join(
tweet_entities_hashtags_list) # join to new string
temp_data.append(
tweet_entities_hashtags) # append it to temp_data
else:
temp_data.append("None") # empty hashtags
# tweet media
if data.get("extended_entities"
): # check whenever extended_entities exist or not
medias = data["extended_entities"]["media"]
# temp list
tweet_extended_entitites_media_type_list = []
tweet_extended_entitites_media_id_list = []
tweet_extended_entitites_media_monetizable_list = []
tweet_extended_entitites_media_urls_list = []
# tweet_extended_entities_media_video_duration_millis_list = []
# look up
for media in medias:
tweet_extended_entitites_media_type_list.append(
media.get("type")) # extract media_type
tweet_extended_entitites_media_id_list.append(
str(media.get("id"))) # extract media_id
tweet_extended_entitites_media_monetizable_list.append(
str(media.get(
"monetizable"))) # extract media_monetizable
tweet_extended_entitites_media_urls_list.append(
str(media.get("media_url"))) # extract media_url
# joins
tweet_extended_entitites_media_type = ", ".join(
tweet_extended_entitites_media_type_list)
tweet_extended_entitites_media_id = ", ".join(
tweet_extended_entitites_media_id_list)
tweet_extended_entitites_media_monetizable = ", ".join(
tweet_extended_entitites_media_monetizable_list)
tweet_extended_entitites_media_urls = ", ".join(
tweet_extended_entitites_media_urls_list)
# append
temp_data.append(
tweet_extended_entitites_media_type
) # append tweet_extended_entitites_media_type
temp_data.append(tweet_extended_entitites_media_id
) # append tweet_extended_entitites_media_id
temp_data.append(
tweet_extended_entitites_media_monetizable
) # append tweet_extended_entitites_media_monetizable
temp_data.append(
tweet_extended_entitites_media_urls
) # append tweet_extended_entitites_media_urls
else:
temp_data.append(
"None"
) # empty tweet_extended_entitites_media_type set to none
temp_data.append(
"None"
) # empty tweet_extended_entitites_media_id set to none
temp_data.append(
"None"
) # empty tweet_extended_entitites_media_monetizable set to none
temp_data.append(
"None"
) # empty tweet_extended_entitites_media_urls set to none
# clean up
temp_data_replaced = [
"None" if v is None else v for v in temp_data
]
# send it to post_data
post_data.append(temp_data_replaced)
else:
print("failed to crunch data..")
# create csv
def produceCsv(self, data, query, querytype):
if querytype in {"search", "stream", "user_timeline"}:
df = pd.DataFrame.from_records(data, columns=post_data_labels)
elif querytype in {"following", "followers"}:
df = pd.DataFrame.from_records(data, columns=user_data_labels)
if query.startswith("#"):
filename = "hashtag-" + query.replace("#", "")
elif query.startswith("@"):
filename = "user-" + query.replace("@", "")
elif " " in query == True:
print("ada spasi..")
filename = str(query.replace(" ", "_"))
elif " " in query:
filename = str(query.replace(" ", "_").replace(",", ""))
else:
filename = "keyword-" + query
filename = filename
dir_path = './result'
today = time.strftime("%Y_%m_%d")
fullfilename = str(querytype) + "-" + filename + '-' + today + '.csv'
filepath = os.path.join(dir_path, fullfilename)
if os.path.isfile(filepath):
# when result csv exist
df.to_csv(filepath,
mode="a",
header=False,
index=False,
encoding="utf-8-sig",
line_terminator="\r\n")
print("appending to exisiting csv file at: " + str(filepath))
print("cleaning up temporary data...")
post_data[:] = []
else:
# when result csv NOT exist
df.to_csv(filepath,
index=False,
encoding="utf-8-sig",
line_terminator="\r\n")
print("writing new csv file at: " + str(filepath))
print("cleaning up temporary data...")
post_data[:] = []
# save raw json file
def saveRawJson(self, data, query, sequence, querytype):
today = time.strftime("%Y_%m_%d")
if query.startswith("#"):
filename = "hashtag-" + query.replace("#", "")
elif query.startswith("@"):
filename = "user-" + query.replace("@", "")
elif " " in query:
filename = str(query.replace(" ", "_").replace(",", ""))
else:
filename = "keyword-" + query
dir_path = os.path.join('./data', filename)
fullfilename = str(
querytype) + "-" + filename + '-' + today + '-' + str(
sequence) + ".json"
file_path = os.path.join(dir_path, fullfilename)
# check dirs
if not os.path.exists(dir_path):
os.makedirs(dir_path)
else:
pass
# writer get ready
file_writer = open(file_path, "w")
file_appender = open(file_path, "a")
raw_json = json.dumps(data)
# writer on action
if os.path.isfile(file_path):
file_appender.write(raw_json.decode("utf-8"))
else:
file_writer.write(raw_json.decode("utf-8"))
REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token"
AUTHORIZE_URL = "https://api.twitter.com/oauth/authorize?oauth_token="
ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token"
CONSUMER_KEY = "K7ddHMoZcHXu0WyWdnp6wdQxf"
CONSUMER_SECRET = "OcvSpsGJ5e1f9GvHbT1y5TXLzwETVkL9wkd1WDzL3FHgC7WDzk"
ACCESS_KEY = "2539409241-FFZPA5lh2vZklUx3q5jASFesRVVXPxqfS6nY8QB"
ACCESS_SECRET = "zzQVXih1Gy2cR1WdMtcMhmXbWpi1S45Jc7vlLlkHFWwm9"
twitter = Twython(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET)
user_Friends = twitter.get_friends_list(screen_name="Francis_Pruter", count=200, skip_status='true', include_user_entities='false')
ctr = 1
fcountlist={}
for friend in user_Friends['users']:
fcountlist[friend['name']] = friend['friends_count']
ctr = ctr+1
fcountlist["ME"] = ctr-1
#sort list by friend count https://docs.python.org/2/library/collections.html#ordereddict-examples-and-recipes
fcountlist = OrderedDict(sorted(fcountlist.items(), key=lambda t: t[1]))
class TwythonEndpointsTestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
# This is so we can hit coverage that Twython sets
# User-Agent for us if none is supplied
oauth2_client_args = {
'headers': {}
}
self.api = Twython(app_key, app_secret,
oauth_token, oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key, access_token=access_token,
client_args=oauth2_client_args)
# Timelines
@unittest.skip('skipping non-updated test')
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
@unittest.skip('skipping non-updated test')
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(screen_name='twitter')
# Random User Timeline
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError, self.api.get_user_timeline,
screen_name=protected_twitter_2)
@unittest.skip('skipping non-updated test')
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
@unittest.skip('skipping non-updated test')
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
@unittest.skip('skipping non-updated test')
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
@unittest.skip('skipping non-updated test')
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(status='Test post just to get \
deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
@unittest.skip('skipping non-updated test')
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
@unittest.skip('skipping non-updated test')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
@unittest.skip('skipping non-updated test')
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
@unittest.skip('skipping non-updated test')
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
@unittest.skip('skipping non-updated test')
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
@unittest.skip('skipping non-updated test')
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s\
' % int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
@unittest.skip('skipping non-updated test')
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError, self.api.send_direct_message,
screen_name=protected_twitter_2, text='Yo, man! \
%s' % int(time.time()))
# Friends & Followers
@unittest.skip('skipping non-updated test')
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
@unittest.skip('skipping non-updated test')
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
@unittest.skip('skipping non-updated test')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
@unittest.skip('skipping non-updated test')
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
@unittest.skip('skipping non-updated test')
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
@unittest.skip('skipping non-updated test')
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
@unittest.skip('skipping non-updated test')
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
@unittest.skip('skipping non-updated test')
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
@unittest.skip('skipping non-updated test')
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
@unittest.skip('skipping non-updated test')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError, self.api.update_delivery_service,
device='none')
@unittest.skip('skipping non-updated test')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
@unittest.skip('skipping non-updated test')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
@unittest.skip('skipping non-updated test')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
@unittest.skip('skipping non-updated test')
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
@unittest.skip('skipping non-updated test')
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
@unittest.skip('skipping non-updated test')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
@unittest.skip('skipping non-updated test')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
@unittest.skip('skipping non-updated test')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError, self.api.get_contributors,
screen_name=screen_name)
@unittest.skip('skipping non-updated test')
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
@unittest.skip('skipping non-updated test')
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
@unittest.skip('skipping non-updated test')
def test_list_mutes(self):
"""Test listing users who are muted by the authenticated user
succeeds"""
self.api.list_mutes()
@unittest.skip('skipping non-updated test')
def test_list_mute_ids(self):
"""Test listing user ids who are muted by the authenticated user
succeeds"""
self.api.list_mute_ids()
@unittest.skip('skipping non-updated test')
def test_create_mute(self):
"""Test muting a user succeeds"""
self.api.create_mute(screen_name='justinbieber')
@unittest.skip('skipping non-updated test')
def test_destroy_mute(self):
"""Test muting a user succeeds"""
self.api.destroy_mute(screen_name='justinbieber')
# Suggested Users
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
@unittest.skip('skipping non-updated test')
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
@unittest.skip('skipping non-updated test')
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
@unittest.skip('skipping non-updated test')
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
@unittest.skip('skipping non-updated test')
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id, name='Stuff Renamed \
%s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id,
screen_name=screen_names)
self.api.delete_list_members(list_id=list_id,
screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id,
screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
@unittest.skip('skipping non-updated test')
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
@unittest.skip('skipping non-updated test')
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
@unittest.skip('skipping non-updated test')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name)
@unittest.skip('skipping non-updated test')
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
@unittest.skip('skipping non-updated test')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
@unittest.skip('skipping non-updated test')
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
@unittest.skip('skipping non-updated test')
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
@unittest.skip('skipping non-updated test')
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
@unittest.skip('skipping non-updated test')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
@unittest.skip('skipping non-updated test')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
@unittest.skip('skipping non-updated test')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
@unittest.skip('skipping non-updated test')
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
@unittest.skip('skipping non-updated test')
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
@unittest.skip('skipping non-updated test')
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
@unittest.skip('skipping non-updated test')
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
@unittest.skip('skipping non-updated test')
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
@unittest.skip('skipping non-updated test')
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
@unittest.skip('skipping non-updated test')
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
@unittest.skip('skipping non-updated test')
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)
friends_file = unicodecsv.writer(open(FILE_LOC, "wb"))
#Run basic program
if(ADVANCED==0):
#Write csv column headers
friends_file.writerow(["User Name","Name","ID","Description","Location","URL"])
next_cursor=-1
p = 1
while(next_cursor != 0):
print "Loading page", p , "of results from Twitter"
#Loop/set variables for each user in friends list
friends = twitter.get_friends_list(screen_name = USERNAME, count = 200, cursor = next_cursor)
print "Writing page", p , "to csv file"
for result in friends['users']:
screen_name = result['screen_name']
name = result['name']
tw_id = result['id']
description = result['description']
location = result['location']
entities = result['entities']
expanded_url = None
if('url' in entities):
entities_url = entities['url']
entities_urls = entities_url['urls']
for entities_result in entities_urls:
expanded_url = entities_result['expanded_url']
friends_file = unicodecsv.writer(open(FILE_LOC, "wb"))
#Run basic program
if (ADVANCED == 0):
#Write csv column headers
friends_file.writerow(
["User Name", "Name", "ID", "Description", "Location", "URL"])
next_cursor = -1
p = 1
while (next_cursor != 0):
print "Loading page", p, "of results from Twitter"
#Loop/set variables for each user in friends list
friends = twitter.get_friends_list(screen_name=USERNAME,
count=200,
cursor=next_cursor)
print "Writing page", p, "to csv file"
for result in friends['users']:
screen_name = result['screen_name']
name = result['name']
tw_id = result['id']
description = result['description']
location = result['location']
entities = result['entities']
expanded_url = None
if ('url' in entities):
entities_url = entities['url']
entities_urls = entities_url['urls']
for entities_result in entities_urls:
class TwythonAPITestCase(unittest.TestCase):
def setUp(self):
client_args = {
'headers': {
'User-Agent': '__twython__ Test'
},
'allow_redirects': False
}
oauth2_client_args = {
'headers': {
} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied
}
self.api = Twython(app_key,
app_secret,
oauth_token,
oauth_token_secret,
client_args=client_args)
self.oauth2_api = Twython(app_key,
access_token=access_token,
client_args=oauth2_client_args)
def test_construct_api_url(self):
"""Test constructing a Twitter API url works as we expect"""
url = 'https://api.twitter.com/1.1/search/tweets.json'
constructed_url = self.api.construct_api_url(url, q='#twitter')
self.assertEqual(
constructed_url,
'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter')
def test_get(self):
"""Test Twython generic GET request works"""
self.api.get('account/verify_credentials')
def test_post(self):
"""Test Twython generic POST request works, with a full url and
with just an endpoint"""
update_url = 'https://api.twitter.com/1.1/statuses/update.json'
status = self.api.post(
update_url,
params={'status': 'I love Twython! %s' % int(time.time())})
self.api.post('statuses/destroy/%s' % status['id_str'])
def test_get_lastfunction_header(self):
"""Test getting last specific header of the last API call works"""
self.api.get('statuses/home_timeline')
self.api.get_lastfunction_header('x-rate-limit-remaining')
def test_get_lastfunction_header_not_present(self):
"""Test getting specific header that does not exist from the last call returns None"""
self.api.get('statuses/home_timeline')
header = self.api.get_lastfunction_header('does-not-exist')
self.assertEqual(header, None)
def test_get_lastfunction_header_no_last_api_call(self):
"""Test attempting to get a header when no API call was made raises a TwythonError"""
self.assertRaises(TwythonError, self.api.get_lastfunction_header,
'no-api-call-was-made')
def test_cursor(self):
"""Test looping through the generator results works, at least once that is"""
search = self.api.cursor(self.api.search, q='twitter', count=1)
counter = 0
while counter < 2:
counter += 1
result = next(search)
new_id_str = int(result['id_str'])
if counter == 1:
prev_id_str = new_id_str
time.sleep(
1) # Give time for another tweet to come into search
if counter == 2:
self.assertTrue(new_id_str > prev_id_str)
def test_encode(self):
"""Test encoding UTF-8 works"""
self.api.encode('Twython is awesome!')
def test_html_for_tweet(self):
"""Test HTML for Tweet returns what we want"""
tweet_text = self.api.html_for_tweet(test_tweet_object)
self.assertEqual(test_tweet_html, tweet_text)
def test_html_for_tweet_expanded_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object,
use_expanded_url=True)
# Make sure full url is in HTML
self.assertTrue('http://google.com' in tweet_text)
def test_html_for_tweet_short_url(self):
"""Test using expanded url in HTML for Tweet displays full urls"""
tweet_text = self.api.html_for_tweet(test_tweet_object, False)
# Make sure HTML doesn't contain the display OR exapanded url
self.assertTrue(not 'http://google.com' in tweet_text)
self.assertTrue(not 'google.com' in tweet_text)
def test_raise_error_on_bad_ssl_cert(self):
"""Test TwythonError is raised by a RequestException when an actual HTTP happens"""
self.assertRaises(TwythonError, self.api.get, 'https://example.com')
# Timelines
def test_get_mentions_timeline(self):
"""Test returning mentions timeline for authenticated user succeeds"""
self.api.get_mentions_timeline()
def test_get_user_timeline(self):
"""Test returning timeline for authenticated user and random user
succeeds"""
self.api.get_user_timeline() # Authenticated User Timeline
self.api.get_user_timeline(
screen_name='twitter') # Random User Timeline
def test_get_protected_user_timeline_following(self):
"""Test returning a protected user timeline who you are following
succeeds"""
self.api.get_user_timeline(screen_name=protected_twitter_1)
def test_get_protected_user_timeline_not_following(self):
"""Test returning a protected user timeline who you are not following
fails and raise a TwythonAuthError"""
self.assertRaises(TwythonAuthError,
self.api.get_user_timeline,
screen_name=protected_twitter_2)
def test_retweeted_of_me(self):
"""Test that getting recent tweets by authenticated user that have
been retweeted by others succeeds"""
self.api.retweeted_of_me()
def test_get_home_timeline(self):
"""Test returning home timeline for authenticated user succeeds"""
self.api.get_home_timeline()
# Tweets
def test_get_retweets(self):
"""Test getting retweets of a specific tweet succeeds"""
self.api.get_retweets(id=test_tweet_id)
def test_show_status(self):
"""Test returning a single status details succeeds"""
self.api.show_status(id=test_tweet_id)
def test_update_and_destroy_status(self):
"""Test updating and deleting a status succeeds"""
status = self.api.update_status(
status='Test post just to get deleted :( %s' % int(time.time()))
self.api.destroy_status(id=status['id_str'])
def test_get_oembed_tweet(self):
"""Test getting info to embed tweet on Third Party site succeeds"""
self.api.get_oembed_tweet(id='99530515043983360')
def test_get_retweeters_ids(self):
"""Test getting ids for people who retweeted a tweet succeeds"""
self.api.get_retweeters_ids(id='99530515043983360')
# Search
def test_search(self):
"""Test searching tweets succeeds"""
self.api.search(q='twitter')
# Direct Messages
def test_get_direct_messages(self):
"""Test getting the authenticated users direct messages succeeds"""
self.api.get_direct_messages()
def test_get_sent_messages(self):
"""Test getting the authenticated users direct messages they've
sent succeeds"""
self.api.get_sent_messages()
def test_send_get_and_destroy_direct_message(self):
"""Test sending, getting, then destory a direct message succeeds"""
message = self.api.send_direct_message(screen_name=protected_twitter_1,
text='Hey d00d! %s' %
int(time.time()))
self.api.get_direct_message(id=message['id_str'])
self.api.destroy_direct_message(id=message['id_str'])
def test_send_direct_message_to_non_follower(self):
"""Test sending a direct message to someone who doesn't follow you
fails"""
self.assertRaises(TwythonError,
self.api.send_direct_message,
screen_name=protected_twitter_2,
text='Yo, man! %s' % int(time.time()))
# Friends & Followers
def test_get_user_ids_of_blocked_retweets(self):
"""Test that collection of user_ids that the authenticated user does
not want to receive retweets from succeeds"""
self.api.get_user_ids_of_blocked_retweets(stringify_ids=True)
def test_get_friends_ids(self):
"""Test returning ids of users the authenticated user and then a random
user is following succeeds"""
self.api.get_friends_ids()
self.api.get_friends_ids(screen_name='twitter')
def test_get_followers_ids(self):
"""Test returning ids of users the authenticated user and then a random
user are followed by succeeds"""
self.api.get_followers_ids()
self.api.get_followers_ids(screen_name='twitter')
def test_lookup_friendships(self):
"""Test returning relationships of the authenticating user to the
comma-separated list of up to 100 screen_names or user_ids provided
succeeds"""
self.api.lookup_friendships(screen_name='twitter,ryanmcgrath')
def test_get_incoming_friendship_ids(self):
"""Test returning incoming friendship ids succeeds"""
self.api.get_incoming_friendship_ids()
def test_get_outgoing_friendship_ids(self):
"""Test returning outgoing friendship ids succeeds"""
self.api.get_outgoing_friendship_ids()
def test_create_friendship(self):
"""Test creating a friendship succeeds"""
self.api.create_friendship(screen_name='justinbieber')
def test_destroy_friendship(self):
"""Test destroying a friendship succeeds"""
self.api.destroy_friendship(screen_name='justinbieber')
def test_update_friendship(self):
"""Test updating friendships succeeds"""
self.api.update_friendship(screen_name=protected_twitter_1,
retweets='true')
self.api.update_friendship(screen_name=protected_twitter_1,
retweets=False)
def test_show_friendships(self):
"""Test showing specific friendship succeeds"""
self.api.show_friendship(target_screen_name=protected_twitter_1)
def test_get_friends_list(self):
"""Test getting list of users authenticated user then random user is
following succeeds"""
self.api.get_friends_list()
self.api.get_friends_list(screen_name='twitter')
def test_get_followers_list(self):
"""Test getting list of users authenticated user then random user are
followed by succeeds"""
self.api.get_followers_list()
self.api.get_followers_list(screen_name='twitter')
# Users
def test_get_account_settings(self):
"""Test getting the authenticated user account settings succeeds"""
self.api.get_account_settings()
def test_verify_credentials(self):
"""Test representation of the authenticated user call succeeds"""
self.api.verify_credentials()
def test_update_account_settings(self):
"""Test updating a user account settings succeeds"""
self.api.update_account_settings(lang='en')
def test_update_delivery_service(self):
"""Test updating delivery settings fails because we don't have
a mobile number on the account"""
self.assertRaises(TwythonError,
self.api.update_delivery_service,
device='none')
def test_update_profile(self):
"""Test updating profile succeeds"""
self.api.update_profile(include_entities='true')
def test_update_profile_colors(self):
"""Test updating profile colors succeeds"""
self.api.update_profile_colors(profile_background_color='3D3D3D')
def test_list_blocks(self):
"""Test listing users who are blocked by the authenticated user
succeeds"""
self.api.list_blocks()
def test_list_block_ids(self):
"""Test listing user ids who are blocked by the authenticated user
succeeds"""
self.api.list_block_ids()
def test_create_block(self):
"""Test blocking a user succeeds"""
self.api.create_block(screen_name='justinbieber')
def test_destroy_block(self):
"""Test unblocking a user succeeds"""
self.api.destroy_block(screen_name='justinbieber')
def test_lookup_user(self):
"""Test listing a number of user objects succeeds"""
self.api.lookup_user(screen_name='twitter,justinbieber')
def test_show_user(self):
"""Test showing one user works"""
self.api.show_user(screen_name='twitter')
def test_search_users(self):
"""Test that searching for users succeeds"""
self.api.search_users(q='Twitter API')
def test_get_contributees(self):
"""Test returning list of accounts the specified user can
contribute to succeeds"""
self.api.get_contributees(screen_name='TechCrunch')
def test_get_contributors(self):
"""Test returning list of accounts that contribute to the
authenticated user fails because we are not a Contributor account"""
self.assertRaises(TwythonError,
self.api.get_contributors,
screen_name=screen_name)
def test_remove_profile_banner(self):
"""Test removing profile banner succeeds"""
self.api.remove_profile_banner()
def test_get_profile_banner_sizes(self):
"""Test getting list of profile banner sizes fails because
we have not uploaded a profile banner"""
self.assertRaises(TwythonError, self.api.get_profile_banner_sizes)
# Suggested Users
def test_get_user_suggestions_by_slug(self):
"""Test getting user suggestions by slug succeeds"""
self.api.get_user_suggestions_by_slug(slug='twitter')
def test_get_user_suggestions(self):
"""Test getting user suggestions succeeds"""
self.api.get_user_suggestions()
def test_get_user_suggestions_statuses_by_slug(self):
"""Test getting status of suggested users succeeds"""
self.api.get_user_suggestions_statuses_by_slug(slug='funny')
# Favorites
def test_get_favorites(self):
"""Test getting list of favorites for the authenticated
user succeeds"""
self.api.get_favorites()
def test_create_and_destroy_favorite(self):
"""Test creating and destroying a favorite on a tweet succeeds"""
self.api.create_favorite(id=test_tweet_id)
self.api.destroy_favorite(id=test_tweet_id)
# Lists
def test_show_lists(self):
"""Test show lists for specified user"""
self.api.show_lists(screen_name='twitter')
def test_get_list_statuses(self):
"""Test timeline of tweets authored by members of the
specified list succeeds"""
self.api.get_list_statuses(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_create_update_destroy_list_add_remove_list_members(self):
"""Test create a list, adding and removing members then
deleting the list succeeds"""
the_list = self.api.create_list(name='Stuff %s' % int(time.time()))
list_id = the_list['id_str']
self.api.update_list(list_id=list_id,
name='Stuff Renamed %s' % int(time.time()))
screen_names = ['johncena', 'xbox']
# Multi add/delete members
self.api.create_list_members(list_id=list_id, screen_name=screen_names)
self.api.delete_list_members(list_id=list_id, screen_name=screen_names)
# Single add/delete member
self.api.add_list_member(list_id=list_id, screen_name='justinbieber')
self.api.delete_list_member(list_id=list_id,
screen_name='justinbieber')
self.api.delete_list(list_id=list_id)
def test_get_list_memberships(self):
"""Test list of memberhips the authenticated user succeeds"""
self.api.get_list_memberships()
def test_get_list_subscribers(self):
"""Test list of subscribers of a specific list succeeds"""
self.api.get_list_subscribers(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_subscribe_is_subbed_and_unsubscribe_to_list(self):
"""Test subscribing, is a list sub and unsubbing to list succeeds"""
self.api.subscribe_to_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
# Returns 404 if user is not a subscriber
self.api.is_list_subscriber(
slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name=screen_name)
self.api.unsubscribe_from_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_is_list_member(self):
"""Test returning if specified user is member of a list succeeds"""
# Returns 404 if not list member
self.api.is_list_member(slug=test_list_slug,
owner_screen_name=test_list_owner_screen_name,
screen_name='themattharris')
def test_get_list_members(self):
"""Test listing members of the specified list succeeds"""
self.api.get_list_members(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_get_specific_list(self):
"""Test getting specific list succeeds"""
self.api.get_specific_list(
slug=test_list_slug, owner_screen_name=test_list_owner_screen_name)
def test_get_list_subscriptions(self):
"""Test collection of the lists the specified user is
subscribed to succeeds"""
self.api.get_list_subscriptions(screen_name='twitter')
def test_show_owned_lists(self):
"""Test collection of lists the specified user owns succeeds"""
self.api.show_owned_lists(screen_name='twitter')
# Saved Searches
def test_get_saved_searches(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
self.api.get_saved_searches()
def test_create_get_destroy_saved_search(self):
"""Test getting list of saved searches for authenticated
user succeeds"""
saved_search = self.api.create_saved_search(query='#Twitter')
saved_search_id = saved_search['id_str']
self.api.show_saved_search(id=saved_search_id)
self.api.destroy_saved_search(id=saved_search_id)
# Places & Geo
def test_get_geo_info(self):
"""Test getting info about a geo location succeeds"""
self.api.get_geo_info(place_id='df51dec6f4ee2b2c')
def test_reverse_geo_code(self):
"""Test reversing geocode succeeds"""
self.api.reverse_geocode(lat='37.76893497', long='-122.42284884')
def test_search_geo(self):
"""Test search for places that can be attached
to a statuses/update succeeds"""
self.api.search_geo(query='Toronto')
def test_get_similar_places(self):
"""Test locates places near the given coordinates which
are similar in name succeeds"""
self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ')
# Trends
def test_get_place_trends(self):
"""Test getting the top 10 trending topics for a specific
WOEID succeeds"""
self.api.get_place_trends(id=1)
def test_get_available_trends(self):
"""Test returning locations that Twitter has trending
topic information for succeeds"""
self.api.get_available_trends()
def test_get_closest_trends(self):
"""Test getting the locations that Twitter has trending topic
information for, closest to a specified location succeeds"""
self.api.get_closest_trends(lat='37', long='-122')
# Help
def test_get_twitter_configuration(self):
"""Test getting Twitter's configuration succeeds"""
self.api.get_twitter_configuration()
def test_get_supported_languages(self):
"""Test getting languages supported by Twitter succeeds"""
self.api.get_supported_languages()
def test_privacy_policy(self):
"""Test getting Twitter's Privacy Policy succeeds"""
self.api.get_privacy_policy()
def test_get_tos(self):
"""Test getting the Twitter Terms of Service succeeds"""
self.api.get_tos()
def test_get_application_rate_limit_status(self):
"""Test getting application rate limit status succeeds"""
self.oauth2_api.get_application_rate_limit_status()
def add_alert(request):
if not request.user.is_authenticated():
return HttpResponseRedirect('/')
context = {}
context.update(csrf(request))
user = request.user
profile = TwitterProfile.objects.all().filter(user=user).get()
alerts_count = len(GrainVacuum.objects.all().filter(profile=profile))
screen_name_list = None
name_list = None
if request.method == "POST":
follow_user = request.POST['follow_user']
key_words = request.POST['key_words_list']
follow_user = follow_user.replace('@','')
new_grain = GrainVacuum(profile=profile,user=user,follow_user=follow_user,key_words=key_words)
new_grain.save()
return HttpResponseRedirect('/dashboard/')
else:
# Update FriendList
try:
# Determine if friend_list exists
try:
friend_list = FriendList.objects.all().filter(twitter_profile=profile).get()
except:
# if it doesn't exist create a blank one
new_friend_list = FriendList(twitter_profile=profile)
new_friend_list.save()
friend_list = FriendList.objects.all().filter(twitter_profile=profile).get()
est = pytz.timezone('US/Eastern') # Eastern Time Zone
last_checked = friend_list.last_checked
now = datetime.datetime.now()
tdelta = 0
if last_checked != None:
last_checked = last_checked.astimezone(est).replace(tzinfo=None)
tdelta = int((now - last_checked).total_seconds() / 60)
print last_checked
if tdelta > 15 or last_checked == None:
APP_KEY = Config.objects.all().latest('id').twitter_key
APP_SECRET = Config.objects.all().latest('id').twitter_secret
# Get Authorization Credentials
OAUTH_TOKEN = profile.oauth_token
OAUTH_TOKEN_SECRET = profile.oauth_secret
friend_list_update = FriendList.objects.all().filter(twitter_profile=profile)
twitter = Twython(APP_KEY, APP_SECRET,OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
# Query Twitter Friends List API until none
screen_name_list = ""
name_list = ""
next_cursor=-1
count = 0
while next_cursor:
follow_list = twitter.get_friends_list(count=200,cursor=next_cursor)
y = 0
for user in follow_list['users']:
screen_name = user['screen_name']
name = user['name']
if y == 0:
screen_name_list += screen_name
name_list += name
else:
screen_name_list += "," + screen_name
name_list += "," + name
y += 1
print screen_name + ' ' + name
count += 1
if count == 15 or next_cursor == 0:
break
next_cursor = follow_list["next_cursor"]
friend_list_update.update(last_checked=now,name=name_list,handle=screen_name_list)
else:
screen_name_list = friend_list.handle
name_list = friend_list.name
except Exception as error:
print error
return render_to_response('add_alert.html', {'screen_name_list':screen_name_list,'name_list':name_list,'alerts_count':alerts_count}, context_instance=RequestContext(request))
class twitterbot:
__twitter = None
__db = None
def __init__(self, file_path_bot=None, file_path_db=None):
if file_path_bot is None:
file_path_bot = 'twitter_aut.json'
access = {'apiKey': '', 'apiSecret': '', 'accessToken': '', 'accessTokenSecret': ''}
if os.path.isfile(file_path_bot):
with open(file_path_bot, 'r') as f:
access = json.load(f)
else:
with open(file_path_bot, 'w') as outfile:
json.dump(access, outfile)
self.__twitter = Twython(access['apiKey'], access['apiSecret'], access['accessToken'],
access['accessTokenSecret'])
self.__db = BotDb(file_path_db=file_path_db)
def get_limits(self):
limits = self.__twitter.get_application_rate_limit_status()
return limits
def tweet(self, msg):
self.__twitter.update_status(status=msg)
def __check_tags(self, tweet_tags, tags):
for tweet_tag in tweet_tags:
if tweet_tag in tags:
return True
return False
def __check_tweets(self):
try:
tweets = self.__db.get_open_tweets()
for tweet in tweets:
self.tweet(tweet.message)
self.__db.set_tweet_to_sended(tweet.id)
except twython.exceptions.TwythonError:
self.__db.set_tweet_to_sended(tweet.id)
except:
info = sys.exc_info()
print(info)
def loop(self):
while True:
self.__check_tweets()
tags = self.__db.get_tags()
followers = self.__twitter.get_friends_list()['users']
for follower in followers:
screenName = follower["screen_name"]
tweets = self.__twitter.get_user_timeline(screen_name=screenName, tweet_mode='extended')
for status in tweets:
id = status['id']
tweet_tags = [tag['text'].lower() for tag in status['entities']['hashtags']]
if self.__check_tags(tweet_tags, tags):
if self.__db.add_try_retweet(int(id)):
try:
self.__twitter.retweet(id=id)
except:
pass
break
time.sleep(120)
def run(self):
_thread.start_new(self.loop, ())
