def scrape(arg): results = {} twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2) ACCESS_TOKEN = twitter.obtain_access_token() twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) friends = twitter.get_friends_ids(screen_name=arg) #hard cap for time limit = 10 for follower_id in friends['ids']: if limit > 0: params = {'user_id': follower_id, 'count': 3} tweets = twitter.get_user_timeline(**params) for tweet in tweets: encoded = tweet['text'].encode("utf8") results = dict(Counter(results) + Counter(text_tags(encoded))) #random.choice(news) #f.write(str(text_tags(encoded))) limit = limit - 1 else: break #print results key = max(results.iteritems(), key=operator.itemgetter(1))[0] n = random.choice(news) return [key, n]
def get_queryset(self): provider = self.request.QUERY_PARAMS.get('provider', None) if provider == 'facebook': # TODO: what does it look like when a user has more than one social auth for a provider? Is this a thing # that can happen? How does it affect SocialShareMixin? The first one is the oldest--do we actually want # the last one? user_social_auth = self.request.user.social_auth.filter(provider='facebook').first() graph = facebook.GraphAPI(user_social_auth.extra_data['access_token']) facebook_friends = graph.request("v2.2/me/friends")["data"] friends = User.objects.filter(social_auth__provider='facebook', social_auth__uid__in=[user["id"] for user in facebook_friends]) return friends elif provider == 'twitter': user_social_auth = self.request.user.social_auth.filter(provider='twitter').first() twitter = Twython( app_key=settings.SOCIAL_AUTH_TWITTER_KEY, app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET, oauth_token=user_social_auth.tokens['oauth_token'], oauth_token_secret=user_social_auth.tokens['oauth_token_secret'] ) twitter_friends = twitter.get_friends_ids()["ids"] friends = User.objects.filter(social_auth__provider='twitter', social_auth__uid__in=twitter_friends) return friends else: return Response({"errors": "{} is not a valid social provider".format(provider)}, status=status.HTTP_400_BAD_REQUEST)
def scrape(arg): results = {} twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2) ACCESS_TOKEN = twitter.obtain_access_token() twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) friends = twitter.get_friends_ids(screen_name = arg) #hard cap for time limit = 10 for follower_id in friends['ids']: if limit > 0 : params = {'user_id':follower_id, 'count':3 } tweets = twitter.get_user_timeline(**params) for tweet in tweets: encoded = tweet['text'].encode("utf8") results = dict(Counter(results) + Counter(text_tags(encoded))) #random.choice(news) #f.write(str(text_tags(encoded))) limit = limit - 1 else: break #print results key = max(results.iteritems(), key=operator.itemgetter(1))[0] n = random.choice(news) return [key, n]
def get_queryset(self): provider = self.request.QUERY_PARAMS.get('provider', None) if provider == 'facebook': # TODO: what does it look like when a user has more than one social auth for a provider? Is this a thing # that can happen? How does it affect SocialShareMixin? The first one is the oldest--do we actually want # the last one? user_social_auth = self.request.user.social_auth.filter( provider='facebook').first() graph = facebook.GraphAPI( user_social_auth.extra_data['access_token']) facebook_friends = graph.request("v2.2/me/friends")["data"] friends = User.objects.filter( social_auth__provider='facebook', social_auth__uid__in=[user["id"] for user in facebook_friends]) return friends elif provider == 'twitter': user_social_auth = self.request.user.social_auth.filter( provider='twitter').first() twitter = Twython( app_key=settings.SOCIAL_AUTH_TWITTER_KEY, app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET, oauth_token=user_social_auth.tokens['oauth_token'], oauth_token_secret=user_social_auth. tokens['oauth_token_secret']) twitter_friends = twitter.get_friends_ids()["ids"] friends = User.objects.filter(social_auth__provider='twitter', social_auth__uid__in=twitter_friends) return friends else: return Response( { "errors": "{} is not a valid social provider".format(provider) }, status=status.HTTP_400_BAD_REQUEST)
def get_followers_following(usr_id): index = 0 print "GETTING FOLLOWERS AND FOLLOWING: " + str(usr_id) friend_cursor = -1 follower_cursor = -1 APP_KEY = APP_KEYS[index] APP_SECRET = APP_SECRETS[index] OAUTH_TOKEN = OAUTH_TOKENS[index] OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index] while (1==1): try: twitter = Twython (APP_KEY, APP_SECRET) auth = twitter.get_authentication_tokens() twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) friends = "" while (friend_cursor != 0): following = twitter.get_friends_ids(id = usr_id, cursor = friend_cursor) for ID in following['ids']: friends += str(ID) + " " friend_cursor = following["next_cursor"] friends = friends[:-1] if len(friends) == 0: friends = "null" follow = "" while (follower_cursor != 0): followers = twitter.get_followers_ids(id = usr_id,cursor= follower_cursor) for ID in followers['ids']: follow += str(ID) + " " follower_cursor = followers["next_cursor"] follow= follow[:-1] if len(follow) == 0: follow = "null" return (friends,follow) except Exception as e: print e if "429 (Too Many Requests)" in str(e): #global index index += 1 if index == len(APP_KEYS): index = 0 print "sleepy time - 15 minutes" print datetime.datetime.now() time.sleep(910) return get_followers_following(usr_id) elif "401 (Unauthorized)" in str(e): print "401 error" return ("null","null") elif "404 (Not Found)" in str(e): print "404 error" return ("null","null") else: print e return ("null","null")
def get_friends(user_social_auth): twitter = Twython( app_key=settings.SOCIAL_AUTH_TWITTER_KEY, app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET, oauth_token=user_social_auth.tokens['oauth_token'], oauth_token_secret=user_social_auth.tokens['oauth_token_secret']) twitter_friends = twitter.get_friends_ids()["ids"] friends = User.objects.filter(social_auth__provider='twitter', social_auth__uid__in=twitter_friends) return friends
def get_friends(user_social_auth): twitter = Twython( app_key=settings.SOCIAL_AUTH_TWITTER_KEY, app_secret=settings.SOCIAL_AUTH_TWITTER_SECRET, oauth_token=user_social_auth.tokens['oauth_token'], oauth_token_secret=user_social_auth.tokens['oauth_token_secret'] ) twitter_friends = twitter.get_friends_ids()["ids"] friends = User.objects.filter(social_auth__provider='twitter', social_auth__uid__in=twitter_friends) return friends
def main(): APP_KEY = 'ORbbZFlJWipi3HFblfPuQ' APP_SECRET = 's2em5f68RBV8UDgW7iXOH5PMFArF5aVj8GG6IgsU' twitter = Twython(APP_KEY, APP_SECRET) auth = twitter.get_authentication_tokens() OAUTH_TOKEN = '165916420-31NTOhsitabrC6tgo0Zd87q0f60mGUdjBZan56HQ' OAUTH_TOKEN_SECRET = '8gpWg3g207b4QMuabv2xkw2VkkMyBPnDEImTquPgZC0vV' main_user = 100 # Requires Authentication as of Twitter API v1.1 twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) try: user_timeline = twitter.get_friends_ids(screen_name='alex_pardo5', count=5000) except TwythonError as e: print e i = 0 print user_timeline #G=nx.Graph() #G.add_node(main_user) #print user_timeline #server = xmlrpclib.Server('http://localhost:20738/RPC2') #G = server.ubigraph G = ubigraph.Ubigraph() G.clear() main_user = G.newVertex(shape="sphere", color="#ffff00", label='alex_pardo5') red = G.newVertexStyle(shape="sphere", color="#ff0000", size="1") for user in user_timeline['ids']: #print twitter.show_user(user_id=user) #tmp = twitter.show_user(user_id=user)['screen_name'] tmp = user tmp = G.newVertex(style=red, label=str(tmp)) G.newEdge(main_user,tmp,arrow=True, width=random.randint(1, 5)) i += 1 print i
def fetch_friend_ids(self, user): """ fetches friend id's from twitter Return: collection of friend ids """ auth_data = self._auth_data(user) tw = Twython(**auth_data) cursor = -1 friend_ids = [] while True: data = tw.get_friends_ids(cursor=cursor) friend_ids += data.get('ids', []) next_cursor = data.get('next_cursor', 0) prev_cursor = data.get('prev_cursor', 0) if not next_cursor or next_cursor == prev_cursor: break else: cursor = next_cursor return friend_ids
cursor.execute("select id from t_gh_json") ids = cursor.fetchall() cursor.execute("create table ghtwdirectextra(user_id text, followerids text, friendids text)") cursor.execute("create table ghtwdirecttimeline(user_id text, tweets text, tweetids text)") for i in ids: k="" for j in i[0]: if j != "'": k = k + j print k try: followers = t.get_followers_ids(user_id = k, count=5000) #print followers['ids'] friends = t.get_friends_ids(user_id = k, count=5000) #print friends['ids'] timeline = t.get_user_timeline(user_id = k, count=200, include_rts=1) for i in timeline: #print i['text'] #print i['id'] cursor.execute("insert into ghtwdirecttimeline(user_id, tweets, tweetids) values(%s,%s,%s)",(k, i['text'], i['id'])) cursor.execute("insert into ghtwdirectextra(user_id, followerids, friendids) values(%s,%s,%s)",(k, followers['ids'], friends['ids'])) print "inserted" except TwythonRateLimitError as e: print "rate limit" time.sleep(60*15) continue except TwythonAuthError as e: continue except TwythonError as e:
class TwitterHelper(object): def __init__(self, identity): self.identity = identity if not self.identity.tokens: self.identity.tokens = authorisationhelper.get_tokens(identity.screen_name) self.twitter = Twython( self.identity.tokens[0], self.identity.tokens[1], self.identity.tokens[2], self.identity.tokens[3] ) # self.identity.twid = self.twitter.lookup_user(screen_name=identity.screen_name)[0]["id_str"] self.mutation = [" ,", " .", " *", " `", " -", " _"] self.twitter_configuration = self.twitter.get_twitter_configuration() logger.debug(self.twitter_configuration) giphyhelper.set_photo_size_limit(self.twitter_configuration["photo_size_limit"]) me = self.twitter.lookup_user(screen_name=self.identity.screen_name)[0] logger.debug(me) self.identity.update(me) self.rates = None self.update_rate_limits() @retry(**retry_args) def send(self, outbox_item): if type(outbox_item) is OutgoingTweet: outbox_item.display() if outbox_item.filePaths and any(outbox_item.filePaths): for filePath in outbox_item.filePaths: # ext = os.path.splitext(filePath) # if ext == "mp4": # media_id = self._upload_video(filePath) # else: media_id = self._upload_media(filePath) if media_id: outbox_item.media_ids.append(media_id) split_tweets = tweet_splitter.split_tweet(outbox_item, self.twitter_configuration) in_reply_to_id_str = outbox_item.in_reply_to_id_str for split_tweet in split_tweets: split_tweet.in_reply_to_id_str = in_reply_to_id_str response = self.twitter.update_status(**split_tweet.get_tweet_params()) split_tweet.id_str = response["id_str"] self.identity.conversations.outgoing(split_tweet) self.identity.statistics.record_outgoing_tweet() in_reply_to_id_str = split_tweet.id_str return in_reply_to_id_str if type(outbox_item) is OutgoingDirectMessage: if not outbox_item.screen_name and not outbox_item.user_id: outbox_item.screen_name = self.identity.admin_screen_name outbox_item.display() self.twitter.send_direct_message( text=outbox_item.text, screen_name=outbox_item.screen_name, user_id=outbox_item.user_id) self.identity.statistics.record_outgoing_direct_message() return None def quote_tweet(self, inbox_item, text=None, file_paths=None): reply_as_quote_tweet = inbox_item.is_tweet reply_as_dm = inbox_item.is_direct_message if reply_as_quote_tweet: logger.info("quoting to %s as quote tweet", inbox_item.text) tweet = OutgoingTweet( quote=inbox_item, text=text, file_paths=file_paths, ) return self.send(tweet) if reply_as_dm: logger.info("replying to %s as DM", inbox_item.text) dm = OutgoingDirectMessage( reply_to=inbox_item, text=text) return self.send(dm) return None def reply_with(self, inbox_item, text=None, as_tweet=False, as_direct_message=False, file_paths=None, in_reply_to_id_str=None): reply_as_tweet = as_tweet or not as_direct_message and inbox_item.is_tweet reply_as_dm = as_direct_message or not as_tweet and inbox_item.is_direct_message if reply_as_tweet: logger.info("replying to %s as tweet", inbox_item.text) tweet = OutgoingTweet( reply_to=inbox_item, text=text, file_paths=file_paths, in_reply_to_id_str=in_reply_to_id_str) return self.send(tweet) if reply_as_dm: logger.info("replying to %s as DM", inbox_item.text) dm = OutgoingDirectMessage( reply_to=inbox_item, text=text) return self.send(dm) return None def _upload_media(self, file_path): file = None try: file = open(file_path, 'rb') media = self.twitter.upload_media(media=file) self.identity.statistics.increment("Media Uploads") return media["media_id_string"] finally: if file: file.close() def get_streamer(self, topic=None, topic_name=None, responses=None, filter_level=None): return Streamer(self.identity, topic, topic_name, responses, filter_level) def _upload_video(self, file_path): logger.info('[MyTwitter] uploading ' + file_path) url = 'https://upload.twitter.com/1.1/media/upload.json' file_size = os.path.getsize(file_path) logger.info('[MyTwitter] Init') init_params = { "command": "INIT", "media_type": "video/mp4", "total_bytes": file_size } init_response = self.twitter.post(url, init_params) logger.info(init_response) media_id = init_response["media_id_string"] logger.info('[MyTwitter] media_id ' + media_id) segment = 0 chunk_size = 4 * 1024 * 1024 for chunk in fsh.bytes_from_file(file_path, chunk_size): logger.info('[MyTwitter] Append ' + str(segment)) append_params = { 'command': 'APPEND', 'media_id': media_id, 'segment_index': segment } append_response = self.twitter.post(url, append_params, {'media': chunk}) logger.info(append_response) segment += 1 logger.info('[MyTwitter] Finalize') finalize_params = { "command": "FINALIZE", "media_id": media_id, } self.twitter.post(url, finalize_params) return media_id def get_trending_topics_for(self, woeids): trending_topics = [] for woeid in woeids: trends = self.twitter.get_place_trends(id=woeid)[0].get('trends', []) for trend in trends: trending_topics.append(trend['name']) return trending_topics def get_saved_searches(self): saved_searches = [] searches = self.twitter.get_saved_searches() for srch in searches: saved_searches.append(srch['name']) return saved_searches def delete_saved_searches(self): searches = self.twitter.get_saved_searches() for srch in searches: self.twitter.destroy_saved_search(id=srch["id_str"]) def search(self, text, result_type="popular"): query = quote_plus(text) return self.twitter.search(q=query, result_type=result_type)["statuses"] @retry(**retry_args) def favourite(self, id_str): try: self.twitter.create_favorite(id=id_str) self.identity.statistics.record_favourite() except TwythonError as ex: if "You have already favourited this tweet" in str(ex): logger.warning(ex) else: raise @retry(**retry_args) def retweet(self, id_str): try: self.twitter.retweet(id=id_str) self.identity.statistics.record_retweet() except TwythonError as ex: if "You have already retweeted this tweet" in str(ex): logger.warning(ex) else: raise @retry(**retry_args) def add_user_to_list(self, list_id, user_id, screen_name): self.twitter.create_list_members(list_id=list_id, user_id=user_id, screen_name=screen_name) @retry(**retry_args) def block_user(self, user_id, user_screen_name=None): self.twitter.create_block(user_id=user_id, screen_name=user_screen_name) @retry(**retry_args) def get_user_timeline(self, **kwargs): return self._rate_limit("/statuses/user_timeline", self.twitter.get_user_timeline, **kwargs) def unblock_user(self, user): self.twitter.destroy_block(user_id=user.id, screen_name=user.screen_name) @retry(**retry_args) def unblock_users(self): user_ids = self.twitter.list_block_ids(stringify_ids=True) for user_id in user_ids["ids"]: self.twitter.destroy_block(user_id=user_id) @retry(**retry_args) def show_owned_lists(self): return self.twitter.show_owned_lists()["lists"] @retry(**retry_args) def get_list_members(self, list_id): return self.twitter.get_list_members(list_id=list_id, count=5000, include_entities=False) @blocked @retry(**retry_args) def create_list(self, **kwargs): return self.twitter.create_list(**kwargs) @retry(**retry_args) def follow(self, user_id, screen_name): logger.info("following user id {} @{}".format(user_id, screen_name)) self.twitter.create_friendship(user_id=user_id, screen_name=screen_name) self.identity.statistics.increment("Follows") @retry(**retry_args) def unfollow(self, user_id, screen_name): logger.info("unfollowing user id {} @{}".format(user_id, screen_name)) self.twitter.destroy_friendship(user_id=user_id, screen_name=screen_name) self.identity.statistics.increment("Unfollows") @retry(**retry_args) def block(self, user_id, screen_name): logger.info("blocking user id {} @{}".format(user_id, screen_name)) self.twitter.create_block(user_id=user_id, screen_name=screen_name) self.identity.statistics.increment("Blocks") @retry(**retry_args) def report(self, user_id, screen_name): logger.info("reporting user id {} @{}".format(user_id, screen_name)) self.twitter.report_spam(user_id=user_id, screen_name=screen_name) self.identity.statistics.increment("Reports") @retry(**retry_args) def lookup_user(self, user_id): return self._rate_limit("/users/lookup", self.twitter.lookup_user, user_id=user_id) def sing_song(self, song, target=None, inbox_item=None, text=None, hashtag=None): if not text: text = random.choice(["All together now!", "Sing along!"]) text += ' ' + song["video"] if hashtag: text += ' ' + hashtag in_reply_to_id_str = self._send_to( inbox_item=inbox_item, text=text, target=target, in_reply_to_id_str=None) time.sleep(5) lastlyrics = set([]) for lyric in song["lyrics"]: lyric = lyric.strip() if lyric: if "<<screen_name>>" in lyric: lyric = lyric.replace("<<screen_name>>", "@" + target) if hashtag: lyric += " " + hashtag while lyric in lastlyrics: lyric += random.choice(self.mutation) lastlyrics.add(lyric) self.identity.statistics.record_outgoing_song_lyric() in_reply_to_id_str = self._send_to( inbox_item, lyric, target, in_reply_to_id_str) time.sleep(5) def _send_to(self, inbox_item, text, target, in_reply_to_id_str): if inbox_item: return self.reply_with( inbox_item=inbox_item, text=text, in_reply_to_id_str=in_reply_to_id_str) else: status = "" if target: # noinspection PyUnresolvedReferences if isinstance(target, basestring): status = ".@" + target elif isinstance(target, User.User): status = ".@" + target.screen_name status += " " + text tweet = OutgoingTweet( text=status, in_reply_to_id_str=in_reply_to_id_str) return self.send(tweet) @retry(**retry_args) def get_list_subscriptions(self): return self.twitter.get_list_subscriptions() @retry(**retry_args) def subscribe_to_list(self, list_id): return self.twitter.subscribe_to_list(list_id=list_id) @retry(**retry_args) def geocode(self, location): result = self.twitter.search_geo( query=location.full_name, max_results=5, lat=location.latitude, long=location.longitude) logger.info(result) if result["result"]["places"]: # for place in result["result"]["places"]: # logger.info(place["full_name"]) place = result["result"]["places"][0] location.place_id_twitter = place["id"] return location else: return None @retry(**retry_args) def reverse_geocode(self, location): result = self.twitter.reverse_geocode( max_results=5, lat=location.latitude, long=location.longitude) logger.info(result) if result["result"]["places"]: # for place in result["result"]["places"]: # logger.info(place["full_name"]) place = result["result"]["places"][0] location.place_id_twitter = place["id"] return location else: return None @retry(**retry_args) def update_profile_image(self, file_path): if file_path: logger.info("updating profile image %s" % file_path) with open(file_path, 'rb') as file: self.twitter.update_profile_image(image=file) @retry(**retry_args) def update_profile_banner_image(self, file_path): if file_path: logger.info("updating banner image %s" % file_path) with open(file_path, 'rb') as file: try: self.twitter.update_profile_banner_image(banner=file) except TwythonError as ex: if "Response was not valid JSON" in str(ex): # twython issue i think logger.warning(ex) else: raise @retry(**retry_args) def update_profile(self, **kwargs): return self.twitter.update_profile(**kwargs) @retry(**retry_args) def get_list_statuses(self, **kwargs): return self.twitter.get_list_statuses(**kwargs) @retry(**retry_args) def get_user_suggestions_by_slug(self, **kwargs): return self.twitter.get_user_suggestions_by_slug(**kwargs) @retry(**retry_args) def get_user_suggestions(self, **kwargs): return self.twitter.get_user_suggestions(**kwargs) @retry(**retry_args) def lookup_status(self, **kwargs): return self.twitter.lookup_status(**kwargs) @retry(**retry_args) def get_followers(self, **kwargs): kwargs["stringify_ids"] = True followers = set() cursor = -1 while cursor != "0": kwargs["cursor"] = cursor logger.info("getting followers") response = self.twitter.get_followers_ids(**kwargs) cursor = response["next_cursor_str"] followers = followers.union(set(response["ids"])) return followers @retry(**retry_args) def get_following(self, **kwargs): kwargs["stringify_ids"] = True following = set() cursor = -1 while cursor != "0": kwargs["cursor"] = cursor logger.info("getting following") response = self.twitter.get_friends_ids(**kwargs) cursor = response["next_cursor_str"] following = following.union(set(response["ids"])) return following @retry(**retry_args) def update_rate_limits(self): data = self.twitter.get_application_rate_limit_status() self.rates = RateLimits(data) logger.info("Updated rate limits for {}: {}".format(self.identity.screen_name, self.rates.display())) def _rate_limit(self, limit_name, func, *args, **kwargs): if self.rates.can(limit_name): try: return func(*args, **kwargs) except Exception as ex: logger.warning(ex) return None else: logger.warning("{} limit exceeded".format(limit_name)) return None @retry(**retry_args) def get_statuses(self, id_strs): id_strs_csv = ",".join(id_strs) return self.twitter.lookup_status(id=id_strs_csv) @retry(**retry_args) def get_status(self, id_str): return self.twitter.show_status(id=id_str)
def get_friends_ids(oauth_token, oauth_token_secret, user_id, count=500, cursor=-1): twitter = Twython(consumer_key, consumer_secret, oauth_token, oauth_token_secret) result = twitter.get_friends_ids(user_id=user_id, count=count, cursor=cursor) return result
class TwythonAPITestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_get(self): """Test Twython generic GET request works""" self.api.get('account/verify_credentials') def test_post(self): """Test Twython generic POST request works, with a full url and with just an endpoint""" update_url = 'https://api.twitter.com/1.1/statuses/update.json' status = self.api.post(update_url, params={'status': 'I love Twython!'}) self.api.post('statuses/destroy/%s' % status['id_str']) def test_get_lastfunction_header(self): """Test getting last specific header of the last API call works""" self.api.get('statuses/home_timeline') self.api.get_lastfunction_header('x-rate-limit-remaining') def test_get_lastfunction_header_not_present(self): """Test getting specific header that does not exist from the last call returns None""" self.api.get('statuses/home_timeline') header = self.api.get_lastfunction_header('does-not-exist') self.assertEqual(header, None) def test_get_lastfunction_header_no_last_api_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') def test_search_gen(self): """Test looping through the generator results works, at least once that is""" search = self.api.search_gen('twitter', count=1) counter = 0 while counter < 2: counter += 1 result = next(search) new_id_str = int(result['id_str']) if counter == 1: prev_id_str = new_id_str time.sleep(1) # Give time for another tweet to come into search if counter == 2: self.assertTrue(new_id_str > prev_id_str) def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR exapanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text) def test_raise_error_on_bad_ssl_cert(self): """Test TwythonError is raised by a RequestException when an actual HTTP happens""" self.assertRaises(TwythonError, self.api.get, 'https://example.com') # Timelines def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get deleted :(') self.api.destroy_status(id=status['id_str']) def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man!') # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(list_id=test_list_id) def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff') list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed') screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(list_id=test_list_id) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(list_id=test_list_id) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(list_id=test_list_id, screen_name=screen_name) self.api.unsubscribe_from_list(list_id=test_list_id) def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(list_id=test_list_id, screen_name='jack') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(list_id=test_list_id) def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(list_id=test_list_id) def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
def get_following_to_db(dbpath, sn_table = "tweets9_users"): distinct = get_D_UsrID(dbpath,sn_table ) index=0 num_checked = 0 rate_ex = 0 ind = 0 already_checked = False follower_cursor = -1 friend_cursor = -1 print "Starting @ %d"%(ind) print datetime.datetime.now() while ind < len(distinct): #for i in distinct: i = distinct[ind][0] APP_KEY = APP_KEYS[index] APP_SECRET = APP_SECRETS[index] OAUTH_TOKEN = OAUTH_TOKENS[index] OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index] try: #twitter = Twython (APP_KEY, APP_SECRET) #auth = twitter.get_authentication_tokens() twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) #friend_results = twitter.cursor(twitter.get_friends_ids, id = i, cursor= friend_cursor) friends = "" while (friend_cursor): following = twitter.get_friends_ids(id = i,cursor= friend_cursor) for ID in following['ids']: friends += str(ID) + " " friend_cursor = following["next_cursor"] num_checked += 1 friends = friends[:-1] if len(friends) == 0: friends = "null" follow = "" while (follower_cursor): followers = twitter.get_followers_ids(id = i,cursor= follower_cursor) for ID in followers['ids']: follow += str(ID) + " " follower_cursor = followers["next_cursor"] num_checked += 1 follow= follow[:-1] if len(follow) == 0: follow = "null" conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = '%s' where Usr_ID = '%s'"% (sn_table, friends, i) cursor.execute(query) #print query print "____________________ %dth following updated ________ %d" % (ind, i) conn.commit() query = "UPDATE %s SET Followers = '%s' where Usr_ID = '%s' "% (sn_table, follow, i) cursor.execute(query) #print query conn.commit() conn.close() print "%dth follower updated_______________________ %d" % (ind, i) print num_checked print follower_cursor = -1 friend_cursor = -1 ind+=1 except Exception as e: print e if "429 (Too Many Requests)" in str(e): index += 1 if index == len(APP_KEYS): index = 0 print "sleepy time - 15 minutes" print datetime.datetime.now() time.sleep(910) elif "401 (Unauthorized)" in str(e): print "401 error" f = open("skipped.txt", "a") f.write("skipped %dth element, ID: %d\n"%(ind, i)) f.write("__________________________________________"+str(datetime.datetime.now()) + "\n") f.close() print "401 nulled %d"%ind conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i) cursor.execute(query) conn.commit() conn.close() ind+=1 time.sleep(1) elif "404 (Not Found)" in str(e): print "404 error" f = open("skipped.txt", "a") f.write("404: skipped %dth element, ID: %d \n"%(ind, i)) f.write("__________________________________________"+str(datetime.datetime.now()) + "\n") f.close() print "404 nulled %d"%ind conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i) cursor.execute(query) conn.commit() conn.close() ind+=1 time.sleep(1) else: print e rate_ex += 1 print ind print num_checked print rate_ex '''
# Get your keys from https://dev.twitter.com/ # Change the api key values marked ##### below with the twitter keys # Schedule twitterFF.py using crontab or another task scheduler to run every Friday. import time import random from twython import Twython CONSUMER_KEY = '####' CONSUMER_SECRET = '####' ACCESS_KEY = '####' ACCESS_SECRET = '####' api = Twython(CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET) friends = api.get_friends_ids(screen_name="yourtwitterhandle") new_list = [] for x in friends["ids"]: new_list.append(x) #print random id 6 times ff = '' for y in range(1,7): id_position = random.randint(0, len(new_list) -1) userid = new_list[id_position] data = api.show_user(user_id=userid) following = data["screen_name"] ff = ff + ' ' + '@' + following api.update_status(status='#FF ' + ff)
API_SECRET = 'INSERT HERE' ACCESS_TOKEN = 'INSERT HERE' ACCESS_TOKEN_SECRET = 'INSERT HERE' twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) return(twitter) ''' Determining your Twitter followers and friends ''' twitter = twitter_oauth_login() friends_ids = twitter.get_friends_ids(count=5000) friends_ids = friends_ids['ids'] followers_ids = twitter.get_followers_ids(count=5000) followers_ids = followers_ids['ids'] print len(friends_ids), len(followers_ids) friends_set = set(friends_ids) followers_set = set(followers_ids) print('Number of Twitter users who either are our friend or follow you (union):') print(len(friends_set.union(followers_set))) len(friends_set | followers_set)
def updateFollowerFollowings(dbpath, user_table = "users"): distinct = get_D_UsrID(dbpath,sn_table ) conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "select Usr_ID from %s where Following is null or Followers is null" cursor.execute(query) distinct = cursor.fetchall() follower_cursor = -1 friend_cursor = -1 print "Starting @ %d"%(ind) print datetime.datetime.now() while ind < len(distinct): #for i in distinct: i = distinct[ind][0] # Usr_ID APP_KEY = APP_KEYS[index] APP_SECRET = APP_SECRETS[index] OAUTH_TOKEN = OAUTH_TOKENS[index] OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index] try: #twitter = Twython (APP_KEY, APP_SECRET) #auth = twitter.get_authentication_tokens() twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) friends = "" while (friend_cursor): following = twitter.get_friends_ids(id = i,cursor= friend_cursor) for ID in following['ids']: friends += str(ID) + " " friend_cursor = following["next_cursor"] num_checked += 1 friends = friends[:-1] if len(friends) == 0: friends = "null" follow = "" while (follower_cursor): followers = twitter.get_followers_ids(id = i,cursor= follower_cursor) for ID in followers['ids']: follow += str(ID) + " " follower_cursor = followers["next_cursor"] num_checked += 1 follow= follow[:-1] if len(follow) == 0: follow = "null" conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = '%s', Followers = '%s', NumFollowing = '%d', NumFollowers = '%d' where Usr_ID = '%s'"% (sn_table, friends,follow, len(friends.split(" ")), len(follow.split(" ")), i) cursor.execute(query) #print query conn.close() print "____________________ %dth following and friends and their corresponding counts updated ________ %d" % (ind, i) follower_cursor = -1 friend_cursor = -1 ind+=1 except Exception as e: print e if "429 (Too Many Requests)" in str(e): index += 1 if index == len(APP_KEYS): index = 0 print "sleepy time; pausing for 15 minutes for APP Keys to renew" print datetime.datetime.now() #time.sleep(910) print "App Rate Limits has been exceeded- we will wait for next runtime before running rest of followers and followings" return elif "401 (Unauthorized)" in str(e): print "401 Unauthorized Error nulling %d________________Usr_ID:%d"%(ind,i) conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i) cursor.execute(query) conn.commit() conn.close() ind+=1 elif "404 (Not Found)" in str(e): print "404 Not Found Error nulling %d________________Usr_ID:%d"%(ind,i) conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'"% (sn_table, i) cursor.execute(query) conn.commit() conn.close() ind+=1 else: print e
class TwythonEndpointsTestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } # This is so we can hit coverage that Twython sets # User-Agent for us if none is supplied oauth2_client_args = { 'headers': {} } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) # Timelines @unittest.skip('skipping non-updated test') def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() @unittest.skip('skipping non-updated test') def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) @unittest.skip('skipping non-updated test') def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() @unittest.skip('skipping non-updated test') def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets @unittest.skip('skipping non-updated test') def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get \ deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) @unittest.skip('skipping non-updated test') def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') @unittest.skip('skipping non-updated test') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search @unittest.skip('skipping non-updated test') def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages @unittest.skip('skipping non-updated test') def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() @unittest.skip('skipping non-updated test') def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() @unittest.skip('skipping non-updated test') def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s\ ' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) @unittest.skip('skipping non-updated test') def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! \ %s' % int(time.time())) # Friends & Followers @unittest.skip('skipping non-updated test') def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) @unittest.skip('skipping non-updated test') def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') @unittest.skip('skipping non-updated test') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() @unittest.skip('skipping non-updated test') def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() @unittest.skip('skipping non-updated test') def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) @unittest.skip('skipping non-updated test') def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users @unittest.skip('skipping non-updated test') def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() @unittest.skip('skipping non-updated test') def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() @unittest.skip('skipping non-updated test') def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') @unittest.skip('skipping non-updated test') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') @unittest.skip('skipping non-updated test') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') @unittest.skip('skipping non-updated test') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') @unittest.skip('skipping non-updated test') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() @unittest.skip('skipping non-updated test') def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() @unittest.skip('skipping non-updated test') def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') @unittest.skip('skipping non-updated test') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') @unittest.skip('skipping non-updated test') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') @unittest.skip('skipping non-updated test') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) @unittest.skip('skipping non-updated test') def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() @unittest.skip('skipping non-updated test') def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) @unittest.skip('skipping non-updated test') def test_list_mutes(self): """Test listing users who are muted by the authenticated user succeeds""" self.api.list_mutes() @unittest.skip('skipping non-updated test') def test_list_mute_ids(self): """Test listing user ids who are muted by the authenticated user succeeds""" self.api.list_mute_ids() @unittest.skip('skipping non-updated test') def test_create_mute(self): """Test muting a user succeeds""" self.api.create_mute(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_mute(self): """Test muting a user succeeds""" self.api.destroy_mute(screen_name='justinbieber') # Suggested Users @unittest.skip('skipping non-updated test') def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') @unittest.skip('skipping non-updated test') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() @unittest.skip('skipping non-updated test') def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites @unittest.skip('skipping non-updated test') def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() @unittest.skip('skipping non-updated test') def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists @unittest.skip('skipping non-updated test') def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed \ %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) @unittest.skip('skipping non-updated test') def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() @unittest.skip('skipping non-updated test') def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') @unittest.skip('skipping non-updated test') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches @unittest.skip('skipping non-updated test') def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() @unittest.skip('skipping non-updated test') def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo @unittest.skip('skipping non-updated test') def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') @unittest.skip('skipping non-updated test') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') @unittest.skip('skipping non-updated test') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') @unittest.skip('skipping non-updated test') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends @unittest.skip('skipping non-updated test') def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) @unittest.skip('skipping non-updated test') def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() @unittest.skip('skipping non-updated test') def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help @unittest.skip('skipping non-updated test') def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() @unittest.skip('skipping non-updated test') def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() @unittest.skip('skipping non-updated test') def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() @unittest.skip('skipping non-updated test') def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() @unittest.skip('skipping non-updated test') def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
def get_active_users_around(center, start_date, end_date, threshold, num_friends, num_followers, waiting): twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) user_ids = [] all_tweets = [] # get friends and followers around a central user print "Getting users..." while True: try: following = twitter.get_friends_ids(screen_name=center, count=5000) time.sleep(0.1) followers = twitter.get_followers_ids(screen_name=center, count=5000) break except TwythonAuthError as e: print "TwythonAuthError..." time.sleep(waiting) continue except TwythonRateLimitError as e: print "TwythonRateLimitError... Waiting for 3 minutes (may take multiple waits)" for timer in tqdm(range(3*60)): time.sleep(1) twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) time.sleep(waiting) continue except TwythonError as e: print "TwythonError: " + str(e.error_code) + "..." time.sleep(waiting) continue # choose out random friends and followers from the previous list, with a safety factor of 2 times as many as asked for to account for the fact that some users will not be active following = map(lambda x: str(x), np.random.choice(following['ids'], min(int(num_friends * 2), len(following['ids'])), replace=False)) followers = map(lambda x: str(x), np.random.choice(followers['ids'], min(int(num_followers * 2), len(followers['ids'])), replace=False)) following = get_usernames(following, waiting) followers = get_usernames(followers, waiting) # figure out how many tweets are necessary to deem a user "active" start = datetime.strptime(start_date, "%Y-%m-%d") end = datetime.strptime(end_date, "%Y-%m-%d") threshold = abs(end-start).days * threshold # collect out as many active followed users from the central user (and their tweets) as we can, up to a limit of the number asked for collected_friends = [] for user in following: user_timeline = get_timeline_in_range(user, start_date, end_date, waiting) if len(user_timeline) >= threshold: collected_friends.append(user) all_tweets.extend(user_timeline) if len(collected_friends) >= num_friends: break # collect out as many active followers from the central user (and their tweets) as we can, up to a limit of the number asked for collected_followers = [] for user in followers: user_timeline = get_timeline_in_range(user, start_date, end_date, waiting) if len(user_timeline) >= threshold: collected_followers.append(user) all_tweets.extend(user_timeline) if len(collected_followers) >= num_followers: break # return our list of users and their tweets from the time period in question user_ids.extend(collected_friends) user_ids.extend(collected_followers) return (user_ids, all_tweets)
cur.execute("SELECT Profiles.idFollower FROM Profiles LEFT OUTER JOIN Users on Profiles.idFollower = Users.idUser WHERE Users.idUser is null GROUP BY Profiles.idFollower LIMIT 0,200") rows = cur.fetchall() currentAccount = 0 iRow = 1 for row in rows: cur2 = con.cursor() cur2.execute("SELECT Count(*) FROM Users WHERE idUser="******"idFollower"])) numberOfRows = cur2.fetchone()[0] if(numberOfRows==0): try: error = False friends = twitter.get_friends_ids(user_id = row["idFollower"]) except TwythonAuthError: error = True cur4 = con.cursor() cur4.execute("DELETE FROM Users WHERE idUser="******"idFollower"])) cur4.execute("DELETE FROM Profiles WHERE idFollower="+str(row["idFollower"])) except TwythonRateLimitError: error = True currentAccount+=1 if currentAccount == len(APP_KEY): print 'Limit excedeed and all accounts have been used, quitting ...' con.commit() con.close() execfile('/Users/cyriltrosset/Dropbox/IIT/Special project/ctrosset/remainRows.py') sys.exit() else:
def get_following(): distinct = getScreenname.getDistinctScreenames( "all_tweets_dict_fully_checked") index = 0 num_checked = 0 rate_ex = 0 for sname in distinct: file = open("following.txt", "r+") already_checked = False for line in file: words = line.split('\t') if words[0] == sname: already_checked = True if index >= 8: break APP_KEY = APP_KEYS[index] APP_SECRET = APP_SECRETS[index] OAUTH_TOKEN = OAUTH_TOKENS[index] OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index] if not already_checked: try: twitter = Twython(APP_KEY, APP_SECRET) auth = twitter.get_authentication_tokens() twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) following = twitter.get_friends_ids(screen_name=sname) print sname file.write(sname) file.write("\t") for ID in following['ids']: file.write(str(ID) + " ") file.write("\n") num_checked += 1 except Exception as e: if "429 (Too Many Requests)" in str(e): index += 1 elif "401 (Unauthorized)" in str(e): print "401 error" file.write(sname) file.write("\t") file.write("Unauthorized") file.write("\n") elif "404 (Not Found)" in str(e): print "404 error" file.write(sname) file.write("\t") file.write("NotFound") file.write("\n") else: print e rate_ex += 1 file.close() if (num_checked != 0 or rate_ex != 0): print "sleepy time" time.sleep(900) get_following()
def updateFollowerFollowings(dbpath, user_table="users"): distinct = get_D_UsrID(dbpath, sn_table) conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "select Usr_ID from %s where Following is null or Followers is null" cursor.execute(query) distinct = cursor.fetchall() follower_cursor = -1 friend_cursor = -1 print "Starting @ %d" % (ind) print datetime.datetime.now() while ind < len(distinct): #for i in distinct: i = distinct[ind][0] # Usr_ID APP_KEY = APP_KEYS[index] APP_SECRET = APP_SECRETS[index] OAUTH_TOKEN = OAUTH_TOKENS[index] OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index] try: #twitter = Twython (APP_KEY, APP_SECRET) #auth = twitter.get_authentication_tokens() twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) friends = "" while (friend_cursor): following = twitter.get_friends_ids(id=i, cursor=friend_cursor) for ID in following['ids']: friends += str(ID) + " " friend_cursor = following["next_cursor"] num_checked += 1 friends = friends[:-1] if len(friends) == 0: friends = "null" follow = "" while (follower_cursor): followers = twitter.get_followers_ids(id=i, cursor=follower_cursor) for ID in followers['ids']: follow += str(ID) + " " follower_cursor = followers["next_cursor"] num_checked += 1 follow = follow[:-1] if len(follow) == 0: follow = "null" conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = '%s', Followers = '%s', NumFollowing = '%d', NumFollowers = '%d' where Usr_ID = '%s'" % ( sn_table, friends, follow, len( friends.split(" ")), len(follow.split(" ")), i) cursor.execute(query) #print query conn.close() print "____________________ %dth following and friends and their corresponding counts updated ________ %d" % ( ind, i) follower_cursor = -1 friend_cursor = -1 ind += 1 except Exception as e: print e if "429 (Too Many Requests)" in str(e): index += 1 if index == len(APP_KEYS): index = 0 print "sleepy time; pausing for 15 minutes for APP Keys to renew" print datetime.datetime.now() #time.sleep(910) print "App Rate Limits has been exceeded- we will wait for next runtime before running rest of followers and followings" return elif "401 (Unauthorized)" in str(e): print "401 Unauthorized Error nulling %d________________Usr_ID:%d" % ( ind, i) conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'" % ( sn_table, i) cursor.execute(query) conn.commit() conn.close() ind += 1 elif "404 (Not Found)" in str(e): print "404 Not Found Error nulling %d________________Usr_ID:%d" % ( ind, i) conn = sqlite3.connect(dbpath) cursor = conn.cursor() query = "UPDATE %s SET Following = 'null', Followers = 'null' where Usr_ID = '%s'" % ( sn_table, i) cursor.execute(query) conn.commit() conn.close() ind += 1 else: print e
class TwythonAPITestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': { } # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual( constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_get(self): """Test Twython generic GET request works""" self.api.get('account/verify_credentials') def test_post(self): """Test Twython generic POST request works, with a full url and with just an endpoint""" update_url = 'https://api.twitter.com/1.1/statuses/update.json' status = self.api.post( update_url, params={'status': 'I love Twython! %s' % int(time.time())}) self.api.post('statuses/destroy/%s' % status['id_str']) def test_get_lastfunction_header(self): """Test getting last specific header of the last API call works""" self.api.get('statuses/home_timeline') self.api.get_lastfunction_header('x-rate-limit-remaining') def test_get_lastfunction_header_not_present(self): """Test getting specific header that does not exist from the last call returns None""" self.api.get('statuses/home_timeline') header = self.api.get_lastfunction_header('does-not-exist') self.assertEqual(header, None) def test_get_lastfunction_header_no_last_api_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') def test_cursor(self): """Test looping through the generator results works, at least once that is""" search = self.api.cursor(self.api.search, q='twitter', count=1) counter = 0 while counter < 2: counter += 1 result = next(search) new_id_str = int(result['id_str']) if counter == 1: prev_id_str = new_id_str time.sleep( 1) # Give time for another tweet to come into search if counter == 2: self.assertTrue(new_id_str > prev_id_str) def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR exapanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text) def test_raise_error_on_bad_ssl_cert(self): """Test TwythonError is raised by a RequestException when an actual HTTP happens""" self.assertRaises(TwythonError, self.api.get, 'https://example.com') # Timelines def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline( screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status( status='Test post just to get deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time())) # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
class TwythonEndpointsTestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) # Timelines @unittest.skip('skipping non-updated test') def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() @unittest.skip('skipping non-updated test') def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) @unittest.skip('skipping non-updated test') def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() @unittest.skip('skipping non-updated test') def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets @unittest.skip('skipping non-updated test') def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) @unittest.skip('skipping non-updated test') def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) @unittest.skip('skipping non-updated test') def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') @unittest.skip('skipping non-updated test') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search @unittest.skip('skipping non-updated test') def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages @unittest.skip('skipping non-updated test') def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() @unittest.skip('skipping non-updated test') def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() @unittest.skip('skipping non-updated test') def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) @unittest.skip('skipping non-updated test') def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time())) # Friends & Followers @unittest.skip('skipping non-updated test') def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) @unittest.skip('skipping non-updated test') def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') @unittest.skip('skipping non-updated test') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() @unittest.skip('skipping non-updated test') def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() @unittest.skip('skipping non-updated test') def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) @unittest.skip('skipping non-updated test') def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) @unittest.skip('skipping non-updated test') def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users @unittest.skip('skipping non-updated test') def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() @unittest.skip('skipping non-updated test') def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() @unittest.skip('skipping non-updated test') def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') @unittest.skip('skipping non-updated test') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') @unittest.skip('skipping non-updated test') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') @unittest.skip('skipping non-updated test') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') @unittest.skip('skipping non-updated test') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() @unittest.skip('skipping non-updated test') def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() @unittest.skip('skipping non-updated test') def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') @unittest.skip('skipping non-updated test') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') @unittest.skip('skipping non-updated test') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') @unittest.skip('skipping non-updated test') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') @unittest.skip('skipping non-updated test') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) @unittest.skip('skipping non-updated test') def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() @unittest.skip('skipping non-updated test') def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users @unittest.skip('skipping non-updated test') def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') @unittest.skip('skipping non-updated test') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() @unittest.skip('skipping non-updated test') def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites @unittest.skip('skipping non-updated test') def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() @unittest.skip('skipping non-updated test') def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists @unittest.skip('skipping non-updated test') def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) @unittest.skip('skipping non-updated test') def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() @unittest.skip('skipping non-updated test') def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') @unittest.skip('skipping non-updated test') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) @unittest.skip('skipping non-updated test') def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') @unittest.skip('skipping non-updated test') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches @unittest.skip('skipping non-updated test') def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() @unittest.skip('skipping non-updated test') def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo @unittest.skip('skipping non-updated test') def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') @unittest.skip('skipping non-updated test') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') @unittest.skip('skipping non-updated test') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') @unittest.skip('skipping non-updated test') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends @unittest.skip('skipping non-updated test') def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) @unittest.skip('skipping non-updated test') def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() @unittest.skip('skipping non-updated test') def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help @unittest.skip('skipping non-updated test') def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() @unittest.skip('skipping non-updated test') def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() @unittest.skip('skipping non-updated test') def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() @unittest.skip('skipping non-updated test') def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() @unittest.skip('skipping non-updated test') def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
#this part sucks, getting all your secret keys and such from Twitter API: https://dev.twitter.com/docs/auth/tokens-devtwittercom APP_KEY = "app key here, a bunch of numbers" APP_SECRET = "app secret here, a bunch more numbers" OAUTH_TOKEN = "more numbers for your oauth token" OAUTH_TOKEN_SECRET = "last secret token" twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) #make a list of friends allfriendsIDs = [] sn = "your_twitter_username" #get your friends_ids first response = twitter.get_friends_ids(screen_name=sn) friends = response['ids'] allfriendsIDs += friends #now get their information, in batches of 100 because someone told me Twitter doesn't like you to get lots of people at once (what do i know though) def getUserDetails(ids): userDetails = [] if len(ids) > 100: response = twitter.lookup_user(user_id=','.join( str(a) for a in ids[0:99])) userDetails += response return getUserDetails(ids[100:]) + userDetails else: response = twitter.lookup_user(user_id=','.join(str(a) for a in ids)) userDetails += response
friends_query_size = 5000 res = db.friends.find_one({"screen_name": screen_name}) if res is None: db.friends.insert_one({"screen_name": screen_name, "ids": []}) next_cursor = -1 friends_ids = list() ids_count = 0 n_friends = 600 ## Tras iniciar, se hace la llamada a la API con get_friends_ids(). while (next_cursor != 0) and (ids_count < n_friends): try: result = twitter.get_friends_ids(screen_name=screen_name, count=friends_query_size, cursor=next_cursor) friends_ids = friends_ids + result['ids'] next_cursor = result['next_cursor'] ids_count += len(result['ids']) friends_ids = list(set(friends_ids)) friends_ids.sort() ## Se inserta la lista de id en la BB. DD. res = db.friends.update_one({"screen_name": screen_name}, {'$set': { "ids": friends_ids }}) except TwythonRateLimitError as e:
API_SECRET = 'INSERT HERE' ACCESS_TOKEN = 'INSERT HERE' ACCESS_TOKEN_SECRET = 'INSERT HERE' twitter = Twython(API_KEY, API_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) return (twitter) ''' Determining your Twitter followers and friends ''' twitter = twitter_oauth_login() friends_ids = twitter.get_friends_ids(count=5000) friends_ids = friends_ids['ids'] followers_ids = twitter.get_followers_ids(count=5000) followers_ids = followers_ids['ids'] print len(friends_ids), len(followers_ids) friends_set = set(friends_ids) followers_set = set(followers_ids) print( 'Number of Twitter users who either are our friend or follow you (union):') print(len(friends_set.union(followers_set))) len(friends_set | followers_set)
""" #full_list = ["mbfhunzaker"] #single user for testing scraped_users = [] bad_ids = [] for leader in full_list: scraped_users.append( leader ) #keeps list of who's already been scraped incase you have to stop/start the code #^^ is output as the file that will be imported as the "already_done_list" above cur = -1 # used to function argument to 1st page of followers--idk why -1 if the first page... try: while cur != 0: # until you run out of pages of followers... followers = t.get_friends_ids( screen_name=leader, cursor=cur ) #this line is the API call that looks up the people that user follows in chunks of 5000 follower_ids = followers[ "ids"] #extracts list of follower ids from API call data for user_id in follower_ids: #iterates over list of ids if user_id not in scrapedInfo: #if we dont already have the user in our dictionary of users print(user_id) #scrapedInfo[user_id]=leader #method to keep track of who follows scrapedInfo[ user_id] = 1 #method to keep track of how many follow; sets to 1 if not in our set yet elif user_id in scrapedInfo: #if we DO have the user in our dictionary of users (e.g. if they're followed by someone we already scraped) #scrapedInfo[user_id] = scrapedInfo[user_id] + ", " + leader #method to keep track of *who* follows scrapedInfo[user_id] = scrapedInfo[ user_id] + 1 #method to keep track of how many follows them; increments count of elites who follow them by 1 else: print(
for line in fp1: user = int(line.rstrip()) userIDs.append(user) apiKey = 'Tl4x7q0fMxTTnwU4gacWOaTBW' #str(argList[1]) accessToken = 'AAAAAAAAAAAAAAAAAAAAAA2BewAAAAAABUUGEYgpIQ1z056Xaib2hyrc6mI%3DTjG1tqzGOv7PfiTJAjPtkA2AmVkw4gqmPduB4UnIteNK2xEpAw' #str(argList[2]) twitter = Twython(apiKey,access_token = accessToken,client_args = client_args) for user in userIDs: cur = -1 friendList = [] while cur != 0: try: response = twitter.get_friends_ids(user_id=user,count=5000,cursor = cur) print response except Exception as e: print e continue cur = response['next_cursor'] friendList = friendList + response['ids'] if cur != 0: continue dictionary = {} dictionary[str(user)] = friendList fp2.write("{0}\n".format(dictionary)) fp1.close() fp2.close()
def on_demand(items): """ Use generate() to provide a new tweet for a user when they tweet 'hit me' """ for tweet in reversed(items): text = tweet['text'][12:] if ((text[:6].strip().lower() == "hit me") and (int(tweet['id']) > lastTweet)): newJargon = generate() newTweet = "@" + tweet['user']['screen_name'] + " " + newJargon newTweet = newTweet[:139] twitter.update_status(status=newTweet, in_reply_to_status_id=int(tweet['id'])) def periodic(): """ Periodically tweet out a jargon phrase using generate() """ tweetCheck = randint(0,35) if (tweetCheck == 5): newTweet = generate() twitter.update_status(status=newTweet) tweets = twitter.get_mentions_timeline() get_last() intake(simplify(tweets, twitter.get_friends_ids()['ids'])) on_demand(tweets) periodic()
t = Twython(app_key=TWITTER_API_KEY, app_secret=TWITTER_API_KEY_SECRET, oauth_token=TWITTER_ACCESS_TOKEN, oauth_token_secret=TWITTER_ACCESS_TOKEN_SECRET) client2=pymongo.MongoClient(mdc.u2) i=0 for nn in N: print i; i+=1 teste=client2.sna.amizades.find({str(IDS[snames.index(nn)]):{"$exists":True}}) teste=[testee for testee in teste] if not teste: print nn, "not teste" tids=t.get_friends_ids(screen_name=nn)["ids"] id_orig=W[snames.index(nn)]["user"]["id"] client2.sna.amizades.insert({str(id_orig):tids}) else: print nn, "teste" T.sleep(90) # #while 1: # search = t.search(q='#arenaNETmundial',count=150,max_id=tweets[-1]['id']-1) # i=0 # while len(search['statuses'])>0: # tweets +=search['statuses'] # #search = t.search(q='#arenaNETmundial', count=150, max_id=tweets[-1]['id']-1) # print "older", i, len(tweets),search['statuses']; i+=1 #
class Twitter(object): """ Class for twitter use """ query_string = '#севастополь -украина OR северная OR катер OR SevastopolBoats OR sevboats :) since:%s lang:ru' weather_string = u'#погода в Севастополе на %s.' timelimit = 90 max_follow = 100 def __init__(self, auth=TWITTER_OAUTH_INFO): self.twitter = Twython(**auth) def post_tweet(self, message): """ Writes message into twitter stream """ return self.twitter.update_status(status=message, trim_user=True) def delete_tweet(self, tweet_id): """ Remove tweet from timeline """ return self.twitter.destroy_status(id=tweet_id, trim_user=True) def post_debug(self, message): """ Writes message into console to test """ print message return dict(id=0, id_str='0', text=message) def search(self, query=None, count=10): """ Search tweet with sevboats ref """ if query is None: since = (datetime.datetime.now() - datetime.timedelta(days=7)).date().strftime('%Y-%m-%d') query = self.query_string % since if 'since:%s' in self.query_string else self.query_string return self.twitter.search(q=query, count=count) def follow(self, user_id): """ Follow user by id """ try: return self.twitter.create_friendship(user_id=user_id, follow=True) except TwythonError: return dict(status='error', id_str=None) def unfollow(self, user_id): """ Un Follow user by id """ try: return self.twitter.destroy_friendship(user_id=user_id) except TwythonError: return dict(status='error', id_str=None) def follow_list(self, friends_ids): """ Follow user in search result """ friends = [] for user_id in friends_ids: friends.append(self.follow(user_id)['id_str']) return friends def unfollow_list(self, friends_ids): """ Un Follow user in ids list """ friends = [] for user_id in friends_ids: friends.append(self.unfollow(user_id)['id_str']) return friends def search_to_list(self, search_list=None): """ Follow user in search result """ if search_list is None: search_list = self.search() users_ids = [] for tweet in search_list['statuses']: users_ids.append(tweet['user']['id_str']) return users_ids def my_followers(self): """ Get list of my followers """ followers_ids = [] next_cursor = -1 while next_cursor != '0': cursor = next_cursor fids = self.twitter.get_followers_ids(cursor=cursor, stringify_ids=True, count=1000) followers_ids += fids['ids'] next_cursor = fids['next_cursor_str'] time.sleep(self.timelimit) return followers_ids def follow_search(self): """ Follow for all user in search results """ users_ids = self.search_to_list() self.follow_list(users_ids) return users_ids def follow_followers(self): """ Follow for all followers """ followers_ids = self.my_followers() friends_ids = self.i_follow() users_ids = [] for follower in followers_ids: if follower not in friends_ids: users_ids.append(follower) self.follow_list(users_ids[:self.max_follow]) return users_ids def i_follow(self): """ Get list of user i follow what """ friends_ids = [] next_cursor = -1 while next_cursor != '0': cursor = next_cursor fids = self.twitter.get_friends_ids(cursor=cursor, stringify_ids=True, count=1000) friends_ids += fids['ids'] next_cursor = fids['next_cursor_str'] time.sleep(self.timelimit) return friends_ids def post_image_weather(self): """ Create weather image and post to twitter """ # get weather image print 'start' weth = Weather() weather_img, date_title = weth.draw_img() # create stream print 'streem' image_io = StringIO() # weather_img.save(image_io, format='JPEG', quality=100, subsampling=0, optimize=True, progressive=True) weather_img.save(image_io, format='PNG') image_io.seek(0) # post tweet print 'post' return self.twitter.update_status_with_media(media=image_io, status=self.weather_string % date_title)
def addUserToDB(sn): APP_KEY = "UHz2Yr3qedZeNZxblgzqVbER6" APP_SECRET = "KavH3dnHPOPybPWBWXAWD7jAqfJRxDLqEK24N1m64O2fax0vM0" twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2) ACCESS_TOKEN = twitter.obtain_access_token() twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) analyzer = SentimentIntensityAnalyzer() totalSentimentScore = 0 tweetCount = 0 compoundText = "" tweets = [] try: user_timeline = twitter.get_user_timeline(screen_name=sn, count=30) except TwythonError as e: print(e) return False for tweet in user_timeline: tweetCount += 1 sentiment_dict = analyzer.polarity_scores(tweet['text']) totalSentimentScore += sentiment_dict['compound'] compoundText = compoundText + " " + tweet['text'] tweets.append(tweet) syntax_dict = count_by_word(compoundText) averageSentiment = 0 if tweetCount != 0: averageSentiment = totalSentimentScore / tweetCount ## FOLLOWERS ## followers = [] try: user_followers = twitter.get_followers_ids(screen_name=sn) followers = user_followers except TwythonError as e: print(e) return False following = [] try: user_friends = twitter.get_friends_ids(screen_name=sn) following = user_friends except TwythonError as e: print(e) return False lastUpdated = datetime.datetime.utcnow().isoformat() syntax_dict = count_by_word(compoundText) syntax_dict = list(filter(lambda x: x[1] != 1, syntax_dict)) userStore = { "_id": sn, "last_updated": lastUpdated, "avg_sentiment": averageSentiment, "syntax_dict": syntax_dict[:30], "following": following['ids'], "followers": followers['ids'], "tweets": tweets } client = pymongo.MongoClient( "mongodb+srv://tgabrie2:[email protected]/cs432?retryWrites=true&w=majority", ssl_cert_reqs=ssl.CERT_NONE) db = client.cs432 collection = db['test'] collection.insert_one(userStore) return True
twitter = Twython(APP_KEY,APP_SECRET, access_token=ACCESS_TOKEN) #creates the dictionary the results will be stored in stored=defaultdict(list) #Reads in the csv with the Twitter handles ifile = open('inpath.csv', "rb") for name in ifile: next_cursor=-1 #initiates the cursor try: while(next_cursor != 0):#keeps looping through as long as there are pages there print(name) time.sleep(60) # sets a time delay of 60 seconds to keep within the rate limits #gets the people followed by the screen name following=twitter.get_friends_ids(screen_name=str(name), cursor=next_cursor) for x in following[u'ids']: #loops through the ids returned by the Twitter api name=name.encode('utf-8') stored[str(name)].append(x) # and appends them to the dictionary where the key is the associated persons user id next_cursor=following['next_cursor'] #gets the next cursor value except: # if the call fails which may be because there are no more pages print('unable to access the people followed by '+str(name)+','+str(next_cursor)+'\n') #writes the dictionary as a json file to the computer with open('outpath.json', 'w') as f: json.dump(stored, f)
# from __future__ import print_function from twython import Twython, TwythonError twitter = Twython('PVRb7LmKFwBctrLKvULQTnSqi', access_token='AAAAAAAAAAAAAAAAAAAAAPaPWwAAAAAAkiazdF9EzC%2FtcilOFW0S7kYxh94%3DtI84ctlH9V3fZ4Cia3BX0jUmUWkQp2ERtTFfajoFherjwMJJrl') for line in open("217189228_follows"): listid = line.split(", ") # print listid num_attempt=0 for a in listid: try: num_attempt+=1; if num_attempt>=14: print "sleep" time.sleep(15*60) num_attempt=0 ids1 = twitter.get_friends_ids(id = a) target = open(a+"_follows",'w') for id in ids1['ids']: target.write(str(id)+', ') # target.write('\n') except TwythonError as e: print e # print(ids1['ids'],file = target) # print ids1['ids'] # for line in open("followee_217189228"): # listid = line.split(", ") # for a in listid: # user_timeline=twitter.get_user_timeline(user_id=a, count=200) # target = open(a,'a') # for tweet in user_timeline: # target.write(tweet['text'].encode('utf-8')+'\n')
def retrieve_friends_ids(users, file_name): # CONSUMER_KEY1 = my_creds['CONSUMER_KEY'] # CONSUMER_SECRET1 = my_creds['CONSUMER_SECRET'] # OAUTH_TOKEN1 = my_creds['TOKEN_KEY'] # OAUTH_TOKEN_SECRET1 = my_creds['TOKEN_SECRET'] CONSUMER_KEY2 = my_credsa['CONSUMER_KEY'] CONSUMER_SECRET2 = my_credsa['CONSUMER_SECRET'] OAUTH_TOKEN2 = my_credsa['TOKEN_KEY'] OAUTH_TOKEN_SECRET2 = my_credsa['TOKEN_SECRET'] CONSUMER_KEY3 = my_credsb['CONSUMER_KEY'] CONSUMER_SECRET3 = my_credsb['CONSUMER_SECRET'] OAUTH_TOKEN3 = my_credsb['TOKEN_KEY'] OAUTH_TOKEN_SECRET3 = my_credsb['TOKEN_SECRET'] CONSUMER_KEY4 = my_credsc['CONSUMER_KEY'] CONSUMER_SECRET4 = my_credsc['CONSUMER_SECRET'] OAUTH_TOKEN4 = my_credsc['TOKEN_KEY'] OAUTH_TOKEN_SECRET4 = my_credsc['TOKEN_SECRET'] CONSUMER_KEY5 = my_credsd['CONSUMER_KEY'] CONSUMER_SECRET5 = my_credsd['CONSUMER_SECRET'] OAUTH_TOKEN5 = my_credsd['TOKEN_KEY'] OAUTH_TOKEN_SECRET5 = my_credsd['TOKEN_SECRET'] CONSUMER_KEY6 = my_creds['CONSUMER_KEY'] CONSUMER_SECRET6 = my_creds['CONSUMER_SECRET'] OAUTH_TOKEN6 = my_creds['TOKEN_KEY'] OAUTH_TOKEN_SECRET6 = my_creds['TOKEN_SECRET'] creds = defaultdict(str) # creds['creds1'] = {'CONSUMER_KEY': CONSUMER_KEY1, 'CONSUMER_SECRET': CONSUMER_SECRET1, 'TOKEN_KEY': OAUTH_TOKEN1, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET1} creds['creds1'] = { 'CONSUMER_KEY': CONSUMER_KEY2, 'CONSUMER_SECRET': CONSUMER_SECRET2, 'TOKEN_KEY': OAUTH_TOKEN2, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET2 } creds['creds2'] = { 'CONSUMER_KEY': CONSUMER_KEY3, 'CONSUMER_SECRET': CONSUMER_SECRET3, 'TOKEN_KEY': OAUTH_TOKEN3, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET3 } creds['creds3'] = { 'CONSUMER_KEY': CONSUMER_KEY4, 'CONSUMER_SECRET': CONSUMER_SECRET4, 'TOKEN_KEY': OAUTH_TOKEN4, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET4 } creds['creds4'] = { 'CONSUMER_KEY': CONSUMER_KEY5, 'CONSUMER_SECRET': CONSUMER_SECRET5, 'TOKEN_KEY': OAUTH_TOKEN5, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET5 } creds['creds5'] = { 'CONSUMER_KEY': CONSUMER_KEY6, 'CONSUMER_SECRET': CONSUMER_SECRET6, 'TOKEN_KEY': OAUTH_TOKEN6, 'TOKEN_SECRET': OAUTH_TOKEN_SECRET6 } cred_inuse = 'creds1' last_use_time = [{ 'name': 'creds1', 'time': 0 }, { 'name': 'creds2', 'time': 0 }, { 'name': 'creds3', 'time': 0 }, { 'name': 'creds4', 'time': 0 }, { 'name': 'creds5', 'time': 0 }] print(creds[cred_inuse]) twitter_twy = Twython(creds[cred_inuse]['CONSUMER_KEY'], creds[cred_inuse]['CONSUMER_SECRET'], creds[cred_inuse]['TOKEN_KEY'], creds[cred_inuse]['TOKEN_SECRET']) # twitter_twy = Twython(CONSUMER_KEY2, CONSUMER_SECRET2, OAUTH_TOKEN2, OAUTH_TOKEN_SECRET2) count_rate_exceed = 0 with codecs.open(file_name, 'w') as out_file: for u in users: ids = [] next_cursor = -1 # print("Retrieving friends of {}".format(u)) out_file.write(' '.join(['-user', u, '\n'])) while (next_cursor): try: returned = twitter_twy.get_friends_ids(user_id=u, count=5000, cursor=next_cursor, stringify_ids=True) print('.', end='') for result in returned['ids']: ids.append(result) next_cursor = returned["next_cursor"] count_rate_exceed = 0 except TwythonRateLimitError: count_rate_exceed += 1 print('Too many requests') # if count_rate_exceed > 3: # time.sleep(200) # else: last_use_time = sorted(last_use_time, key=lambda x: x['time']) time_diff = time.time() - last_use_time[0]['time'] if time_diff < 900 and time_diff > 0: print('sleeping for {} seconds'.format(900 - time_diff)) time.sleep(900 - time_diff) last_use_time[0]['time'] = time.time() cred_inuse = last_use_time[0]['name'] print('New credential: {}'.format(cred_inuse)) twitter_twy = Twython(creds[cred_inuse]['CONSUMER_KEY'], creds[cred_inuse]['CONSUMER_SECRET'], creds[cred_inuse]['TOKEN_KEY'], creds[cred_inuse]['TOKEN_SECRET']) except TwythonAuthError: print("Couldn't authenticate {}".format(cred_inuse)) cred_inuse = cred_inuse[0:-1] + str( int(cred_inuse[-1]) % 5 + 1) print('trying {}'.format(cred_inuse)) twitter_twy = Twython(creds[cred_inuse]['CONSUMER_KEY'], creds[cred_inuse]['CONSUMER_SECRET'], creds[cred_inuse]['TOKEN_KEY'], creds[cred_inuse]['TOKEN_SECRET']) print('\n') print("friends of {}.***count:{}".format(u, len(ids))) out_file.write(','.join(ids)) out_file.write('\n') out_file.write('-------------------------\n') print("Process finished.")
newTweet = "@" + tweet['user']['screen_name'] + " " + newJargon newTweet = newTweet[:139] twitter.update_status(status=newTweet, in_reply_to_status_id=int(tweet['id'])) def periodic(): """ Periodically tweet out a jargon phrase using generate() """ tweetCheck = random.randint(0,15) if (tweetCheck == 5): newTweet = generate() twitter.update_status(status=newTweet) def administration(items): """ follow or reject new users who put in commands """ for tweet in reversed(items): if (tweet['user']['id'] == 22884755): text = tweet['text'][10:].split(" ") if (text[1] == "approve"): twitter.create_friendship(screen_name=text[2]) twitter.update_status(status="@"+ text[2] + " Good news, you've been approved! Please retry any additions prior to this message again.") elif (text[1] == "reject"): twitter.update_status(status="@"+ text[2] + " Sorry, I'm not going to add you right now.") tweets = twitter.get_mentions_timeline() getLast() intake(simplify(twitter.get_mentions_timeline(), twitter.get_friends_ids()['ids'])) onDemand(tweets) administration(tweets) periodic()
class TwitterBot(): ''' Crawls twitter and updates social network and tagged resources to repository ''' def __init__(self): self.service = TWITTER APP_KEY = "w7zjbW1qHtzX2GxbrHbeNgyPr" APP_SECRET = "qJFawkFxwYZX6U3DYNvJI3czWEjix2owU9wTXZ9BorMkhFU4Yb" self.twapi = Twython(APP_KEY, APP_SECRET) self.userBaseUri = 'http://twitter.com/%s' # % user screen_name self.tagBaseUri = 'http://twitter.com/#search?q=#%s' # % trend key word self.tweetBaseUri = 'http://twitter.com/%s/status/%s' # % (user screen_name, status id) @staticmethod def getname(): return self.service @staticmethod def getDefaultAvatar(): return defAvatar def setStartUserId(self, start_user_id): self.start_user_id = start_user_id def setMaxLevel(self, max_level): self.max_level = max_level def setLastAdded(self, last_added): self.last_added = last_added def updateDatabase(self, user_metadata, user_ntw, udate=None): raise NotImplementedError() def fetchUser(self, username): return (None, {'knows':[], 'fans':[]}) def getUserUri(self, username): return self.userBaseUri % username @staticmethod def factory(user, params=None, depth=2, last_added=None, verbose=False): # Twitter crawler twcrawl = TwitterBot() twcrawl.setStartUserId(user) twcrawl.setMaxLevel(depth) twcrawl.setLastAdded(last_added) return twcrawl def updateDatabase(self, user_metadata, user_ntw, udate=None): username = user_metadata['username'] userUri = self.userBaseUri % username resourceUri = None added = 0 # add user to repository dbaccess.User.addToDB(userUri, self.service, username, TwitterBot.getDefaultAvatar(), udate=udate) # add resources, tags and relations between them to repository for resource in user_metadata['data']: # add tweet resource to repository resourceId = resource['id'] resourceTitle = resource['text'] resourceUri = self.tweetBaseUri % (username, resourceId) dbaccess.Tweet.addToDB(resourceUri, resourceTitle, udate=udate) added += 1 # add tag(trend) relations to repository for tag in self.parseTags(resourceTitle): tagUri = self.tagBaseUri % urllib.quote(tag.encode('utf-8')) # add tag to repository dbaccess.Tag.addToDB(tagUri, tag, udate=udate) # add user, resource, tag relation to repository dbaccess.UserResourceTag.addToDB(userUri, resourceUri, tagUri, udate=udate) # add url relations between the tweet and urls contained urls = map(self.getRealUrl, self.parseUrls(resourceTitle)) for url in urls: dbaccess.Bookmark.addToDB(url, None) dbaccess.Tweet.addReferenceToDB(resourceUri, url) # add user social network relations to repository # add friends for u in user_ntw['knows']: knownUserUri = self.userBaseUri % u dbaccess.User.addToDB(knownUserUri, self.service, u, TwitterBot.getDefaultAvatar(), udate=udate) dbaccess.User.addToDBUserKnowsUser(userUri, knownUserUri, udate=udate) # add followers for u in user_ntw['fans']: otherUserUri = self.userBaseUri % u dbaccess.User.addToDB(otherUserUri, self.service, u, TwitterBot.getDefaultAvatar(), udate=udate) dbaccess.User.addToDBUserKnowsUser(otherUserUri, userUri, udate=udate) # return number of added resources and last added resource uri return added def fetchUserMetadata(self, username): # get user tweets (only last 20) data = self.twapi.get_user_timeline(screen_name=username) return { 'username': username, 'data': data} def fetchUserNetwork(self, username): chunk_size = 100 # max returned by bulkUserLookup # Get Friends fids, cursor = [], -1 fchunks = [fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)] while True: f = self.twapi.get_friends_ids(screen_name=username, cursor=cursor) fids.extend(f['ids']) if f['next_cursor'] == cursor or len(f['ids']) == 0: break cursor = f['next_cursor'] # get screen names from ids in chunks of 100 screen_names = [] for chunk in fchunks: screen_names = [] screen_names.extend([userdata['screen_name'] for userdata in self.twapi.lookup_user(user_id=chunk)]) ntw_knows = screen_names print ntw_knows # Get Followers fids, cursor = [], -1 while True: f = self.twapi.get_followers_ids(screen_name=username, cursor=cursor) fids.extend(f['ids']) if f['next_cursor'] == cursor or len(f['ids']) == 0: break cursor = f['next_cursor'] fchunks = [fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size)] fan_screen_names = [] for chunk in fchunks: # get all data for all user ids fan_screen_names.extend([userdata['screen_name'] for userdata in self.twapi.lookup_user(user_id=chunk)]) ntw_fans = fan_screen_names return {'knows': ntw_knows, 'fans': ntw_fans} def fetchUser(self, username): return (self.fetchUserMetadata(username), self.fetchUserNetwork(username)) def parseTags(self, text): ''' Parses trends from tweet: '#trend #tre#nd ## #trend,two #tren%^& smth,#da #1dc #100 #1dc$ #100^', finds trends: trend, tre, trend, tren, da, 1dc, 100, 1dc, 100. Text parameter MUST be UNICODE encoded: u'some text' ''' return re.findall('(?<=[\W^#]#)\w+', ' ' + text, re.UNICODE) def parseUrls(self, text): '''Returns a list of urls from the text. ''' return re.findall('http://\S+', text) def getRealUrl(self, url): '''Returns the redirected url if *url* parameter is a shortened url (a redirect was produced), or itself if not. ''' try: u = urllib2.urlopen(url) realurl = u.geturl() if realurl == None: raise urllib2.HTTPError('Could not fetch url') except: realurl = url return realurl def crawlUserNetwork(self, user_id=None, max_level=0, start_time=None): ''' user_id - an id to uniquely identify a user (can be a username or userid) ''' user_id = user_id or self.start_user_id max_level = max_level or self.max_level queue = [user_id] # init queue visited = {user_id: 0} # init visited nodes added = 0 # set number of added resources while queue: v = queue.pop(0) # pop next user node level = visited[v] # get their level # get user metadata and social network user_metadata, user_ntw = self.fetchUser(v) # update database with user data, tagged resources and social network relations a = self.updateDatabase(user_metadata, user_ntw, udate=start_time) added += a # explore following nodes for w in user_ntw['knows']: if w not in visited and level < max_level: queue.append(w) visited[w] = level + 1 return added
def get_followers_following(usr_id): index = 0 print "GETTING FOLLOWERS AND FOLLOWING: " + str(usr_id) friend_cursor = -1 follower_cursor = -1 APP_KEY = APP_KEYS[index] APP_SECRET = APP_SECRETS[index] OAUTH_TOKEN = OAUTH_TOKENS[index] OAUTH_TOKEN_SECRET = OAUTH_TOKEN_SECRETS[index] while (1 == 1): try: twitter = Twython(APP_KEY, APP_SECRET) auth = twitter.get_authentication_tokens() twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) friends = "" while (friend_cursor != 0): following = twitter.get_friends_ids(id=usr_id, cursor=friend_cursor) for ID in following['ids']: friends += str(ID) + " " friend_cursor = following["next_cursor"] friends = friends[:-1] if len(friends) == 0: friends = "null" follow = "" while (follower_cursor != 0): followers = twitter.get_followers_ids(id=usr_id, cursor=follower_cursor) for ID in followers['ids']: follow += str(ID) + " " follower_cursor = followers["next_cursor"] follow = follow[:-1] if len(follow) == 0: follow = "null" return (friends, follow) except Exception as e: print e if "429 (Too Many Requests)" in str(e): #global index index += 1 if index == len(APP_KEYS): index = 0 print "sleepy time - 15 minutes" print datetime.datetime.now() time.sleep(910) return get_followers_following(usr_id) elif "401 (Unauthorized)" in str(e): print "401 error" return ("null", "null") elif "404 (Not Found)" in str(e): print "404 error" return ("null", "null") else: print e return ("null", "null")
class TwythonAPITestCase(unittest.TestCase): def setUp(self): self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret) # Timelines def test_get_mentions_timeline(self): '''Test returning mentions timeline for authenticated user succeeds''' self.api.get_mentions_timeline() def test_get_user_timeline(self): '''Test returning timeline for authenticated user and random user succeeds''' self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): '''Test returning a protected user timeline who you are following succeeds''' self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): '''Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError''' self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_get_home_timeline(self): '''Test returning home timeline for authenticated user succeeds''' self.api.get_home_timeline() # Tweets def test_get_retweets(self): '''Test getting retweets of a specific tweet succeeds''' self.api.get_retweets(id=test_tweet_id) def test_show_status(self): '''Test returning a single status details succeeds''' self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): '''Test updating and deleting a status succeeds''' status = self.api.update_status(status='Test post just to get deleted :(') self.api.destroy_status(id=status['id_str']) def test_retweet(self): '''Test retweeting a status succeeds''' retweet = self.api.retweet(id='99530515043983360') self.api.destroy_status(id=retweet['id_str']) def test_retweet_twice(self): '''Test that trying to retweet a tweet twice raises a TwythonError''' retweet = self.api.retweet(id='99530515043983360') self.assertRaises(TwythonError, self.api.retweet, id='99530515043983360') # Then clean up self.api.destroy_status(id=retweet['id_str']) def test_get_oembed_tweet(self): '''Test getting info to embed tweet on Third Party site succeeds''' self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): '''Test getting ids for people who retweeted a tweet succeeds''' self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): '''Test searching tweets succeeds''' self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): '''Test getting the authenticated users direct messages succeeds''' self.api.get_direct_messages() def test_get_sent_messages(self): '''Test getting the authenticated users direct messages they've sent succeeds''' self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): '''Test sending, getting, then destory a direct message succeeds''' message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d!') self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): '''Test sending a direct message to someone who doesn't follow you fails''' self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man!') # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): '''Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds''' self.api.get_user_ids_of_blocked_retweets(stringify_ids='true') def test_get_friends_ids(self): '''Test returning ids of users the authenticated user and then a random user is following succeeds''' self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): '''Test returning ids of users the authenticated user and then a random user are followed by succeeds''' self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): '''Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds''' self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): '''Test returning incoming friendship ids succeeds''' self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): '''Test returning outgoing friendship ids succeeds''' self.api.get_outgoing_friendship_ids() def test_create_friendship(self): '''Test creating a friendship succeeds''' self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): '''Test destroying a friendship succeeds''' self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): '''Test updating friendships succeeds''' self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets='false') def test_show_friendships(self): '''Test showing specific friendship succeeds''' self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): '''Test getting list of users authenticated user then random user is following succeeds''' self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): '''Test getting list of users authenticated user then random user are followed by succeeds''' self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): '''Test getting the authenticated user account settings succeeds''' self.api.get_account_settings() def test_verify_credentials(self): '''Test representation of the authenticated user call succeeds''' self.api.verify_credentials() def test_update_account_settings(self): '''Test updating a user account settings succeeds''' self.api.update_account_settings(lang='en') def test_update_delivery_service(self): '''Test updating delivery settings fails because we don't have a mobile number on the account''' self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): '''Test updating profile succeeds''' self.api.update_profile(include_entities='true') def test_update_profile_colors(self): '''Test updating profile colors succeeds''' self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): '''Test listing users who are blocked by the authenticated user succeeds''' self.api.list_blocks() def test_list_block_ids(self): '''Test listing user ids who are blocked by the authenticated user succeeds''' self.api.list_block_ids() def test_create_block(self): '''Test blocking a user succeeds''' self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): '''Test unblocking a user succeeds''' self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): '''Test listing a number of user objects succeeds''' self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): '''Test showing one user works''' self.api.show_user(screen_name='twitter') def test_search_users(self): '''Test that searching for users succeeds''' self.api.search_users(q='Twitter API') def test_get_contributees(self): '''Test returning list of accounts the specified user can contribute to succeeds''' self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): '''Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account''' self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): '''Test removing profile banner succeeds''' self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): '''Test getting list of profile banner sizes fails because we have not uploaded a profile banner''' self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): '''Test getting user suggestions by slug succeeds''' self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): '''Test getting user suggestions succeeds''' self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): '''Test getting status of suggested users succeeds''' self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): '''Test getting list of favorites for the authenticated user succeeds''' self.api.get_favorites() def test_create_and_destroy_favorite(self): '''Test creating and destroying a favorite on a tweet succeeds''' self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): '''Test show lists for specified user''' self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): '''Test timeline of tweets authored by members of the specified list succeeds''' self.api.get_list_statuses(list_id=test_list_id) def test_create_update_destroy_list_add_remove_list_members(self): '''Test create a list, adding and removing members then deleting the list succeeds''' the_list = self.api.create_list(name='Stuff') list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed') # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name='johncena,xbox') self.api.delete_list_members(list_id=list_id, screen_name='johncena,xbox') # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_memberships(self): '''Test list of lists the authenticated user is a member of succeeds''' self.api.get_list_memberships() def test_get_list_subscribers(self): '''Test list of subscribers of a specific list succeeds''' self.api.get_list_subscribers(list_id=test_list_id) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): '''Test subscribing, is a list sub and unsubbing to list succeeds''' self.api.subscribe_to_list(list_id=test_list_id) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(list_id=test_list_id, screen_name=screen_name) self.api.unsubscribe_from_list(list_id=test_list_id) def test_is_list_member(self): '''Test returning if specified user is member of a list succeeds''' # Returns 404 if not list member self.api.is_list_member(list_id=test_list_id, screen_name='jack') def test_get_list_members(self): '''Test listing members of the specified list succeeds''' self.api.get_list_members(list_id=test_list_id) def test_get_specific_list(self): '''Test getting specific list succeeds''' self.api.get_specific_list(list_id=test_list_id) def test_get_list_subscriptions(self): '''Test collection of the lists the specified user is subscribed to succeeds''' self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): '''Test collection of lists the specified user owns succeeds''' self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): '''Test getting list of saved searches for authenticated user succeeds''' self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): '''Test getting list of saved searches for authenticated user succeeds''' saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): '''Test getting info about a geo location succeeds''' self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): '''Test reversing geocode succeeds''' self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): '''Test search for places that can be attached to a statuses/update succeeds''' self.api.search_geo(query='Toronto') def test_get_similar_places(self): '''Test locates places near the given coordinates which are similar in name succeeds''' self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): '''Test getting the top 10 trending topics for a specific WOEID succeeds''' self.api.get_place_trends(id=1) def test_get_available_trends(self): '''Test returning locations that Twitter has trending topic information for succeeds''' self.api.get_available_trends() def test_get_closest_trends(self): '''Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds''' self.api.get_closest_trends(lat='37', long='-122')
class TwitterBot(): ''' Crawls twitter and updates social network and tagged resources to repository ''' def __init__(self): self.service = TWITTER APP_KEY = "w7zjbW1qHtzX2GxbrHbeNgyPr" APP_SECRET = "qJFawkFxwYZX6U3DYNvJI3czWEjix2owU9wTXZ9BorMkhFU4Yb" self.twapi = Twython(APP_KEY, APP_SECRET) self.userBaseUri = 'http://twitter.com/%s' # % user screen_name self.tagBaseUri = 'http://twitter.com/#search?q=#%s' # % trend key word self.tweetBaseUri = 'http://twitter.com/%s/status/%s' # % (user screen_name, status id) @staticmethod def getname(): return self.service @staticmethod def getDefaultAvatar(): return defAvatar def setStartUserId(self, start_user_id): self.start_user_id = start_user_id def setMaxLevel(self, max_level): self.max_level = max_level def setLastAdded(self, last_added): self.last_added = last_added def updateDatabase(self, user_metadata, user_ntw, udate=None): raise NotImplementedError() def fetchUser(self, username): return (None, {'knows': [], 'fans': []}) def getUserUri(self, username): return self.userBaseUri % username @staticmethod def factory(user, params=None, depth=2, last_added=None, verbose=False): # Twitter crawler twcrawl = TwitterBot() twcrawl.setStartUserId(user) twcrawl.setMaxLevel(depth) twcrawl.setLastAdded(last_added) return twcrawl def updateDatabase(self, user_metadata, user_ntw, udate=None): username = user_metadata['username'] userUri = self.userBaseUri % username resourceUri = None added = 0 # add user to repository dbaccess.User.addToDB(userUri, self.service, username, TwitterBot.getDefaultAvatar(), udate=udate) # add resources, tags and relations between them to repository for resource in user_metadata['data']: # add tweet resource to repository resourceId = resource['id'] resourceTitle = resource['text'] resourceUri = self.tweetBaseUri % (username, resourceId) dbaccess.Tweet.addToDB(resourceUri, resourceTitle, udate=udate) added += 1 # add tag(trend) relations to repository for tag in self.parseTags(resourceTitle): tagUri = self.tagBaseUri % urllib.quote(tag.encode('utf-8')) # add tag to repository dbaccess.Tag.addToDB(tagUri, tag, udate=udate) # add user, resource, tag relation to repository dbaccess.UserResourceTag.addToDB(userUri, resourceUri, tagUri, udate=udate) # add url relations between the tweet and urls contained urls = map(self.getRealUrl, self.parseUrls(resourceTitle)) for url in urls: dbaccess.Bookmark.addToDB(url, None) dbaccess.Tweet.addReferenceToDB(resourceUri, url) # add user social network relations to repository # add friends for u in user_ntw['knows']: knownUserUri = self.userBaseUri % u dbaccess.User.addToDB(knownUserUri, self.service, u, TwitterBot.getDefaultAvatar(), udate=udate) dbaccess.User.addToDBUserKnowsUser(userUri, knownUserUri, udate=udate) # add followers for u in user_ntw['fans']: otherUserUri = self.userBaseUri % u dbaccess.User.addToDB(otherUserUri, self.service, u, TwitterBot.getDefaultAvatar(), udate=udate) dbaccess.User.addToDBUserKnowsUser(otherUserUri, userUri, udate=udate) # return number of added resources and last added resource uri return added def fetchUserMetadata(self, username): # get user tweets (only last 20) data = self.twapi.get_user_timeline(screen_name=username) return {'username': username, 'data': data} def fetchUserNetwork(self, username): chunk_size = 100 # max returned by bulkUserLookup # Get Friends fids, cursor = [], -1 fchunks = [ fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size) ] while True: f = self.twapi.get_friends_ids(screen_name=username, cursor=cursor) fids.extend(f['ids']) if f['next_cursor'] == cursor or len(f['ids']) == 0: break cursor = f['next_cursor'] # get screen names from ids in chunks of 100 screen_names = [] for chunk in fchunks: screen_names = [] screen_names.extend([ userdata['screen_name'] for userdata in self.twapi.lookup_user(user_id=chunk) ]) ntw_knows = screen_names print ntw_knows # Get Followers fids, cursor = [], -1 while True: f = self.twapi.get_followers_ids(screen_name=username, cursor=cursor) fids.extend(f['ids']) if f['next_cursor'] == cursor or len(f['ids']) == 0: break cursor = f['next_cursor'] fchunks = [ fids[i:i + chunk_size] for i in range(0, len(fids), chunk_size) ] fan_screen_names = [] for chunk in fchunks: # get all data for all user ids fan_screen_names.extend([ userdata['screen_name'] for userdata in self.twapi.lookup_user(user_id=chunk) ]) ntw_fans = fan_screen_names return {'knows': ntw_knows, 'fans': ntw_fans} def fetchUser(self, username): return (self.fetchUserMetadata(username), self.fetchUserNetwork(username)) def parseTags(self, text): ''' Parses trends from tweet: '#trend #tre#nd ## #trend,two #tren%^& smth,#da #1dc #100 #1dc$ #100^', finds trends: trend, tre, trend, tren, da, 1dc, 100, 1dc, 100. Text parameter MUST be UNICODE encoded: u'some text' ''' return re.findall('(?<=[\W^#]#)\w+', ' ' + text, re.UNICODE) def parseUrls(self, text): '''Returns a list of urls from the text. ''' return re.findall('http://\S+', text) def getRealUrl(self, url): '''Returns the redirected url if *url* parameter is a shortened url (a redirect was produced), or itself if not. ''' try: u = urllib2.urlopen(url) realurl = u.geturl() if realurl == None: raise urllib2.HTTPError('Could not fetch url') except: realurl = url return realurl def crawlUserNetwork(self, user_id=None, max_level=0, start_time=None): ''' user_id - an id to uniquely identify a user (can be a username or userid) ''' user_id = user_id or self.start_user_id max_level = max_level or self.max_level queue = [user_id] # init queue visited = {user_id: 0} # init visited nodes added = 0 # set number of added resources while queue: v = queue.pop(0) # pop next user node level = visited[v] # get their level # get user metadata and social network user_metadata, user_ntw = self.fetchUser(v) # update database with user data, tagged resources and social network relations a = self.updateDatabase(user_metadata, user_ntw, udate=start_time) added += a # explore following nodes for w in user_ntw['knows']: if w not in visited and level < max_level: queue.append(w) visited[w] = level + 1 return added
stored = defaultdict(list) #Reads in the csv with the Twitter handles ifile = open('inpath.csv', "rb") for name in ifile: next_cursor = -1 #initiates the cursor try: while (next_cursor != 0): #keeps looping through as long as there are pages there print(name) time.sleep( 60 ) # sets a time delay of 60 seconds to keep within the rate limits #gets the people followed by the screen name following = twitter.get_friends_ids(screen_name=str(name), cursor=next_cursor) for x in following[ u'ids']: #loops through the ids returned by the Twitter api name = name.encode('utf-8') stored[str(name)].append( x ) # and appends them to the dictionary where the key is the associated persons user id next_cursor = following['next_cursor'] #gets the next cursor value except: # if the call fails which may be because there are no more pages print('unable to access the people followed by ' + str(name) + ',' + str(next_cursor) + '\n') #writes the dictionary as a json file to the computer with open('outpath.json', 'w') as f: json.dump(stored, f)
def createDatabase(request): twitter = Twython( settings.TWITTER_APP_KEY, settings.TWITTER_APP_SECRET, request.session['oauth_token'], request.session['oauth_token_secret'] ) rels = {} current_userid = request.session['oauth_token'].split('-')[0]; m = i = 5 mid = m/2 usersToQuery = [current_userid] users = [] current_userdeets = twitter.verify_credentials() users.append(current_userdeets) while i > 0: userid = usersToQuery[m-i] print "Index ", str(i) print "Querying userId ", str(userid) followers = twitter.get_followers_ids(count=100, user_id=userid)['ids'] for f in followers: if not f in rels: rels[f] = [userid] elif userid not in rels[f]: rels[f].append(userid) if len(followers) > 0: users = users + twitter.lookup_user(user_id=",".join(map(str, followers))) friends = twitter.get_friends_ids(count=100, user_id=userid)['ids'] for f in friends: if not userid in rels: rels[userid] = [f] elif f not in rels[userid]: rels[userid].append(f) if len(friends) > 0: users = users + twitter.lookup_user(user_id=",".join(map(str, friends))) if i == m: usersToQuery = usersToQuery + followers[0:mid] + friends[0:mid] i = i-1 fields = ['screen_name', 'name', 'location', 'created_at', 'id', 'favourites_count', 'url', 'lang', 'followers_count', 'friends_count', 'statuses_count', 'profile_image_url'] allUsers = [] usersIndex = {} for u in users: if u['id'] in usersIndex: continue tu = {} for f in fields: tu[f] = u[f] allUsers.append(tu) usersIndex[u['id']] = 1 # print "writing.." # save # with io.open(os.path.dirname(__file__) + '/users.txt', 'w', encoding='utf-8') as f: # f.write(unicode(json.dumps(allUsers, ensure_ascii=False))) # with io.open(os.path.dirname(__file__) + '/rels.txt', 'w', encoding='utf-8') as f: # f.write(unicode(json.dumps(rels, ensure_ascii=False))) print 'Writing to neo4j..' graph_db = getPy2neoService() graph_db.clear() batch = neo4j.WriteBatch(graph_db) batches = {} for u in allUsers: batches[str(u['id'])] = batch.create(node(**u)) for r in rels: for p in rels[r]: if str(r) in batches and str(p) in batches: batch.create(rel(batches[str(r)], "FOLLOWS", batches[str(p)])) else: print "Skipping.." results = batch.submit() print 'Writing done..' return current_userdeets['screen_name']
class Follow_Garbage_Collector(): def __init__(self): logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO) self.config_file = "/config/Follow_Garbage_Collector.ini" try: with open(self.config_file, "r") as f: f.close() try: self.config = configparser.ConfigParser() self.config.read(self.config_file) self.twitter_app_key = self.config['TWITTER']['APP_KEY'] self.twitter_app_secret = self.config['TWITTER']['APP_SECRET'] self.twitter_oauth_token = \ self.config['TWITTER']['OAUTH_TOKEN'] self.twitter_oauth_token_secret = \ self.config['TWITTER']['OAUTH_TOKEN_SECRET'] self.twitter_unfollow = True if ( self.config['TWITTER']['UNFOLLOW'] == "ON") else False self.twitter_years_inactive = \ int(self.config['TWITTER']['YEARS_INACTIVE']) self.twitter_excluded_tweeps = list( self.config['TWITTER']['EXCLUDED_TWEEPS'].split(",")) self.pushover_user_key = self.config['PUSHOVER']['USER_KEY'] self.pushover_token_api = self.config['PUSHOVER']['TOKEN_API'] self.pushover_sound = self.config['PUSHOVER']['SOUND'] except KeyError: logging.error("Can't get keys from INI file. " "Please check for mistakes.") sys.exit() except IOError or FileNotFoundError: logging.error(f"Can't open file {self.config_file}, " f"creating example INI file.") shutil.copyfile('/app/Follow_Garbage_Collector.ini.example', '/config/Follow_Garbage_Collector.ini.example') sys.exit() # Convert UTC times to local times def datetime_from_utc_to_local(self, utc_datetime): now_timestamp = time() offset = datetime.fromtimestamp( now_timestamp) - datetime.utcfromtimestamp(now_timestamp) return utc_datetime + offset # convert a tweetdatetime to datetime_utc def tweetdatetime_to_datetime_utc(self, tweetDate): return self.datetime_from_utc_to_local( datetime.strptime(tweetDate, "%a %b %d %H:%M:%S +0000 %Y")) def run(self): # Log a run logging.info("Executing Follow Garbage Collector.") # Setting for PushOver self.appPushover = Application(self.pushover_token_api) self.userPushover = self.appPushover.get_user(self.pushover_user_key) try: # This time we want to set our q to search for our keywords self.twitter = Twython( self.twitter_app_key, self.twitter_app_secret, self.twitter_oauth_token, self.twitter_oauth_token_secret, ) friends = self.twitter.get_friends_ids() for friend in friends["ids"]: user_timeline = self.twitter.get_user_timeline(user_id=friend, count=1) for tweet in user_timeline: diffDate = datetime.now() - \ self.datetime_from_utc_to_local( self.tweetdatetime_to_datetime_utc( tweet["created_at"])) if diffDate.days >= 365 * self.twitter_years_inactive: tweetDate = datetime.strftime( self.datetime_from_utc_to_local( self.tweetdatetime_to_datetime_utc( tweet["created_at"])), "%Y-%m-%d %H:%M:%S", ) if (not tweet["user"]["screen_name"] in self.twitter_excluded_tweeps): if self.twitter_unfollow: self.twitter.destroy_friendship(user_id=friend) self.message = self.userPushover.send_message( message=f'Flushed @' f'{tweet["user"]["screen_name"]} ' f'- {tweet["user"]["name"]}\n{tweetDate}\n' f'{tweet["text"]}', sound=self.pushover_sound) # Log a flush logging.info( f'Flushed @' f'{tweet["user"]["screen_name"]} ' f'- {tweet["user"]["name"]} - {tweetDate}') # trying not to upset the Twitter Gods sleep(2) except TwythonError as e: print(e)
print "Index ", str(i) print "Querying userId ", str(userid) followers = twitter.get_followers_ids(count=100, user_id=userid)['ids'] t = time.time() for f in followers: if not f in rels: rels[f] = [userid] elif userid not in rels[f]: rels[f].append(userid) print time.time() - t users = users + twitter.lookup_user(user_id=",".join(map(str, followers))) friends = twitter.get_friends_ids(count=100, user_id=userid)['ids'] t = time.time() for f in friends: if not userid in rels: rels[userid] = [f] elif f not in rels[userid]: rels[userid].append(f) print time.time() - t users = users + twitter.lookup_user(user_id=",".join(map(str, friends))) if i == m: usersToQuery = usersToQuery + followers[0:mid] + friends[0:mid] i = i-1
class RecolectorSiguiendoShort(Recolector): def __init__(self, escritores): super(RecolectorSiguiendoShort, self).__init__(escritores) self.authorizator = GetAuthorizations(15) self.twitter = None self.apoyo = ApoyoTwitter() self.tipo_id = 5 self.inicializa() self.cursor = -1 def inicializa(self): self.authorizator.load_twitter_token(self.tipo_id) api_key, access_token = self.authorizator.get_twython_token() self.twitter = Twython(api_key, access_token=access_token) def recolecta(self, query=None, id_user=-1, complete=False): self.cursor = -1 arrayUsuarios = [] if query is None and id_user == -1: raise Exception('Al menos debe haber un parametro usable') if query is not None: if query[0] == '@': query = query[1:] id_user = self.apoyo.getUserIDByScreenName(query) if id_user == None: raise Exception('El usuario debe estar en la base de datos') retornoFinal = {"ids": []} while True: retorno = self.privateRealizaConsultaById(id_user) if retorno == []: break retornoFinal["ids"].extend(retorno["ids"]) if len(retorno["ids"]) < 5000: break if complete == False: break #se almacenan las relaciones en una tupla (usuario1, usuario2) #esto quiere decir que el usuario1 sigue al usuario2. lista_relaciones = [] for identificador in retornoFinal["ids"]: lista_relaciones.append((id_user, identificador)) self.guarda(lista_relaciones) def guarda(self, arrayDatos): for escritor in self.escritores: escritor.escribe(arrayDatos) def privateRealizaConsultaById(self, identificador): if self.authorizator.is_limit_api(self.tipo_id): raise Exception('LIMITE') try: retorno = self.twitter.get_friends_ids(user_id=identificador, cursor=self.cursor, count='5000') self.authorizator.add_query_to_key(self.tipo_id) if len(retorno["ids"]) == 0: return [] return retorno except Exception, e: self.authorizator.add_query_to_key(self.tipo_id) print e if "429" in str(e): raise Exception('LIMITE') return []
newTweet = generate() twitter.update_status(status=newTweet) def administration(items): """ follow or reject new users who put in commands """ for tweet in reversed(items): if (tweet['user']['id'] == 22884755): text = tweet['text'][10:].split(" ") if (text[1] == "approve"): twitter.create_friendship(screen_name=text[2]) twitter.update_status( status="@" + text[2] + " Good news, you've been approved! Please retry any additions prior to this message again." ) elif (text[1] == "reject"): twitter.update_status( status="@" + text[2] + " Sorry, I'm not going to add you right now.") tweets = twitter.get_mentions_timeline() getLast() intake( simplify(twitter.get_mentions_timeline(), twitter.get_friends_ids()['ids'])) onDemand(tweets) administration(tweets) periodic()