class Twitter_Me: def __init__(self, Sinaweibo, logger=None): self.log = logger self.sw = Sinaweibo try: self.client = Twython(wb_cfg.TW_APP_KEY, wb_cfg.TW_APP_SECRET, \ wb_cfg.TW_AC_TOKEN, wb_cfg.TW_AC_TOKEN_SECRET) log_info(u"推特模块", u"User %s Login Successful!" % wb_cfg.TW_OWN_ID, self.log) except Exception: log_warn(u"推特模块", u"User %s Login Failed!" % self.TW_OWN_ID, self.log) def get_timeline(self): if self.client: return self.client.get('users/show', uid=wb_cfg.TW_OWN_ID) def get_home_timeline(self, cnt=200): if self.client: return self.client.get('statuses/home_timeline', params={'count': 200}) def post_statuses(self, str_msg): if self.client: self.client.post('statuses/update', status=str_msg, uid=self.UID) def repost_to_sina(self, Sinaweibo=None): if not self.sw: log_info(u"推特模块", u"Sinaweibo object is null, just return!") return statuses = self.get_home_timeline() if statuses: for item in statuses: if not item['user']['screen_name'] in TW_WHITE_LIST: log_info( u"推特模块", u"Not in the twitter repost_white list, skip [%d]!\n" % item['id'], self.log) continue elif item['user']['screen_name'] == "taozhijiang": log_info(u"推特模块", u"Recycle, Give up!\n") break else: try: ret = self.sw.post_statuses(str_msg=item['text']) except Exception, e: log_warn(u"推特模块", u"Runtime Error:%s" % e, self.log) continue log_info(u"推特模块", u"Repost item for [%d]!\n" % item['id'], self.log) #等待一会儿 time.sleep(random.randint(200, 400))
def drnj_profiles_crawler(ids): keystore = KeyStore() twitter = Twython(keystore.app_consumer_key, keystore.app_consumer_secret, keystore.access_tokens[0][0], keystore.access_tokens[0][1]) if len(ids) > 100: print "Can't retrieve more than 100 profiles" return # Number of calls to the twitter API remain = 0 # True if data is fetched correctly success = True # Seconds to wait before trying again twitter limit wait = 60 print "Retrieving the profiles of %d users\n" % len(ids) # Check if we still have some bandwidth available while remain <= 0: v = twitter.get('application/rate_limit_status', {"resources": 'users'}) remain = v["resources"]['users']["/users/lookup"]["remaining"] if remain > 0: break print "Waiting... Twitter API rate limit reached\n" time.sleep(wait) # First, try to get the profiles try: S = twitter.get('users/lookup', { 'user_id': ids, 'include_entities': True }) print '{} id' 's queried, {} retrieved '.format(len(ids), len(S)) post_data = { "user_id": json_encode(ids), "v": json_encode(S), "auth_user_id": auth_user_id, "auth_password": auth_password } post_response = requests.post(url=app_root_url + '/profiles/store', data=post_data) print "%s" % post_response.content except TwythonError as e: print e print "Error while fetching user profile from twitter, quitting ..." return
class Twitter_Me: def __init__(self, Sinaweibo, logger=None): self.log = logger self.sw = Sinaweibo try: self.client = Twython( wb_cfg.TW_APP_KEY, wb_cfg.TW_APP_SECRET, wb_cfg.TW_AC_TOKEN, wb_cfg.TW_AC_TOKEN_SECRET ) log_info(u"推特模块", u"User %s Login Successful!" % wb_cfg.TW_OWN_ID, self.log) except Exception: log_warn(u"推特模块", u"User %s Login Failed!" % self.TW_OWN_ID, self.log) def get_timeline(self): if self.client: return self.client.get("users/show", uid=wb_cfg.TW_OWN_ID) def get_home_timeline(self, cnt=200): if self.client: return self.client.get("statuses/home_timeline", params={"count": 200}) def post_statuses(self, str_msg): if self.client: self.client.post("statuses/update", status=str_msg, uid=self.UID) def repost_to_sina(self, Sinaweibo=None): if not self.sw: log_info(u"推特模块", u"Sinaweibo object is null, just return!") return statuses = self.get_home_timeline() if statuses: for item in statuses: if not item["user"]["screen_name"] in TW_WHITE_LIST: log_info(u"推特模块", u"Not in the twitter repost_white list, skip [%d]!\n" % item["id"], self.log) continue elif item["user"]["screen_name"] == "taozhijiang": log_info(u"推特模块", u"Recycle, Give up!\n") break else: try: ret = self.sw.post_statuses(str_msg=item["text"]) except Exception, e: log_warn(u"推特模块", u"Runtime Error:%s" % e, self.log) continue log_info(u"推特模块", u"Repost item for [%d]!\n" % item["id"], self.log) # 等待一会儿 time.sleep(random.randint(200, 400))
def search_hashtag(token_id, url): hashtag = '#' + request_helper.parse_get_param(url, '#') soc_token = SocToken.query.get(token_id) twitter = Twython(SocConfig.TWITTER_KEY, SocConfig.TWITTER_SECRET, soc_token.user_token, soc_token.token_secret) params = {'q': hashtag, 'count': 1, 'from': soc_token.soc_username} searchHashtag = twitter.get('search/tweets', params=params) return searchHashtag
def drnj_profiles_crawler(ids): keystore = KeyStore() twitter = Twython(keystore.app_consumer_key, keystore.app_consumer_secret, keystore.access_tokens[0][0], keystore.access_tokens[0][1]) if len(ids)>100: print "Can't retrieve more than 100 profiles" return # Number of calls to the twitter API remain = 0 # True if data is fetched correctly success = True # Seconds to wait before trying again twitter limit wait = 60; print "Retrieving the profiles of %d users\n" % len(ids) # Check if we still have some bandwidth available while remain<=0: v = twitter.get('application/rate_limit_status', {"resources": 'users'}) remain = v["resources"]['users']["/users/lookup"]["remaining"] if remain>0: break print "Waiting... Twitter API rate limit reached\n" time.sleep(wait) # First, try to get the profiles try: S = twitter.get('users/lookup', {'user_id': ids,'include_entities': True}) print '{} id''s queried, {} retrieved '.format(len(ids), len(S)) post_data = {"user_id": json_encode(ids), "v": json_encode(S), "auth_user_id": auth_user_id, "auth_password": auth_password} post_response = requests.post(url=app_root_url + '/profiles/store', data=post_data) print "%s" % post_response.content except TwythonError as e: print e print "Error while fetching user profile from twitter, quitting ..." return
def search_hashtag(token_id, url): hashtag = '#' + request_helper.parse_get_param(url, '#') soc_token = SocToken.query.get(token_id) twitter = Twython( SocConfig.TWITTER_KEY, SocConfig.TWITTER_SECRET, soc_token.user_token, soc_token.token_secret) params = { 'q': hashtag, 'count': 1, 'from': soc_token.soc_username} searchHashtag = twitter.get('search/tweets', params=params) return searchHashtag
def parse_tweets(consumer_key, consumer_secret, access_key, access_secret): """ (str, str, str, str) -> set of str Arguments are Twitter API credentials. Return set of str with codes from parsed tweets (if any are found). Otherwise stop script execution. """ try: twitter = Twython(consumer_key, consumer_secret, access_key, access_secret) except TwythonError as e: print(e.__class__.__name__) print( 'Can not access Twitter API. Check your API credentials, Internet connection and API Rate Limit.' ) sys.exit() # Search query to access tweets of CodesClub1909 # How query is built: https://dev.twitter.com/rest/public/search date = datetime.datetime.today().strftime( '%Y-%m-%d') # Codes expire in one day, so we have to use today's date query = 'https://api.twitter.com/1.1/search/tweets.json?' + \ 'q=from%3ACodesClub1909' + '%20%27code%27' + '%20since%3A' + date # Query results stored in json format. timeline = twitter.get(query) # Tweets that sum up codes for a day usually have words 'today' in their text. # We can use that to filter out irrelevant tweets. # We will also clean up tweet's text on the fly using clean_input function. if timeline['statuses']: tweets = [ clean_input(t['text']) for t in timeline['statuses'] if 'TODAY' in t['text'].upper() ] else: print( 'Query request returned no results. There might be no tweets in specified time frame.' ) sys.exit() # Store codes in set to avoid duplicates. if tweets: codes = set() for tweet in tweets: for word in tweet.split(): # Filter out exceptions and short words that can not be codes. if len(word) > 3 and word.upper() not in EXCEPTIONS: codes.add( word.upper() ) # Ensure that code is in upper case while adding it to set return codes else: print('"CODES ARE" tweets not found. No codes to collect.') sys.exit()
def download_tweets( fetch_list, raw_dir ): # ensure raw data directory exists if not os.path.exists( raw_dir ): os.mkdir( raw_dir ) # stay within rate limits, 180 tweets per 15 mins, + some grace max_tweets_per_hr = 175*4 download_pause_sec = 3600 / max_tweets_per_hr # Requires Authentication as of Twitter API v1.1 twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) # download tweets for idx in range(0,len(fetch_list)): # current item item = fetch_list[idx] # print status trem = get_time_left_str( idx, fetch_list, download_pause_sec ) print '--> downloading tweet #%s (%d of %d) (%s left)' % \ (item[2], idx+1, len(fetch_list), trem) # pull data url = 'https://api.twitter.com/1.1/statuses/show.json?id=' + item[2] #urllib.urlretrieve( url, raw_dir + item[2] + '.json' ) try: data = twitter.get(url) f = open( raw_dir + item[2] + '.json' , 'w') sjson.dump(data,f) f.close() except TwythonError as e: print e # stay in Twitter API rate limits print ' pausing %d sec to obey Twitter API rate limits' % \ (download_pause_sec) time.sleep( download_pause_sec ) return
def get_oauth_dict(self, access_token): """ Uses the access token to find the LinkedIn user object, then returns it and the account uid. """ if not access_token or 'token_rejected' in access_token: raise SocialOauthDictFailed client = Twython(app_key=self.twitter_ck, app_secret=self.twitter_cs, **self.parse_oauth_token(access_token)) oauth_object = client.get('users/show', params={'user_id': access_token['user_id']}) if oauth_object: return oauth_object, str(access_token['user_id']) else: raise SocialOauthDictFailed
def get_oauth_dict(self, access_token): """ Uses the access token to find the LinkedIn user object, then returns it and the account uid. """ if not access_token or 'token_rejected' in access_token: raise SocialOauthDictFailed client = Twython( app_key=self.twitter_ck, app_secret=self.twitter_cs, **self.parse_oauth_token(access_token)) oauth_object = client.get('users/show', params={ 'user_id': access_token['user_id'] }) if oauth_object: return oauth_object, unicode(access_token['user_id']) else: raise SocialOauthDictFailed
def do_search(search_id, url): ''' oauth_token and oauth_token_secret come from the previous step if needed, store those in a session variable or something ''' print('searching url: ' + url) t = Twython(app_key="yMNdpqYz5ke32Z6jCZsE7w", app_secret="5oDqxINNVOH1CHDJdUp2Mz3nwEgbBGeczJufD957S2k", oauth_token="15439239-rJspSTXhfu4hnJ7MyT4iJUagmsg8I3HV9zsTTPs", oauth_token_secret="nUeFkU3HVOXsVUFNJPljGbDvqF0apKimHvXFQ3dvNA") db = dbaccess.DB() regex = db.service_regex(__SERVICE__) terms = re.match(regex, url).groups(1) raw = t.get(endpoint="https://api.twitter.com/1.1/search/tweets.json", params={'q':terms, 'count':100, 'result_type':'recent'}) for img in processImageArray(raw): savePhoto(search_id, img) db.close()
def check_conf(args): ''' Using info from conf file, check JSS connection, check API privileges, and check Twitter Auth''' logger = logging.getLogger('capd') logger.debug("check_conf") config = ConfigParser.ConfigParser() config.read(Software_Repo+'/conf/JSS_Server.conf') #### check JSS connection #### logger.info("[+] Checking JSS Connection ...") try: r = requests.get(config.get('JSS_Server', 'jss_url'), timeout=10, verify=False) logger.info("[+] JSS connection is OK") except requests.exceptions.RequestException as e: logger.error("[-] JSS Server problem with following error: %s", e) sys.exit(1) #### check API Privileges #### logger.info("[+] Checking API Privileges ...") api_user_permissions = [] try: url_api = config.get('JSS_Server', 'url_api') api_user = config.get('JSS_Server', 'api_user') api_pass = config.get('JSS_Server', 'api_pass') r = requests.get(url_api+'accounts/username/'+api_user, auth=(api_user, api_pass), verify=False, headers=headers) except requests.exceptions.RequestException as e: logger.error("[-] JSS Server problem with following error: %s", e) sys.exit(1) tree = ET.fromstring(r.content) for elem in tree.iterfind('./privileges/jss_objects/'): api_user_permissions.append(elem.text) if not list(set(api_account_permissions) - set(api_user_permissions)): logger.info("[+] API Privilegs OK") else: logger.error("[-] You appear to be missing the following API privilege(s): %s", list(set(api_account_permissions) - set(api_user_permissions))) sys.exit(1) #### check Twitter Auth #### logger.info("[+] Checking Twitter Auth ...") try: app_key = config.get('Twitter_Auth', 'twitter_app_key') if not app_key: logger.info("[-] No Twitter App Key provided!") return app_secret = config.get('Twitter_Auth', 'twitter_app_secret') oauth_token = config.get('Twitter_Auth', 'twitter_oauth_token') oauth_token_secret = config.get('Twitter_Auth', 'twitter_oauth_token_secret') twitter = Twython(app_key, app_secret, oauth_token, oauth_token_secret) twitter.get("https://api.twitter.com/1.1/account/verify_credentials.json") logger.info("[+] Twitter Auth OK") except TwythonError as e: logger.error("[-] Check twitter oauth credentials. %s", e) sys.exit(1) #### check Mail Server #### logger.info("Checking connection to Mailserver ...") mailserver = config.get('Mail', 'mailserver') if not mailserver: logger.info("[-] No Mailserver configured!") return try: connect_to_mailserver = SMTP(mailserver, timeout=10) mailserver_status, mailserver_message = connect_to_mailserver.helo() if mailserver_status == 250: logger.info("[+] Mailserver connection OK") except socket.error as e: logger.error("[-] Mailserver connection error: %s", e) sys.exit(1) except SMTPException as e: logger.error("[-] Mailserver connection error: %s", e) sys.exit(1) #### clear apps and screenshots folder #### logger.info("[+] Cleaning out apps folder ...") shutil.rmtree(Software_Repo+'/apps/') shutil.rmtree(Software_Repo+'/logs/screenshots/') os.mkdir(Software_Repo+'/apps') os.mkdir(Software_Repo+'/logs/screenshots')
import numpy as np from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.decomposition import NMF oc_twitter_handle = 'mitsap' # Load credentials from json file with open("../twitter_credentials.json", "r") as file: creds = json.load(file) # Instantiate an object twitter = Twython(creds['CONSUMER_KEY'], creds['CONSUMER_SECRET']) # Get the followers list # TO DO cursor returned user IDs follower_request = twitter.get(endpoint='https://api.twitter.com/1.1/followers/ids.json', params={'screen_name':oc_twitter_handle,'count':5000}) follower_list=follower_request['ids'] # TO DO Check if user name already in database, skip for index, follower in enumerate(follower_list): # TO DO can only query 900 times in 15 minute window. 100,000 times per 24 hours # TO DO reduce amount of data stored -- lots of columns we might not need returned = twitter.get(endpoint='https://api.twitter.com/1.1/statuses/user_timeline.json', params={'user_id':follower,'count':200, 'exclude_replies':True, 'include_rts':False}) if (index==0): tweets_df = pd.DataFrame(returned) tweets_df['user'] = follower else: temp_df = pd.DataFrame(returned) temp_df['user'] = follower tweets_df = tweets_df.append(temp_df)
class TwitterUserStream(TwythonStreamer): """Stream tweets from a user's Twitter feed. Whenever a tweet is received on the stream, each function in callbacks will be called with the Twitter response data as the argument. Note that the Twitter API may deliver messages out of order and may deliver repeat messages. Use the 'show_dupes' property to control whether duplicate messages are reported to the callbacks. The 'store' property controls how many of the most recent messages to store.""" def __init__(self, user=None, callbacks=[], show_dupes=False, store=10): self.data = Storage() super(TwitterUserStream, self).__init__(*self.data.twitter) self.twitter = Twython(*self.data.twitter) self.error = "" # text of last error self._errors = 0 self._max_errors = 5 self._show_dupes = show_dupes self._follow = user self._lastn = list() self._store = store # max size of _lastn self._callbacks = list() self.add_all(callbacks) def __getattr__(self, attr): if attr == 'show_dupes': return self._show_dupes elif attr == 'store': return self._store raise AttributeError def __setattr__(self, attr, value): # show_dupes (bool): whether to report duplicate messages to the callbacks if attr == 'show_dupes': self._show_dupes = bool(value) # store (int): how many tweets to store elif attr == 'store': if isinstance(value, int): # clamp to 0 if value < 0: value = 0 # truncate stored messages if we are reducing the count if value < self._store: self._lastn = self._lastn[:value] self._store = value else: raise TypeError("type of argument 'store' must be integral") else: super(TwitterUserStream, self).__setattr__(attr, value) # Format of the 'created_at' timestamp returned from Twitter # E.g. 'Wed Jan 20 06:35:02 +0000 2016' TIME_FMT = """%a %b %d %H:%M:%S %z %Y""" @staticmethod def timeof(tweet): """Return the 'created_at' time for the given tween as a datetime object.""" return datetime.strptime(tweet['created_at'], TwitterUserStream.TIME_FMT) def _remember(self, tweet): """Remember a new tweet, dropping the oldest one if we have reached our limit on 'store'. Keeps tweets in chronological order. See latest().""" if len(self._lastn) >= self.store: self._lastn.pop(0) # Maintain chronological sorting on insertion (using binary search) i = _bisect_left(self._lastn, tweet, key=self.timeof) self._lastn.insert(i, tweet) def latest(self): """Return the last several tweets by the last-followed user, in order of actual occurrence time. The number of messages stored is limited by the 'store' property.""" return iter(self._lastn) def last(self): """Return the last tweet which was received. Only valid if 'store' is greater than zero. Otherwise returns an empty dictionary.""" if len(self._lastn) > 0: return self._lastn[-1] else: return dict() def stored(self): """Return the number of tweets currently stored in latest().""" return len(self._lastn) def follow_thread(self, *args, **kwargs): """Convenient wrapper for calling follow() in a background thread. The Thread object is started and then returned. Passes on args and kwargs to the follow() method.""" thread = threading.Thread(target=self.follow, args=args, kwargs=kwargs) thread.daemon = True thread.start() return thread def get_user(self, user): """Return the user ID of a Twitter user as a string given his screen name, or None if the user is invalid.""" try: result = self.twitter.get("users/show", params={'screen_name': user}) return result['id_str'] except TwythonError as e: self.on_error(e.error_code, e.msg) return None def follow(self, user=None, get_last=True): """Start streaming tweents from a user. This method will block basically forever, so running it in a Thread is a good idea. If user is None, use the user given on construction. If get_last is True, fetch the user's last tweets before streaming. The number of tweets prefetched depends on the 'store' property. Note that any registered callbacks will NOT be called on these pre-existing tweets. Use latest() to see the prefetched tweets.""" user = user or self.follow_user if user is None: print('No user specified.') return False # Fetch the ID of the user by screen name uid = self.get_user(user) # Fill up the last 'store' tweets if get_last is set if get_last: result = self.twitter.get("statuses/user_timeline", params={ 'user_id': uid, 'count': str(self.store) }) # Results are returned in reverse-chronological order result.reverse() self._lastn.extend(result) # Follow the given user self.statuses.filter(follow=uid) return True def add(self, callback): """Register a function to be called when tweets are received. Returns the TwitterUserStream object. Example: t = TwitterUserStream(...) t.add(myfunc).follow("Bob") """ self._callbacks.append(callback) return self def add_all(self, callbacks): """Register several functions to be called when tweets are received. Returns the TwitterUserStream object. Example: t = TwitterUserStream(...) t.add_all([myfunc1, myfunc2]).follow("Bob") """ self._callbacks.extend(callbacks) def remove(self, callback): """Unregister the function if it is currently a callback. Returns the TwitterUserStream object.""" if callback in self._callbacks: self._callbacks.remove(callback) def remove_all(self, callbacks): """Unregister several functions if they are registered. Returns the TwitterUserStream object.""" for callback in callbacks: self.remove(callback) def _filter_tweet(self, data): """Return True if the given tweet should be passed on to the callbacks, False otherwise.""" # If we don't have an ID, this isn't valid if 'id_str' not in data: return False # Ignore replies, quotes, and retweets if (data.get('in_reply_to_status_id_str') or data.get('quoted_statis_id_str') or data.get('retweeted_status')): return False # If show_dupes is off, ignore duplicate tweets if not self.show_dupes: for tweet in self._lastn: if data['id_str'] == tweet['id_str']: return return True def on_success(self, data): """Called by TwythonStreamer when a message is received on the underlying stream. Dispatches the message to all registered callbacks (in the order they were registered) if the message is not a duplicate or show_dupes is enabled.""" # Make sure this is a tweet we are interested in if not self._filter_tweet(data): return # Remember this message - if we reach our store limit, pop the oldest self._remember(data) # Notify callbacks for callback in self._callbacks: callback(data) def on_error(self, code, data): """Called when there is an error. Disconnects from the stream after receiving too many errors. Sets the 'error' attribute to an appropriate error message.""" errmsg = ("Twitter Error: [{0}] {1}".format(code, data)) print(errmsg) self.error = errmsg self._errors += 1 if self._errors >= self._max_errors: print("Maximum number of errors exceeded, disconnecting...") self.disconnect()
class TwythonAPITestCase(unittest.TestCase): def setUp(self): self.api = Twython('', '', '', '') def get_url(self, endpoint): """Convenience function for mapping from endpoint to URL""" return '%s/%s.json' % (self.api.api_url % self.api.api_version, endpoint) def register_response(self, method, url, body='{}', match_querystring=False, status=200, adding_headers=None, stream=False, content_type='application/json; charset=utf-8'): """Wrapper function for responses for simpler unit tests""" # responses uses BytesIO to hold the body so it needs to be in bytes if not is_py2: body = bytes(body, 'UTF-8') responses.add(method, url, body, match_querystring, status, adding_headers, stream, content_type) @responses.activate def test_request_should_handle_full_endpoint(self): """Test that request() accepts a full URL for the endpoint argument""" url = 'https://api.twitter.com/1.1/search/tweets.json' self.register_response(responses.GET, url) self.api.request(url) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_handle_relative_endpoint(self): """Test that request() accepts a twitter endpoint name for the endpoint argument""" url = 'https://api.twitter.com/1.1/search/tweets.json' self.register_response(responses.GET, url) self.api.request('search/tweets', version='1.1') self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_post_request_regardless_of_case(self): """Test that request() accepts the HTTP method name regardless of case""" url = 'https://api.twitter.com/1.1/statuses/update.json' self.register_response(responses.POST, url) self.api.request(url, method='POST') self.api.request(url, method='post') self.assertEqual(2, len(responses.calls)) self.assertEqual('POST', responses.calls[0].request.method) self.assertEqual('POST', responses.calls[1].request.method) @responses.activate def test_request_should_throw_exception_with_invalid_http_method(self): """Test that request() throws an exception when an invalid HTTP method is passed""" #TODO(cash): should Twython catch the AttributeError and throw a TwythonError self.assertRaises(AttributeError, self.api.request, endpoint='search/tweets', method='INVALID') @responses.activate def test_request_should_encode_boolean_as_lowercase_string(self): """Test that request() encodes a boolean parameter as a lowercase string""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'include_entities': True}) self.api.request(endpoint, params={'include_entities': False}) self.assertEqual(url + '?include_entities=true', responses.calls[0].request.url) self.assertEqual(url + '?include_entities=false', responses.calls[1].request.url) @responses.activate def test_request_should_handle_string_or_number_parameter(self): """Test that request() encodes a numeric or string parameter correctly""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'lang': 'es'}) self.api.request(endpoint, params={'count': 50}) self.assertEqual(url + '?lang=es', responses.calls[0].request.url) self.assertEqual(url + '?count=50', responses.calls[1].request.url) @responses.activate def test_request_should_encode_list_of_strings_as_string(self): """Test that request() encodes a list of strings as a comma-separated string""" endpoint = 'search/tweets' url = self.get_url(endpoint) location = ['37.781157', '-122.39872', '1mi'] self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': location}) # requests url encodes the parameters so , is %2C self.assertEqual(url + '?geocode=37.781157%2C-122.39872%2C1mi', responses.calls[0].request.url) @responses.activate def test_request_should_encode_numeric_list_as_string(self): """Test that request() encodes a list of numbers as a comma-separated string""" endpoint = 'search/tweets' url = self.get_url(endpoint) location = [37.781157, -122.39872, '1mi'] self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': location}) self.assertEqual(url + '?geocode=37.781157%2C-122.39872%2C1mi', responses.calls[0].request.url) @responses.activate def test_request_should_ignore_bad_parameter(self): """Test that request() ignores unexpected parameter types""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': self}) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_handle_file_as_parameter(self): """Test that request() pulls a file out of params for requests lib""" endpoint = 'account/update_profile_image' url = self.get_url(endpoint) self.register_response(responses.POST, url) mock_file = StringIO("Twython test image") self.api.request(endpoint, method='POST', params={'image': mock_file}) self.assertIn(b'filename="image"', responses.calls[0].request.body) self.assertIn(b"Twython test image", responses.calls[0].request.body) @responses.activate def test_request_should_put_params_in_body_when_post(self): """Test that request() passes params as data when the request is a POST""" endpoint = 'statuses/update' url = self.get_url(endpoint) self.register_response(responses.POST, url) self.api.request(endpoint, method='POST', params={'status': 'this is a test'}) self.assertIn(b'status=this+is+a+test', responses.calls[0].request.body) self.assertNotIn('status=this+is+a+test', responses.calls[0].request.url) @responses.activate def test_get_uses_get_method(self): """Test Twython generic GET request works""" endpoint = 'account/verify_credentials' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.get(endpoint) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_post_uses_post_method(self): """Test Twython generic POST request works""" endpoint = 'statuses/update' url = self.get_url(endpoint) self.register_response(responses.POST, url) self.api.post(endpoint, params={'status': 'I love Twython!'}) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) def test_raise_twython_error_on_request_exception(self): """Test if TwythonError is raised by a RequestException""" with mock.patch.object(requests.Session, 'get') as get_mock: # mocking an ssl cert error get_mock.side_effect = requests.RequestException("hostname 'example.com' doesn't match ...") self.assertRaises(TwythonError, self.api.get, 'https://example.com') @responses.activate def test_request_should_get_convert_json_to_data(self): """Test that Twython converts JSON data to a Python object""" endpoint = 'statuses/show' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"id": 210462857140252672}') data = self.api.request(endpoint, params={'id': 210462857140252672}) self.assertEqual({'id': 210462857140252672}, data) @responses.activate def test_request_should_raise_exception_with_invalid_json(self): """Test that Twython handles invalid JSON (though Twitter should not return it)""" endpoint = 'statuses/show' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"id: 210462857140252672}') self.assertRaises(TwythonError, self.api.request, endpoint, params={'id': 210462857140252672}) @responses.activate def test_request_should_handle_401(self): """Test that Twython raises an auth error on 401 error""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Error"}]}', status=401) self.assertRaises(TwythonAuthError, self.api.request, endpoint) @responses.activate def test_request_should_handle_400_for_missing_auth_data(self): """Test that Twython raises an auth error on 400 error when no oauth data sent""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Bad Authentication data"}]}', status=400) self.assertRaises(TwythonAuthError, self.api.request, endpoint) @responses.activate def test_request_should_handle_400_that_is_not_auth_related(self): """Test that Twython raises a normal error on 400 error when unrelated to authorization""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Bad request"}]}', status=400) self.assertRaises(TwythonError, self.api.request, endpoint) @responses.activate def test_request_should_handle_rate_limit(self): """Test that Twython raises an rate limit error on 429""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Rate Limit"}]}', status=429) self.assertRaises(TwythonRateLimitError, self.api.request, endpoint) @responses.activate def test_get_lastfunction_header_should_return_header(self): """Test getting last specific header of the last API call works""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, adding_headers={'x-rate-limit-remaining': 37}) self.api.get(endpoint) value = self.api.get_lastfunction_header('x-rate-limit-remaining') self.assertEqual(37, value) value2 = self.api.get_lastfunction_header('does-not-exist') self.assertIsNone(value2) value3 = self.api.get_lastfunction_header('not-there-either', 96) self.assertEqual(96, value3) def test_get_lastfunction_header_should_raise_error_when_no_previous_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') @responses.activate def test_sends_correct_accept_encoding_header(self): """Test that Twython accepts compressed data.""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.get(endpoint) self.assertEqual(b'gzip, deflate, compress', responses.calls[0].request.headers['Accept-Encoding']) # Static methods def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR expanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text)
class TwythonAPITestCase(unittest.TestCase): def setUp(self): self.api = Twython('', '', '', '') def get_url(self, endpoint): """Convenience function for mapping from endpoint to URL""" return '%s/%s.json' % (self.api.api_url % self.api.api_version, endpoint) def register_response(self, method, url, body='{}', match_querystring=False, status=200, adding_headers=None, stream=False, content_type='application/json; charset=utf-8'): """Wrapper function for responses for simpler unit tests""" # responses uses BytesIO to hold the body so it needs to be in bytes if not is_py2: body = bytes(body, 'UTF-8') responses.add(method, url, body, match_querystring, status, adding_headers, stream, content_type) @responses.activate def test_request_should_handle_full_endpoint(self): """Test that request() accepts a full URL for the endpoint argument""" url = 'https://api.twitter.com/1.1/search/tweets.json' self.register_response(responses.GET, url) self.api.request(url) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_handle_relative_endpoint(self): """Test that request() accepts a twitter endpoint name for the endpoint argument""" url = 'https://api.twitter.com/1.1/search/tweets.json' self.register_response(responses.GET, url) self.api.request('search/tweets', version='1.1') self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_post_request_regardless_of_case(self): """Test that request() accepts the HTTP method name regardless of case""" url = 'https://api.twitter.com/1.1/statuses/update.json' self.register_response(responses.POST, url) self.api.request(url, method='POST') self.api.request(url, method='post') self.assertEqual(2, len(responses.calls)) self.assertEqual('POST', responses.calls[0].request.method) self.assertEqual('POST', responses.calls[1].request.method) @responses.activate def test_request_should_throw_exception_with_invalid_http_method(self): """Test that request() throws an exception when an invalid HTTP method is passed""" # TODO(cash): should Twython catch the AttributeError and throw a TwythonError self.assertRaises(AttributeError, self.api.request, endpoint='search/tweets', method='INVALID') @responses.activate def test_request_should_encode_boolean_as_lowercase_string(self): """Test that request() encodes a boolean parameter as a lowercase string""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'include_entities': True}) self.api.request(endpoint, params={'include_entities': False}) self.assertEqual(url + '?include_entities=true', responses.calls[0].request.url) self.assertEqual(url + '?include_entities=false', responses.calls[1].request.url) @responses.activate def test_request_should_handle_string_or_number_parameter(self): """Test that request() encodes a numeric or string parameter correctly""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'lang': 'es'}) self.api.request(endpoint, params={'count': 50}) self.assertEqual(url + '?lang=es', responses.calls[0].request.url) self.assertEqual(url + '?count=50', responses.calls[1].request.url) @responses.activate def test_request_should_encode_list_of_strings_as_string(self): """Test that request() encodes a list of strings as a comma-separated string""" endpoint = 'search/tweets' url = self.get_url(endpoint) location = ['37.781157', '-122.39872', '1mi'] self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': location}) # requests url encodes the parameters so , is %2C self.assertEqual(url + '?geocode=37.781157%2C-122.39872%2C1mi', responses.calls[0].request.url) @responses.activate def test_request_should_encode_numeric_list_as_string(self): """Test that request() encodes a list of numbers as a comma-separated string""" endpoint = 'search/tweets' url = self.get_url(endpoint) location = [37.781157, -122.39872, '1mi'] self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': location}) self.assertEqual(url + '?geocode=37.781157%2C-122.39872%2C1mi', responses.calls[0].request.url) @responses.activate def test_request_should_ignore_bad_parameter(self): """Test that request() ignores unexpected parameter types""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': self}) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_handle_file_as_parameter(self): """Test that request() pulls a file out of params for requests lib""" endpoint = 'account/update_profile_image' url = self.get_url(endpoint) self.register_response(responses.POST, url) mock_file = StringIO("Twython test image") self.api.request(endpoint, method='POST', params={'image': mock_file}) self.assertIn(b'filename="image"', responses.calls[0].request.body) self.assertIn(b"Twython test image", responses.calls[0].request.body) @responses.activate def test_request_should_put_params_in_body_when_post(self): """Test that request() passes params as data when the request is a POST""" endpoint = 'statuses/update' url = self.get_url(endpoint) self.register_response(responses.POST, url) self.api.request(endpoint, method='POST', params={'status': 'this is a test'}) self.assertIn(b'status=this+is+a+test', responses.calls[0].request.body) self.assertNotIn('status=this+is+a+test', responses.calls[0].request.url) @responses.activate def test_get_uses_get_method(self): """Test Twython generic GET request works""" endpoint = 'account/verify_credentials' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.get(endpoint) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_post_uses_post_method(self): """Test Twython generic POST request works""" endpoint = 'statuses/update' url = self.get_url(endpoint) self.register_response(responses.POST, url) self.api.post(endpoint, params={'status': 'I love Twython!'}) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) def test_raise_twython_error_on_request_exception(self): """Test if TwythonError is raised by a RequestException""" with mock.patch.object(requests.Session, 'get') as get_mock: # mocking an ssl cert error get_mock.side_effect = requests.RequestException("hostname 'example.com' doesn't match ...") self.assertRaises(TwythonError, self.api.get, 'https://example.com') @responses.activate def test_request_should_get_convert_json_to_data(self): """Test that Twython converts JSON data to a Python object""" endpoint = 'statuses/show' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"id": 210462857140252672}') data = self.api.request(endpoint, params={'id': 210462857140252672}) self.assertEqual({'id': 210462857140252672}, data) @responses.activate def test_request_should_raise_exception_with_invalid_json(self): """Test that Twython handles invalid JSON (though Twitter should not return it)""" endpoint = 'statuses/show' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"id: 210462857140252672}') self.assertRaises(TwythonError, self.api.request, endpoint, params={'id': 210462857140252672}) @responses.activate def test_request_should_handle_401(self): """Test that Twython raises an auth error on 401 error""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Error"}]}', status=401) self.assertRaises(TwythonAuthError, self.api.request, endpoint) @responses.activate def test_request_should_handle_400_for_missing_auth_data(self): """Test that Twython raises an auth error on 400 error when no oauth data sent""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Bad Authentication data"}]}', status=400) self.assertRaises(TwythonAuthError, self.api.request, endpoint) @responses.activate def test_request_should_handle_400_that_is_not_auth_related(self): """Test that Twython raises a normal error on 400 error when unrelated to authorization""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Bad request"}]}', status=400) self.assertRaises(TwythonError, self.api.request, endpoint) @responses.activate def test_request_should_handle_rate_limit(self): """Test that Twython raises an rate limit error on 429""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Rate Limit"}]}', status=429) self.assertRaises(TwythonRateLimitError, self.api.request, endpoint) @responses.activate def test_get_lastfunction_header_should_return_header(self): """Test getting last specific header of the last API call works""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, adding_headers={'x-rate-limit-remaining': 37}) self.api.get(endpoint) value = self.api.get_lastfunction_header('x-rate-limit-remaining') self.assertEqual(37, value) value2 = self.api.get_lastfunction_header('does-not-exist') self.assertIsNone(value2) value3 = self.api.get_lastfunction_header('not-there-either', 96) self.assertEqual(96, value3) def test_get_lastfunction_header_should_raise_error_when_no_previous_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') @responses.activate def test_sends_correct_accept_encoding_header(self): """Test that Twython accepts compressed data.""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.get(endpoint) self.assertEqual(b'gzip, deflate, compress', responses.calls[0].request.headers['Accept-Encoding']) # Static methods def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR expanded url self.assertTrue('http://google.com' not in tweet_text) self.assertTrue('google.com' not in tweet_text)
twitter = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET) # try: # file = open("ACCESS_TOKEN.txt") # ACCESS_TOKEN = file.read() # except IOError: # file = open("ACCESS_TOKEN.txt", "w") # ACCESS_TOKEN = twitter.obtain_access_token() # file.write(ACCESS_TOKEN) # finally: # file.close() #twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN) results = twitter.get("search/tweets", params={'geocode': '37.874718,-122.272539,3mi', 'count':'100', 'result_type':'recent'}) id = results['statuses'][len(results['statuses']) - 1]['id'] twitter.verify_credentials() timeline = twitter.get_home_timeline(count = 20) content = [] geo = [] michaels_tweets = twitter.get("statuses/user_timeline", params = {'count': '200', 'screen_name':'cycomachead'}) days = [item['created_at'][0:3] for item in michaels_tweets] hist = simplehist.SimpleHist(days) summary = hist.getHist() print "Summary of Michael's Last 200 Tweets" for day in calendar.weekheader(3).split(" "):
twitter_stuff_rx = re.compile(r"RT |@\S+|(?:#\S+\s*)*$|http://\S+") twitter_hash_noun_rx = re.compile(r"#\S+") def fixTwitterStuff(text): out = twitter_stuff_rx.sub("", text) #out = twitter_hash_noun_rx.sub(lambda m: m.group(0) if isAWord(m.group(0)) else "", out) return out npages_to_use = 15 texts = [] next_results = None for page_num in range(npages_to_use): if next_results is None: response = twyt.search(q=hashtag, lang="en", count=100) else: response = twyt.get('https://api.twitter.com/1.1/search/tweets.json' + next_results) statuses = response['statuses'] for status in statuses: texts.append(fixTwitterStuff(status['text'])) try: next_results = response['search_metadata']['next_results'] except KeyError: break # ============================================================================== def stripStresses(pron): out = pron[:] for i, w in enumerate(out):
class TwythonAPITestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': { } # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual( constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_get(self): """Test Twython generic GET request works""" self.api.get('account/verify_credentials') def test_post(self): """Test Twython generic POST request works, with a full url and with just an endpoint""" update_url = 'https://api.twitter.com/1.1/statuses/update.json' status = self.api.post( update_url, params={'status': 'I love Twython! %s' % int(time.time())}) self.api.post('statuses/destroy/%s' % status['id_str']) def test_get_lastfunction_header(self): """Test getting last specific header of the last API call works""" self.api.get('statuses/home_timeline') self.api.get_lastfunction_header('x-rate-limit-remaining') def test_get_lastfunction_header_not_present(self): """Test getting specific header that does not exist from the last call returns None""" self.api.get('statuses/home_timeline') header = self.api.get_lastfunction_header('does-not-exist') self.assertEqual(header, None) def test_get_lastfunction_header_no_last_api_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') def test_cursor(self): """Test looping through the generator results works, at least once that is""" search = self.api.cursor(self.api.search, q='twitter', count=1) counter = 0 while counter < 2: counter += 1 result = next(search) new_id_str = int(result['id_str']) if counter == 1: prev_id_str = new_id_str time.sleep( 1) # Give time for another tweet to come into search if counter == 2: self.assertTrue(new_id_str > prev_id_str) def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR exapanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text) def test_raise_error_on_bad_ssl_cert(self): """Test TwythonError is raised by a RequestException when an actual HTTP happens""" self.assertRaises(TwythonError, self.api.get, 'https://example.com') # Timelines def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline( screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status( status='Test post just to get deleted :( %s' % int(time.time())) self.api.destroy_status(id=status['id_str']) def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man! %s' % int(time.time())) # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff %s' % int(time.time())) list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed %s' % int(time.time())) screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_memberships(self): """Test list of memberhips the authenticated user succeeds""" self.api.get_list_memberships() def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) # Returns 404 if user is not a subscriber self.api.is_list_subscriber( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name=screen_name) self.api.unsubscribe_from_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(slug=test_list_slug, owner_screen_name=test_list_owner_screen_name, screen_name='themattharris') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list( slug=test_list_slug, owner_screen_name=test_list_owner_screen_name) def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help def test_get_twitter_configuration(self): """Test getting Twitter's configuration succeeds""" self.api.get_twitter_configuration() def test_get_supported_languages(self): """Test getting languages supported by Twitter succeeds""" self.api.get_supported_languages() def test_privacy_policy(self): """Test getting Twitter's Privacy Policy succeeds""" self.api.get_privacy_policy() def test_get_tos(self): """Test getting the Twitter Terms of Service succeeds""" self.api.get_tos() def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()
#!/usr/bin/python3 from twython import Twython import requests consumer_key = 'XXXXX' consumer_secret = 'XXXXX' access_token = 'XXXXX' access_token_secret = 'XXXXX' my_screen_name = 'aoemon1' twitter = Twython(consumer_key, consumer_secret, access_token, access_token_secret) if __name__ == "__main__": # stream = MyStreamer(consumer_key, consumer_secret, access_token, access_token_secret) # stream.user() # ret = twitter.verify_credentials() ret = twitter.get('direct_messages/events/list') print(ret)
class TwitterUserStream(TwythonStreamer): """Stream tweets from a user's Twitter feed. Whenever a tweet is received on the stream, each function in callbacks will be called with the Twitter response data as the argument. Note that the Twitter API may deliver messages out of order and may deliver repeat messages. Use the 'show_dupes' property to control whether duplicate messages are reported to the callbacks. The 'store' property controls how many of the most recent messages to store.""" def __init__(self, user=None, callbacks=[], show_dupes=False, store=10): self.data = Storage() super(TwitterUserStream, self).__init__(*self.data.twitter) self.twitter = Twython(*self.data.twitter) self.error = "" # text of last error self._errors = 0 self._max_errors = 5 self._show_dupes = show_dupes self._follow = user self._lastn = list() self._store = store # max size of _lastn self._callbacks = list() self.add_all(callbacks) def __getattr__(self, attr): if attr == 'show_dupes': return self._show_dupes elif attr == 'store': return self._store raise AttributeError def __setattr__(self, attr, value): # show_dupes (bool): whether to report duplicate messages to the callbacks if attr == 'show_dupes': self._show_dupes = bool(value) # store (int): how many tweets to store elif attr == 'store': if isinstance(value, int): # clamp to 0 if value < 0: value = 0 # truncate stored messages if we are reducing the count if value < self._store: self._lastn = self._lastn[:value] self._store = value else: raise TypeError("type of argument 'store' must be integral") else: super(TwitterUserStream, self).__setattr__(attr, value) # Format of the 'created_at' timestamp returned from Twitter # E.g. 'Wed Jan 20 06:35:02 +0000 2016' TIME_FMT = """%a %b %d %H:%M:%S %z %Y""" @staticmethod def timeof(tweet): """Return the 'created_at' time for the given tween as a datetime object.""" return datetime.strptime(tweet['created_at'], TwitterUserStream.TIME_FMT) def _remember(self, tweet): """Remember a new tweet, dropping the oldest one if we have reached our limit on 'store'. Keeps tweets in chronological order. See latest().""" if len(self._lastn) >= self.store: self._lastn.pop(0) # Maintain chronological sorting on insertion (using binary search) i = _bisect_left(self._lastn, tweet, key=self.timeof) self._lastn.insert(i, tweet) def latest(self): """Return the last several tweets by the last-followed user, in order of actual occurrence time. The number of messages stored is limited by the 'store' property.""" return iter(self._lastn) def last(self): """Return the last tweet which was received. Only valid if 'store' is greater than zero. Otherwise returns an empty dictionary.""" if len(self._lastn) > 0: return self._lastn[-1] else: return dict() def stored(self): """Return the number of tweets currently stored in latest().""" return len(self._lastn) def follow_thread(self, *args, **kwargs): """Convenient wrapper for calling follow() in a background thread. The Thread object is started and then returned. Passes on args and kwargs to the follow() method.""" thread = threading.Thread(target=self.follow, args=args, kwargs=kwargs) thread.daemon = True thread.start() return thread def get_user(self, user): """Return the user ID of a Twitter user as a string given his screen name, or None if the user is invalid.""" try: result = self.twitter.get("users/show", params={'screen_name':user}) return result['id_str'] except TwythonError as e: self.on_error(e.error_code, e.msg) return None def follow(self, user=None, get_last=True): """Start streaming tweents from a user. This method will block basically forever, so running it in a Thread is a good idea. If user is None, use the user given on construction. If get_last is True, fetch the user's last tweets before streaming. The number of tweets prefetched depends on the 'store' property. Note that any registered callbacks will NOT be called on these pre-existing tweets. Use latest() to see the prefetched tweets.""" user = user or self.follow_user if user is None: print('No user specified.') return False # Fetch the ID of the user by screen name uid = self.get_user(user) # Fill up the last 'store' tweets if get_last is set if get_last: result = self.twitter.get("statuses/user_timeline", params={'user_id':uid, 'count':str(self.store)}) # Results are returned in reverse-chronological order result.reverse() self._lastn.extend(result) # Follow the given user self.statuses.filter(follow=uid) return True def add(self, callback): """Register a function to be called when tweets are received. Returns the TwitterUserStream object. Example: t = TwitterUserStream(...) t.add(myfunc).follow("Bob") """ self._callbacks.append(callback) return self def add_all(self, callbacks): """Register several functions to be called when tweets are received. Returns the TwitterUserStream object. Example: t = TwitterUserStream(...) t.add_all([myfunc1, myfunc2]).follow("Bob") """ self._callbacks.extend(callbacks) def remove(self, callback): """Unregister the function if it is currently a callback. Returns the TwitterUserStream object.""" if callback in self._callbacks: self._callbacks.remove(callback) def remove_all(self, callbacks): """Unregister several functions if they are registered. Returns the TwitterUserStream object.""" for callback in callbacks: self.remove(callback) def _filter_tweet(self, data): """Return True if the given tweet should be passed on to the callbacks, False otherwise.""" # If we don't have an ID, this isn't valid if 'id_str' not in data: return False # Ignore replies, quotes, and retweets if (data.get('in_reply_to_status_id_str') or data.get('quoted_statis_id_str') or data.get('retweeted_status')): return False # If show_dupes is off, ignore duplicate tweets if not self.show_dupes: for tweet in self._lastn: if data['id_str'] == tweet['id_str']: return return True def on_success(self, data): """Called by TwythonStreamer when a message is received on the underlying stream. Dispatches the message to all registered callbacks (in the order they were registered) if the message is not a duplicate or show_dupes is enabled.""" # Make sure this is a tweet we are interested in if not self._filter_tweet(data): return # Remember this message - if we reach our store limit, pop the oldest self._remember(data) # Notify callbacks for callback in self._callbacks: callback(data) def on_error(self, code, data): """Called when there is an error. Disconnects from the stream after receiving too many errors. Sets the 'error' attribute to an appropriate error message.""" errmsg = ("Twitter Error: [{0}] {1}".format(code, data)) print(errmsg) self.error = errmsg self._errors += 1 if self._errors >= self._max_errors: print("Maximum number of errors exceeded, disconnecting...") self.disconnect()
class TwythonAPITestCase(unittest.TestCase): def setUp(self): self.api = Twython('', '', '', '') def get_url(self, endpoint): """Convenience function for mapping from endpoint to URL""" return '%s/%s.json' % (self.api.api_url % self.api.api_version, endpoint) def register_response(self, method, url, body='{}', match_querystring=False, status=200, adding_headers=None, stream=False, content_type='application/json; charset=utf-8'): """Wrapper function for responses for simpler unit tests""" # responses uses BytesIO to hold the body so it needs to be in bytes if not is_py2: body = bytes(body, 'UTF-8') responses.add(method, url, body, match_querystring, status, adding_headers, stream, content_type) @responses.activate def test_request_should_handle_full_endpoint(self): """Test that request() accepts a full URL for the endpoint argument""" url = 'https://api.twitter.com/1.1/search/tweets.json' self.register_response(responses.GET, url) self.api.request(url) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_handle_relative_endpoint(self): """Test that request() accepts a twitter endpoint name for the endpoint argument""" url = 'https://api.twitter.com/1.1/search/tweets.json' self.register_response(responses.GET, url) self.api.request('search/tweets', version='1.1') self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_post_request_regardless_of_case(self): """Test that request() accepts the HTTP method name regardless of case""" url = 'https://api.twitter.com/1.1/statuses/update.json' self.register_response(responses.POST, url) self.api.request(url, method='POST') self.api.request(url, method='post') self.assertEqual(2, len(responses.calls)) self.assertEqual('POST', responses.calls[0].request.method) self.assertEqual('POST', responses.calls[1].request.method) @responses.activate def test_request_should_throw_exception_with_invalid_http_method(self): """Test that request() throws an exception when an invalid HTTP method is passed""" # TODO(cash): should Twython catch the AttributeError and throw a TwythonError self.assertRaises(AttributeError, self.api.request, endpoint='search/tweets', method='INVALID') @responses.activate def test_request_should_encode_boolean_as_lowercase_string(self): """Test that request() encodes a boolean parameter as a lowercase string""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'include_entities': True}) self.api.request(endpoint, params={'include_entities': False}) self.assertEqual(url + '?include_entities=true', responses.calls[0].request.url) self.assertEqual(url + '?include_entities=false', responses.calls[1].request.url) @responses.activate def test_request_should_handle_string_or_number_parameter(self): """Test that request() encodes a numeric or string parameter correctly""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'lang': 'es'}) self.api.request(endpoint, params={'count': 50}) self.assertEqual(url + '?lang=es', responses.calls[0].request.url) self.assertEqual(url + '?count=50', responses.calls[1].request.url) @responses.activate def test_request_should_encode_list_of_strings_as_string(self): """Test that request() encodes a list of strings as a comma-separated string""" endpoint = 'search/tweets' url = self.get_url(endpoint) location = ['37.781157', '-122.39872', '1mi'] self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': location}) # requests url encodes the parameters so , is %2C self.assertEqual(url + '?geocode=37.781157%2C-122.39872%2C1mi', responses.calls[0].request.url) @responses.activate def test_request_should_encode_numeric_list_as_string(self): """Test that request() encodes a list of numbers as a comma-separated string""" endpoint = 'search/tweets' url = self.get_url(endpoint) location = [37.781157, -122.39872, '1mi'] self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': location}) self.assertEqual(url + '?geocode=37.781157%2C-122.39872%2C1mi', responses.calls[0].request.url) @responses.activate def test_request_should_ignore_bad_parameter(self): """Test that request() ignores unexpected parameter types""" endpoint = 'search/tweets' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.request(endpoint, params={'geocode': self}) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_request_should_handle_file_as_parameter(self): """Test that request() pulls a file out of params for requests lib""" endpoint = 'account/update_profile_image' url = self.get_url(endpoint) self.register_response(responses.POST, url) mock_file = StringIO("Twython test image") self.api.request(endpoint, method='POST', params={'image': mock_file}) self.assertIn(b'filename="image"', responses.calls[0].request.body) self.assertIn(b"Twython test image", responses.calls[0].request.body) @responses.activate def test_request_should_put_params_in_body_when_post(self): """Test that request() passes params as data when the request is a POST""" endpoint = 'statuses/update' url = self.get_url(endpoint) self.register_response(responses.POST, url) self.api.request(endpoint, method='POST', params={'status': 'this is a test'}) self.assertIn(b'status=this+is+a+test', responses.calls[0].request.body) self.assertNotIn('status=this+is+a+test', responses.calls[0].request.url) @responses.activate def test_get_uses_get_method(self): """Test Twython generic GET request works""" endpoint = 'account/verify_credentials' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.get(endpoint) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) @responses.activate def test_post_uses_post_method(self): """Test Twython generic POST request works""" endpoint = 'statuses/update' url = self.get_url(endpoint) self.register_response(responses.POST, url) self.api.post(endpoint, params={'status': 'I love Twython!'}) self.assertEqual(1, len(responses.calls)) self.assertEqual(url, responses.calls[0].request.url) def test_raise_twython_error_on_request_exception(self): """Test if TwythonError is raised by a RequestException""" with mock.patch.object(requests.Session, 'get') as get_mock: # mocking an ssl cert error get_mock.side_effect = requests.RequestException( "hostname 'example.com' doesn't match ...") self.assertRaises(TwythonError, self.api.get, 'https://example.com') @responses.activate def test_request_should_get_convert_json_to_data(self): """Test that Twython converts JSON data to a Python object""" endpoint = 'statuses/show' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"id": 210462857140252672}') data = self.api.request(endpoint, params={'id': 210462857140252672}) self.assertEqual({'id': 210462857140252672}, data) @responses.activate def test_request_should_raise_exception_with_invalid_json(self): """Test that Twython handles invalid JSON (though Twitter should not return it)""" endpoint = 'statuses/show' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"id: 210462857140252672}') self.assertRaises(TwythonError, self.api.request, endpoint, params={'id': 210462857140252672}) @responses.activate def test_request_should_handle_401(self): """Test that Twython raises an auth error on 401 error""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Error"}]}', status=401) self.assertRaises(TwythonAuthError, self.api.request, endpoint) @responses.activate def test_request_should_handle_400_for_missing_auth_data(self): """Test that Twython raises an auth error on 400 error when no oauth data sent""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response( responses.GET, url, body='{"errors":[{"message":"Bad Authentication data"}]}', status=400) self.assertRaises(TwythonAuthError, self.api.request, endpoint) @responses.activate def test_request_should_handle_400_that_is_not_auth_related(self): """Test that Twython raises a normal error on 400 error when unrelated to authorization""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Bad request"}]}', status=400) self.assertRaises(TwythonError, self.api.request, endpoint) @responses.activate def test_request_should_handle_rate_limit(self): """Test that Twython raises an rate limit error on 429""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, body='{"errors":[{"message":"Rate Limit"}]}', status=429) self.assertRaises(TwythonRateLimitError, self.api.request, endpoint) @responses.activate def test_get_lastfunction_header_should_return_header(self): """Test getting last specific header of the last API call works""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url, adding_headers={'x-rate-limit-remaining': '37'}) self.api.get(endpoint) value = self.api.get_lastfunction_header('x-rate-limit-remaining') self.assertEqual('37', value) value2 = self.api.get_lastfunction_header('does-not-exist') self.assertIsNone(value2) value3 = self.api.get_lastfunction_header('not-there-either', '96') self.assertEqual('96', value3) def test_get_lastfunction_header_should_raise_error_when_no_previous_call( self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') @responses.activate def test_sends_correct_accept_encoding_header(self): """Test that Twython accepts compressed data.""" endpoint = 'statuses/home_timeline' url = self.get_url(endpoint) self.register_response(responses.GET, url) self.api.get(endpoint) self.assertEqual(b'gzip, deflate', responses.calls[0].request.headers['Accept-Encoding']) # Static methods def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual( constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_reply(self): """Test HTML for Tweet links the replied-to username.""" tweet_text = self.api.html_for_tweet(test_tweet_reply) self.assertEqual( tweet_text, u'<span class="twython-tweet-prefix"><a href="https://twitter.com/philgyford" class="twython-mention">@philgyford</a> </span>Here’s a test tweet that goes on as much as possible and includes an image. Hi to my fans in testland!<span class="twython-tweet-suffix"> https://t.co/tzhyk2QWSr</span>' ) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR expanded url self.assertTrue('http://google.com' not in tweet_text) self.assertTrue('google.com' not in tweet_text) def test_html_for_tweet_identical_urls(self): """If the 'url's for different url entities are identical, they should link correctly.""" tweet_text = self.api.html_for_tweet(test_tweet_identical_urls) self.assertEqual( tweet_text, u'Use Cases, Trials and Making 5G a Reality <a href="https://t.co/W0uArTMk9N" class="twython-url">buff.ly/2sEhrgO</a> #5G #innovation via @5GWorldSeries <a href="https://t.co/W0uArTMk9N" class="twython-url">buff.ly/2sEhrgO</a>' ) def test_html_for_tweet_symbols(self): tweet_text = self.api.html_for_tweet(test_tweet_symbols_object) # Should only link symbols listed in entities: self.assertTrue( '<a href="https://twitter.com/search?q=%24AAPL" class="twython-symbol">$AAPL</a>' in tweet_text) self.assertTrue( '<a href="https://twitter.com/search?q=%24ANOTHER" class="twython-symbol">$ANOTHER</a>' not in tweet_text) def test_html_for_tweet_no_symbols(self): """Should still work if tweet object has no symbols list""" tweet = test_tweet_symbols_object # Save a copy: symbols = tweet['entities']['symbols'] del tweet['entities']['symbols'] tweet_text = self.api.html_for_tweet(tweet) self.assertTrue('symbols: $AAPL and' in tweet_text) self.assertTrue('and $ANOTHER and $A.' in tweet_text) # Put the symbols back: test_tweet_symbols_object['entities']['symbols'] = symbols def test_html_for_tweet_compatmode(self): tweet_text = self.api.html_for_tweet(test_tweet_compat_object) # link to compat web status link self.assertTrue( u'<a href="https://t.co/SRmsuks2ru" class="twython-url">twitter.com/i/web/status/7…</a>' in tweet_text) def test_html_for_tweet_extendedmode(self): tweet_text = self.api.html_for_tweet(test_tweet_extended_object) # full tweet rendered with suffix self.assertEqual(test_tweet_extended_html, tweet_text) def test_cursor_requires_twython_function(self): """Test that cursor() raises when called without a Twython function""" def init_and_iterate_cursor(*args, **kwargs): cursor = self.api.cursor(*args, **kwargs) return next(cursor) non_function = object() non_twython_function = lambda x: x self.assertRaises(TypeError, init_and_iterate_cursor, non_function) self.assertRaises(TwythonError, init_and_iterate_cursor, non_twython_function)
def drnj_graph_crawler(fof, root): access_tokens = key_store.acquire_access_tokens() access_token_key = access_tokens[0][0] access_token_secret = access_tokens[0][1] twitter = Twython(consumer_key, consumer_secret, access_token_key, access_token_secret) cur = -1L # The friends/followers IDS to be retrieved will be stored here IDS = list() #SS = list() # Number of calls to the twitter API remain = 0 # True if data is fetched correctly success = True # Seconds to wait before trying again twitter limit wait = 120 print "Retrieving the recent profile of user %d\n" % root # First, try to get the recent profile settings of the user try: v = twitter.get('users/show', {"user_id": root}) print v['screen_name'], v['name'], json_encode(v) post_data = {"user_id": json_encode([root]), "v": json_encode([v]), "auth_user_id": auth_user_id, "auth_password": auth_password} print post_data post_response = requests.post(url=app_root_url + '/profiles/store', data=post_data) # post_response = requests.post(url=app_root_url + '/user/store', data={"user_id": root, "v": v}) except TwythonError as e: print e print "Error while fetching user profile from twitter, quitting ..." key_store.release_access_tokens(access_tokens) return e if v['protected']: post_data = {"user_id": root, "isProtected": 1, "auth_user_id": auth_user_id, "auth_password": auth_password} post_response = requests.post(url=app_root_url+'/scheduler/reportProtectedUserid', data=post_data) print "Reported User %d as having a Protected Account" % root key_store.release_access_tokens(access_tokens) else: print "Retrieving %s of user %d\n" % (fof, root) while 1: # Check if we still have some bandwidth available while remain<=0: v = twitter.get('application/rate_limit_status', {"resources": fof}) remain = v["resources"][fof]["/" + fof + "/ids"]["remaining"] if remain>0: break print "Waiting... Twitter API rate limit reached\n" time.sleep(wait) try: S = twitter.get(fof + '/ids', {'user_id': root, 'cursor': cur}) # We count the number of remaining requests to the Twitter API remain = remain - 1 IDS = IDS + S["ids"] # SS = SS.append(S) print "Total number of %s ID's retrieved so far: %d" % (fof, len(IDS)) cur = S["next_cursor"] if cur==0: break except TwythonError as e: print e success = False print "Error while fetching data from Twitter API" break key_store.release_access_tokens(access_tokens) print "IDS retrieved: " print IDS if success: post_data = {"user_id": root, "ids": json_encode(IDS),"auth_user_id":auth_user_id, "auth_password": auth_password} post_response = requests.post(url=app_root_url + '/' + fof + '/ids/store', data=post_data) print "%s" % post_response.content return post_response.content else: post_data = {"user_id": root, "isProtected": 1, "auth_user_id":auth_user_id, "auth_password": auth_password} post_response = requests.post(url=app_root_url+'/scheduler/reportProtectedUserid', data=post_data) print "Reported User as having a Protected Account %d" % root return post_response.content
class TwythonAPITestCase(unittest.TestCase): def setUp(self): client_args = { 'headers': { 'User-Agent': '__twython__ Test' }, 'allow_redirects': False } oauth2_client_args = { 'headers': {} # This is so we can hit coverage that Twython sets User-Agent for us if none is supplied } self.api = Twython(app_key, app_secret, oauth_token, oauth_token_secret, client_args=client_args) self.oauth2_api = Twython(app_key, access_token=access_token, client_args=oauth2_client_args) def test_construct_api_url(self): """Test constructing a Twitter API url works as we expect""" url = 'https://api.twitter.com/1.1/search/tweets.json' constructed_url = self.api.construct_api_url(url, q='#twitter') self.assertEqual(constructed_url, 'https://api.twitter.com/1.1/search/tweets.json?q=%23twitter') def test_get(self): """Test Twython generic GET request works""" self.api.get('account/verify_credentials') def test_post(self): """Test Twython generic POST request works, with a full url and with just an endpoint""" update_url = 'https://api.twitter.com/1.1/statuses/update.json' status = self.api.post(update_url, params={'status': 'I love Twython!'}) self.api.post('statuses/destroy/%s' % status['id_str']) def test_get_lastfunction_header(self): """Test getting last specific header of the last API call works""" self.api.get('statuses/home_timeline') self.api.get_lastfunction_header('x-rate-limit-remaining') def test_get_lastfunction_header_not_present(self): """Test getting specific header that does not exist from the last call returns None""" self.api.get('statuses/home_timeline') header = self.api.get_lastfunction_header('does-not-exist') self.assertEqual(header, None) def test_get_lastfunction_header_no_last_api_call(self): """Test attempting to get a header when no API call was made raises a TwythonError""" self.assertRaises(TwythonError, self.api.get_lastfunction_header, 'no-api-call-was-made') def test_search_gen(self): """Test looping through the generator results works, at least once that is""" search = self.api.search_gen('twitter', count=1) counter = 0 while counter < 2: counter += 1 result = next(search) new_id_str = int(result['id_str']) if counter == 1: prev_id_str = new_id_str time.sleep(1) # Give time for another tweet to come into search if counter == 2: self.assertTrue(new_id_str > prev_id_str) def test_encode(self): """Test encoding UTF-8 works""" self.api.encode('Twython is awesome!') def test_html_for_tweet(self): """Test HTML for Tweet returns what we want""" tweet_text = self.api.html_for_tweet(test_tweet_object) self.assertEqual(test_tweet_html, tweet_text) def test_html_for_tweet_expanded_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, use_expanded_url=True) # Make sure full url is in HTML self.assertTrue('http://google.com' in tweet_text) def test_html_for_tweet_short_url(self): """Test using expanded url in HTML for Tweet displays full urls""" tweet_text = self.api.html_for_tweet(test_tweet_object, False) # Make sure HTML doesn't contain the display OR exapanded url self.assertTrue(not 'http://google.com' in tweet_text) self.assertTrue(not 'google.com' in tweet_text) def test_raise_error_on_bad_ssl_cert(self): """Test TwythonError is raised by a RequestException when an actual HTTP happens""" self.assertRaises(TwythonError, self.api.get, 'https://example.com') # Timelines def test_get_mentions_timeline(self): """Test returning mentions timeline for authenticated user succeeds""" self.api.get_mentions_timeline() def test_get_user_timeline(self): """Test returning timeline for authenticated user and random user succeeds""" self.api.get_user_timeline() # Authenticated User Timeline self.api.get_user_timeline(screen_name='twitter') # Random User Timeline def test_get_protected_user_timeline_following(self): """Test returning a protected user timeline who you are following succeeds""" self.api.get_user_timeline(screen_name=protected_twitter_1) def test_get_protected_user_timeline_not_following(self): """Test returning a protected user timeline who you are not following fails and raise a TwythonAuthError""" self.assertRaises(TwythonAuthError, self.api.get_user_timeline, screen_name=protected_twitter_2) def test_retweeted_of_me(self): """Test that getting recent tweets by authenticated user that have been retweeted by others succeeds""" self.api.retweeted_of_me() def test_get_home_timeline(self): """Test returning home timeline for authenticated user succeeds""" self.api.get_home_timeline() # Tweets def test_get_retweets(self): """Test getting retweets of a specific tweet succeeds""" self.api.get_retweets(id=test_tweet_id) def test_show_status(self): """Test returning a single status details succeeds""" self.api.show_status(id=test_tweet_id) def test_update_and_destroy_status(self): """Test updating and deleting a status succeeds""" status = self.api.update_status(status='Test post just to get deleted :(') self.api.destroy_status(id=status['id_str']) def test_get_oembed_tweet(self): """Test getting info to embed tweet on Third Party site succeeds""" self.api.get_oembed_tweet(id='99530515043983360') def test_get_retweeters_ids(self): """Test getting ids for people who retweeted a tweet succeeds""" self.api.get_retweeters_ids(id='99530515043983360') # Search def test_search(self): """Test searching tweets succeeds""" self.api.search(q='twitter') # Direct Messages def test_get_direct_messages(self): """Test getting the authenticated users direct messages succeeds""" self.api.get_direct_messages() def test_get_sent_messages(self): """Test getting the authenticated users direct messages they've sent succeeds""" self.api.get_sent_messages() def test_send_get_and_destroy_direct_message(self): """Test sending, getting, then destory a direct message succeeds""" message = self.api.send_direct_message(screen_name=protected_twitter_1, text='Hey d00d! %s' % int(time.time())) self.api.get_direct_message(id=message['id_str']) self.api.destroy_direct_message(id=message['id_str']) def test_send_direct_message_to_non_follower(self): """Test sending a direct message to someone who doesn't follow you fails""" self.assertRaises(TwythonError, self.api.send_direct_message, screen_name=protected_twitter_2, text='Yo, man!') # Friends & Followers def test_get_user_ids_of_blocked_retweets(self): """Test that collection of user_ids that the authenticated user does not want to receive retweets from succeeds""" self.api.get_user_ids_of_blocked_retweets(stringify_ids=True) def test_get_friends_ids(self): """Test returning ids of users the authenticated user and then a random user is following succeeds""" self.api.get_friends_ids() self.api.get_friends_ids(screen_name='twitter') def test_get_followers_ids(self): """Test returning ids of users the authenticated user and then a random user are followed by succeeds""" self.api.get_followers_ids() self.api.get_followers_ids(screen_name='twitter') def test_lookup_friendships(self): """Test returning relationships of the authenticating user to the comma-separated list of up to 100 screen_names or user_ids provided succeeds""" self.api.lookup_friendships(screen_name='twitter,ryanmcgrath') def test_get_incoming_friendship_ids(self): """Test returning incoming friendship ids succeeds""" self.api.get_incoming_friendship_ids() def test_get_outgoing_friendship_ids(self): """Test returning outgoing friendship ids succeeds""" self.api.get_outgoing_friendship_ids() def test_create_friendship(self): """Test creating a friendship succeeds""" self.api.create_friendship(screen_name='justinbieber') def test_destroy_friendship(self): """Test destroying a friendship succeeds""" self.api.destroy_friendship(screen_name='justinbieber') def test_update_friendship(self): """Test updating friendships succeeds""" self.api.update_friendship(screen_name=protected_twitter_1, retweets='true') self.api.update_friendship(screen_name=protected_twitter_1, retweets=False) def test_show_friendships(self): """Test showing specific friendship succeeds""" self.api.show_friendship(target_screen_name=protected_twitter_1) def test_get_friends_list(self): """Test getting list of users authenticated user then random user is following succeeds""" self.api.get_friends_list() self.api.get_friends_list(screen_name='twitter') def test_get_followers_list(self): """Test getting list of users authenticated user then random user are followed by succeeds""" self.api.get_followers_list() self.api.get_followers_list(screen_name='twitter') # Users def test_get_account_settings(self): """Test getting the authenticated user account settings succeeds""" self.api.get_account_settings() def test_verify_credentials(self): """Test representation of the authenticated user call succeeds""" self.api.verify_credentials() def test_update_account_settings(self): """Test updating a user account settings succeeds""" self.api.update_account_settings(lang='en') def test_update_delivery_service(self): """Test updating delivery settings fails because we don't have a mobile number on the account""" self.assertRaises(TwythonError, self.api.update_delivery_service, device='none') def test_update_profile(self): """Test updating profile succeeds""" self.api.update_profile(include_entities='true') def test_update_profile_colors(self): """Test updating profile colors succeeds""" self.api.update_profile_colors(profile_background_color='3D3D3D') def test_list_blocks(self): """Test listing users who are blocked by the authenticated user succeeds""" self.api.list_blocks() def test_list_block_ids(self): """Test listing user ids who are blocked by the authenticated user succeeds""" self.api.list_block_ids() def test_create_block(self): """Test blocking a user succeeds""" self.api.create_block(screen_name='justinbieber') def test_destroy_block(self): """Test unblocking a user succeeds""" self.api.destroy_block(screen_name='justinbieber') def test_lookup_user(self): """Test listing a number of user objects succeeds""" self.api.lookup_user(screen_name='twitter,justinbieber') def test_show_user(self): """Test showing one user works""" self.api.show_user(screen_name='twitter') def test_search_users(self): """Test that searching for users succeeds""" self.api.search_users(q='Twitter API') def test_get_contributees(self): """Test returning list of accounts the specified user can contribute to succeeds""" self.api.get_contributees(screen_name='TechCrunch') def test_get_contributors(self): """Test returning list of accounts that contribute to the authenticated user fails because we are not a Contributor account""" self.assertRaises(TwythonError, self.api.get_contributors, screen_name=screen_name) def test_remove_profile_banner(self): """Test removing profile banner succeeds""" self.api.remove_profile_banner() def test_get_profile_banner_sizes(self): """Test getting list of profile banner sizes fails because we have not uploaded a profile banner""" self.assertRaises(TwythonError, self.api.get_profile_banner_sizes) # Suggested Users def test_get_user_suggestions_by_slug(self): """Test getting user suggestions by slug succeeds""" self.api.get_user_suggestions_by_slug(slug='twitter') def test_get_user_suggestions(self): """Test getting user suggestions succeeds""" self.api.get_user_suggestions() def test_get_user_suggestions_statuses_by_slug(self): """Test getting status of suggested users succeeds""" self.api.get_user_suggestions_statuses_by_slug(slug='funny') # Favorites def test_get_favorites(self): """Test getting list of favorites for the authenticated user succeeds""" self.api.get_favorites() def test_create_and_destroy_favorite(self): """Test creating and destroying a favorite on a tweet succeeds""" self.api.create_favorite(id=test_tweet_id) self.api.destroy_favorite(id=test_tweet_id) # Lists def test_show_lists(self): """Test show lists for specified user""" self.api.show_lists(screen_name='twitter') def test_get_list_statuses(self): """Test timeline of tweets authored by members of the specified list succeeds""" self.api.get_list_statuses(list_id=test_list_id) def test_create_update_destroy_list_add_remove_list_members(self): """Test create a list, adding and removing members then deleting the list succeeds""" the_list = self.api.create_list(name='Stuff') list_id = the_list['id_str'] self.api.update_list(list_id=list_id, name='Stuff Renamed') screen_names = ['johncena', 'xbox'] # Multi add/delete members self.api.create_list_members(list_id=list_id, screen_name=screen_names) self.api.delete_list_members(list_id=list_id, screen_name=screen_names) # Single add/delete member self.api.add_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list_member(list_id=list_id, screen_name='justinbieber') self.api.delete_list(list_id=list_id) def test_get_list_subscribers(self): """Test list of subscribers of a specific list succeeds""" self.api.get_list_subscribers(list_id=test_list_id) def test_subscribe_is_subbed_and_unsubscribe_to_list(self): """Test subscribing, is a list sub and unsubbing to list succeeds""" self.api.subscribe_to_list(list_id=test_list_id) # Returns 404 if user is not a subscriber self.api.is_list_subscriber(list_id=test_list_id, screen_name=screen_name) self.api.unsubscribe_from_list(list_id=test_list_id) def test_is_list_member(self): """Test returning if specified user is member of a list succeeds""" # Returns 404 if not list member self.api.is_list_member(list_id=test_list_id, screen_name='jack') def test_get_list_members(self): """Test listing members of the specified list succeeds""" self.api.get_list_members(list_id=test_list_id) def test_get_specific_list(self): """Test getting specific list succeeds""" self.api.get_specific_list(list_id=test_list_id) def test_get_list_subscriptions(self): """Test collection of the lists the specified user is subscribed to succeeds""" self.api.get_list_subscriptions(screen_name='twitter') def test_show_owned_lists(self): """Test collection of lists the specified user owns succeeds""" self.api.show_owned_lists(screen_name='twitter') # Saved Searches def test_get_saved_searches(self): """Test getting list of saved searches for authenticated user succeeds""" self.api.get_saved_searches() def test_create_get_destroy_saved_search(self): """Test getting list of saved searches for authenticated user succeeds""" saved_search = self.api.create_saved_search(query='#Twitter') saved_search_id = saved_search['id_str'] self.api.show_saved_search(id=saved_search_id) self.api.destroy_saved_search(id=saved_search_id) # Places & Geo def test_get_geo_info(self): """Test getting info about a geo location succeeds""" self.api.get_geo_info(place_id='df51dec6f4ee2b2c') def test_reverse_geo_code(self): """Test reversing geocode succeeds""" self.api.reverse_geocode(lat='37.76893497', long='-122.42284884') def test_search_geo(self): """Test search for places that can be attached to a statuses/update succeeds""" self.api.search_geo(query='Toronto') def test_get_similar_places(self): """Test locates places near the given coordinates which are similar in name succeeds""" self.api.get_similar_places(lat='37', long='-122', name='Twitter HQ') # Trends def test_get_place_trends(self): """Test getting the top 10 trending topics for a specific WOEID succeeds""" self.api.get_place_trends(id=1) def test_get_available_trends(self): """Test returning locations that Twitter has trending topic information for succeeds""" self.api.get_available_trends() def test_get_closest_trends(self): """Test getting the locations that Twitter has trending topic information for, closest to a specified location succeeds""" self.api.get_closest_trends(lat='37', long='-122') # Help def test_get_application_rate_limit_status(self): """Test getting application rate limit status succeeds""" self.oauth2_api.get_application_rate_limit_status()