def user_timeline(self, target_handle, count=2, tweet_mode='extended'): if target_handle == 0: from tweepy.error import RateLimitError raise RateLimitError(reason=88) with open('./test_data/test_timeline', 'rb') as handle: test_timeline = pickle.load(handle) return test_timeline
def upload_execute(self, upload_stage, **kwargs): url = self.api_root + self.path full_url = 'https://' + self.host + url auth = None if self.api.auth: auth = self.api.auth.apply_auth() params = {} post_data = {} file_data = {} if upload_stage in ['init', 'finalize']: post_data = kwargs.pop('post_data', None) self.method = 'POST' elif upload_stage == 'append': post_data = kwargs.pop('post_data', None) file_data = kwargs.pop('file_data', None) self.method = 'POST' elif upload_stage == 'status': self.method = 'GET' params = kwargs.pop('params') else: raise TweepError('Invalid upload stage: %s' % upload_stage) try: resp = self.session.request(self.method, full_url, params=params, data=post_data, files=file_data, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: raise TweepError('Failed to send request: %s' % e) if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception as e: error_msg = 'Twitter error response: status code = %s' % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg) else: error_msg = resp.text raise TweepError(error_msg, resp, api_code=api_error_code) if upload_stage == 'append': # The append async stage only returns a 2XX HTTP response so # the append upload stage just returns a True value. Errors # would be caught above in the status code check return True else: result = self.parser.parse(self, resp.text) return result
def check_api_call_errors(self, api, params, mock_api): setattr(self.real_client, mock_api, MagicMock(side_effect=TweepError(reason='unknown'))) self.assertRaises(APIError, getattr(self.client, api), **params) setattr(self.real_client, mock_api, MagicMock(side_effect=RateLimitError(reason='unknown'))) self.assertRaises(APIQuotaError, getattr(self.client, api), **params)
def search(self, search_term, count=20, result_type='recent', max_id='-1', since_id='-1'): self.latest_since = since_id if search_term == 'Rate limit': from tweepy.error import RateLimitError raise RateLimitError(reason=88) if max_id == -1: with open('./test_data/test_timeline', 'rb') as handle: test_timeline = pickle.load(handle) return test_timeline else: return []
def post(self, url, data=None, json=None, **kwargs): kwargs['auth'] = self.oauth response = requests.post(url=url, data=data, json=json, **kwargs) if response.status_code and not 200 <= response.status_code < 300: try: error_msg, api_error_code = \ JSONParser().parse_error(response.text) except Exception: error_msg = "Twitter error response: status code = %s" % response.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, response) else: raise TweepError(error_msg, response, api_code=api_error_code) return response
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get( '%s?%s' % (url, urlencode(self.session.params))) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # handle running out of api calls if self.wait_on_rate_limit: if self._reset_time is not None: if self._remaining_calls is not None: if self._remaining_calls < 1: sleep_time = self._reset_time - int( time.time()) if sleep_time > 0: if self.wait_on_rate_limit_notify: log.warning( "Rate limit reached. Sleeping for: %d" % sleep_time) time.sleep(sleep_time + 5) # sleep for few extra sec # if self.wait_on_rate_limit and self._reset_time is not None and \ # self._remaining_calls is not None and self._remaining_calls < 1: # sleep_time = self._reset_time - int(time.time()) # if sleep_time > 0: # if self.wait_on_rate_limit_notify: # log.warning("Rate limit reached. Sleeping for: %d" % sleep_time) # time.sleep(sleep_time + 5) # sleep for few extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: resp = self.session.request(self.method, full_url, data=self.post_data, json=self.json_payload, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: raise TweepError('Failed to send request: %s' % e).with_traceback(sys.exc_info()[2]) rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif isinstance(self._remaining_calls, int): self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) if self.wait_on_rate_limit and self._remaining_calls == 0 and ( # if ran out of calls before waiting switching retry last call resp.status_code == 429 or resp.status_code == 420): continue retry_delay = self.retry_delay # Exit request loop if non-retry error code if resp.status_code in (200, 204): break elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit: if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload self.return_cursors = (self.return_cursors or 'cursor' in self.session.params or 'next' in self.session.params) result = self.parser.parse(self, resp.text, return_cursors=self.return_cursors) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store( '%s?%s' % (url, urlencode(self.session.params)), result) return result
def execute(self): self.api.cached_result = False # Build the request URL url = self.api_root + self.path full_url = 'https://' + self.host + url # Query the cache if one is available # and this request uses a GET method. if self.use_cache and self.api.cache and self.method == 'GET': cache_result = self.api.cache.get( '%s?%s' % (url, urlencode(self.session.params))) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.retry_count + 1: # If auth is RateLimitHandler, select the access token if hasattr(self.api.auth, 'tokens'): # safe bet key, limit, remaining, reset = \ self.api.auth.select_access_token(self.resource) self.api.auth.set_access_token(key) self._reset_time = reset self._remaining_calls = remaining # handle running out of api calls if self._reset_time is not None and \ self._remaining_calls is not None and self._remaining_calls == 0: sleep_time = self._reset_time - int(time.time()) if self.wait_on_rate_limit: if sleep_time > 0: if self.wait_on_rate_limit_notify: log.warning( "Rate limit reached. Sleeping for: {}, until: {}" .format( sleep_time, time.strftime( '%H:%M:%S', time.localtime(self._reset_time)))) time.sleep(sleep_time + 5) # sleep for few extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Request compression if configured if self.api.compression: self.session.headers['Accept-encoding'] = 'gzip' # Execute request try: resp = self.session.request(self.method, full_url, data=self.post_data, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2]) rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif isinstance(self._remaining_calls, int): self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) if self.wait_on_rate_limit and self._remaining_calls == 0 and ( # if ran out of calls before waiting switching retry last call resp.status_code == 429 or resp.status_code == 420): # As the twitter reset_time is returned incorrectly early, # the program loops up to here until the rate_limit is reset. # Therefore, add a small sleep to reduce load. time.sleep(10) continue # If auth is RateLimitHandler, update rate limits if hasattr(self.api.auth, 'tokens'): self.api.auth.update_rate_limits( self.api.auth.access_token, self.resource, remaining=self._remaining_calls, reset=self._reset_time) retry_delay = self.retry_delay # Exit request loop if non-retry error code if resp.status_code == 200: break elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit: if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.retry_errors and resp.status_code not in self.retry_errors: break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = \ self.parser.parse_error(resp.text) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status_code api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload result = self.parser.parse(self, resp.text) # Store result into cache if one is available. if self.use_cache and self.api.cache and self.method == 'GET' and result: self.api.cache.store( '%s?%s' % (url, urlencode(self.session.params)), result) return result
def execute(api, method, path, *args, allowed_param=[], params=None, headers=None, json_payload=None, parser=None, payload_list=False, payload_type=None, post_data=None, require_auth=False, return_cursors=False, upload_api=False, use_cache=True, **kwargs): # If authentication is required and no credentials # are provided, throw an error. if require_auth and not api.auth: raise TweepError('Authentication required!') api.cached_result = False # Build the request URL if upload_api: url = api.upload_root + path full_url = 'https://' + api.upload_host + url else: url = api.api_root + path full_url = 'https://' + api.host + url if params is None: params = {} for idx, arg in enumerate(args): if arg is None: continue try: params[allowed_param[idx]] = str(arg) except IndexError: raise TweepError('Too many parameters supplied!') for k, arg in kwargs.items(): if arg is None: continue if k in params: raise TweepError(f'Multiple values for parameter {k} supplied!') params[k] = str(arg) log.debug("PARAMS: %r", params) # Query the cache if one is available # and this request uses a GET method. if use_cache and api.cache and method == 'GET': cache_result = api.cache.get(f'{url}?{urlencode(params)}') # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = api else: if isinstance(cache_result, Model): cache_result._api = api api.cached_result = True return cache_result # Monitoring rate limits remaining_calls = None reset_time = None session = requests.Session() if parser is None: parser = api.parser try: # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed <= api.retry_count: if (api.wait_on_rate_limit and reset_time is not None and remaining_calls is not None and remaining_calls < 1): # Handle running out of API calls sleep_time = reset_time - int(time.time()) if sleep_time > 0: log.warning( f"Rate limit reached. Sleeping for: {sleep_time}") time.sleep(sleep_time + 1) # Sleep for extra sec # Apply authentication auth = None if api.auth: auth = api.auth.apply_auth() # Execute request try: resp = session.request(method, full_url, params=params, headers=headers, data=post_data, json=json_payload, timeout=api.timeout, auth=auth, proxies=api.proxy) except Exception as e: raise TweepError( f'Failed to send request: {e}').with_traceback( sys.exc_info()[2]) if 200 <= resp.status_code < 300: break rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: remaining_calls = int(rem_calls) elif remaining_calls is not None: remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: reset_time = int(reset_time) retry_delay = api.retry_delay if resp.status_code in (420, 429) and api.wait_on_rate_limit: if remaining_calls == 0: # If ran out of calls before waiting switching retry last call continue if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif api.retry_errors and resp.status_code not in api.retry_errors: # Exit request loop if non-retry error code break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = parser.parse_error(resp.text) except Exception: error_msg = f"Twitter error response: status code = {resp.status_code}" api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload return_cursors = return_cursors or 'cursor' in params or 'next' in params result = parser.parse(resp.text, api=api, payload_list=payload_list, payload_type=payload_type, return_cursors=return_cursors) # Store result into cache if one is available. if use_cache and api.cache and method == 'GET' and result: api.cache.store(f'{url}?{urlencode(params)}', result) return result finally: session.close()
def execute(self, method, path, *, headers=None, json_payload=None, post_data=None, require_auth=False, return_cursors=False, upload_api=False, use_cache=True): # If authentication is required and no credentials # are provided, throw an error. if require_auth and not self.api.auth: raise TweepError('Authentication required!') self.api.cached_result = False # Pick correct URL root to use if upload_api: api_root = self.api.upload_root else: api_root = self.api.api_root if upload_api: host = self.api.upload_host else: host = self.api.host # Build the request URL url = api_root + path full_url = 'https://' + host + url # Query the cache if one is available # and this request uses a GET method. if use_cache and self.api.cache and method == 'GET': cache_result = self.api.cache.get( f'{url}?{urlencode(self.session.params)}') # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: if isinstance(result, Model): result._api = self.api else: if isinstance(cache_result, Model): cache_result._api = self.api self.api.cached_result = True return cache_result # Continue attempting request until successful # or maximum number of retries is reached. retries_performed = 0 while retries_performed < self.api.retry_count + 1: if (self.api.wait_on_rate_limit and self._reset_time is not None and self._remaining_calls is not None and self._remaining_calls < 1): # Handle running out of API calls sleep_time = self._reset_time - int(time.time()) if sleep_time > 0: if self.api.wait_on_rate_limit_notify: log.warning( f"Rate limit reached. Sleeping for: {sleep_time}") time.sleep(sleep_time + 1) # Sleep for extra sec # Apply authentication auth = None if self.api.auth: auth = self.api.auth.apply_auth() # Execute request try: resp = self.session.request(method, full_url, headers=headers, data=post_data, json=json_payload, timeout=self.api.timeout, auth=auth, proxies=self.api.proxy) except Exception as e: raise TweepError( f'Failed to send request: {e}').with_traceback( sys.exc_info()[2]) if 200 <= resp.status_code < 300: break rem_calls = resp.headers.get('x-rate-limit-remaining') if rem_calls is not None: self._remaining_calls = int(rem_calls) elif self._remaining_calls is not None: self._remaining_calls -= 1 reset_time = resp.headers.get('x-rate-limit-reset') if reset_time is not None: self._reset_time = int(reset_time) retry_delay = self.api.retry_delay if resp.status_code in (420, 429) and self.api.wait_on_rate_limit: if self._remaining_calls == 0: # If ran out of calls before waiting switching retry last call continue if 'retry-after' in resp.headers: retry_delay = float(resp.headers['retry-after']) elif self.api.retry_errors and resp.status_code not in self.api.retry_errors: # Exit request loop if non-retry error code break # Sleep before retrying request again time.sleep(retry_delay) retries_performed += 1 # If an error was returned, throw an exception self.api.last_response = resp if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg, api_error_code = self.parser.parse_error(resp.text) except Exception: error_msg = f"Twitter error response: status code = {resp.status_code}" api_error_code = None if is_rate_limit_error_message(error_msg): raise RateLimitError(error_msg, resp) else: raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload return_cursors = (return_cursors or 'cursor' in self.session.params or 'next' in self.session.params) result = self.parser.parse(self, resp.text, return_cursors=return_cursors) # Store result into cache if one is available. if use_cache and self.api.cache and method == 'GET' and result: self.api.cache.store(f'{url}?{urlencode(self.session.params)}', result) return result