def upload_execute(self, upload_stage, **kwargs):
url = self.api_root + self.path
full_url = 'https://' + self.host + url
auth = None
if self.api.auth:
auth = self.api.auth.apply_auth()
params = {}
post_data = {}
file_data = {}
if upload_stage in ['init', 'finalize']:
post_data = kwargs.pop('post_data', None)
self.method = 'POST'
elif upload_stage == 'append':
post_data = kwargs.pop('post_data', None)
file_data = kwargs.pop('file_data', None)
self.method = 'POST'
elif upload_stage == 'status':
self.method = 'GET'
params = kwargs.pop('params')
else:
raise TweepError('Invalid upload stage: %s' % upload_stage)
try:
resp = self.session.request(self.method,
full_url,
params=params,
data=post_data,
files=file_data,
timeout=self.api.timeout,
auth=auth,
proxies=self.api.proxy)
except Exception as e:
raise TweepError('Failed to send request: %s' % e)
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg, api_error_code = \
self.parser.parse_error(resp.text)
except Exception as e:
error_msg = 'Twitter error response: status code = %s' % resp.status_code
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg)
else:
error_msg = resp.text
raise TweepError(error_msg, resp, api_code=api_error_code)
if upload_stage == 'append':
# The append async stage only returns a 2XX HTTP response so
# the append upload stage just returns a True value. Errors
# would be caught above in the status code check
return True
else:
result = self.parser.parse(self, resp.text)
return result
def post(self, url, data=None, json=None, **kwargs):
kwargs['auth'] = self.oauth
response = requests.post(url=url, data=data, json=json, **kwargs)
if response.status_code and not 200 <= response.status_code < 300:
try:
error_msg, api_error_code = \
JSONParser().parse_error(response.text)
except Exception:
error_msg = "Twitter error response: status code = %s" % response.status_code
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, response)
else:
raise TweepError(error_msg, response, api_code=api_error_code)
return response
def execute(self):
self.api.cached_result = False
# Build the request URL
url = self.api_root + self.path
full_url = 'https://' + self.host + url
# Query the cache if one is available
# and this request uses a GET method.
if self.use_cache and self.api.cache and self.method == 'GET':
cache_result = self.api.cache.get(
'%s?%s' % (url, urlencode(self.session.params)))
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
if isinstance(result, Model):
result._api = self.api
else:
if isinstance(cache_result, Model):
cache_result._api = self.api
self.api.cached_result = True
return cache_result
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
while retries_performed < self.retry_count + 1:
# handle running out of api calls
if self.wait_on_rate_limit:
if self._reset_time is not None:
if self._remaining_calls is not None:
if self._remaining_calls < 1:
sleep_time = self._reset_time - int(
time.time())
if sleep_time > 0:
if self.wait_on_rate_limit_notify:
log.warning(
"Rate limit reached. Sleeping for: %d"
% sleep_time)
time.sleep(sleep_time +
5) # sleep for few extra sec
# if self.wait_on_rate_limit and self._reset_time is not None and \
# self._remaining_calls is not None and self._remaining_calls < 1:
# sleep_time = self._reset_time - int(time.time())
# if sleep_time > 0:
# if self.wait_on_rate_limit_notify:
# log.warning("Rate limit reached. Sleeping for: %d" % sleep_time)
# time.sleep(sleep_time + 5) # sleep for few extra sec
# Apply authentication
auth = None
if self.api.auth:
auth = self.api.auth.apply_auth()
# Request compression if configured
if self.api.compression:
self.session.headers['Accept-encoding'] = 'gzip'
# Execute request
try:
resp = self.session.request(self.method,
full_url,
data=self.post_data,
json=self.json_payload,
timeout=self.api.timeout,
auth=auth,
proxies=self.api.proxy)
except Exception as e:
raise TweepError('Failed to send request: %s' %
e).with_traceback(sys.exc_info()[2])
rem_calls = resp.headers.get('x-rate-limit-remaining')
if rem_calls is not None:
self._remaining_calls = int(rem_calls)
elif isinstance(self._remaining_calls, int):
self._remaining_calls -= 1
reset_time = resp.headers.get('x-rate-limit-reset')
if reset_time is not None:
self._reset_time = int(reset_time)
if self.wait_on_rate_limit and self._remaining_calls == 0 and (
# if ran out of calls before waiting switching retry last call
resp.status_code == 429 or resp.status_code == 420):
continue
retry_delay = self.retry_delay
# Exit request loop if non-retry error code
if resp.status_code in (200, 204):
break
elif (resp.status_code == 429
or resp.status_code == 420) and self.wait_on_rate_limit:
if 'retry-after' in resp.headers:
retry_delay = float(resp.headers['retry-after'])
elif self.retry_errors and resp.status_code not in self.retry_errors:
break
# Sleep before retrying request again
time.sleep(retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
self.api.last_response = resp
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg, api_error_code = \
self.parser.parse_error(resp.text)
except Exception:
error_msg = "Twitter error response: status code = %s" % resp.status_code
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, resp)
else:
raise TweepError(error_msg, resp, api_code=api_error_code)
# Parse the response payload
self.return_cursors = (self.return_cursors
or 'cursor' in self.session.params
or 'next' in self.session.params)
result = self.parser.parse(self,
resp.text,
return_cursors=self.return_cursors)
# Store result into cache if one is available.
if self.use_cache and self.api.cache and self.method == 'GET' and result:
self.api.cache.store(
'%s?%s' % (url, urlencode(self.session.params)), result)
return result
def execute(self):
self.api.cached_result = False
# Build the request URL
url = self.api_root + self.path
full_url = 'https://' + self.host + url
# Query the cache if one is available
# and this request uses a GET method.
if self.use_cache and self.api.cache and self.method == 'GET':
cache_result = self.api.cache.get(url)
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
if isinstance(result, Model):
result._api = self.api
else:
if isinstance(cache_result, Model):
cache_result._api = self.api
self.api.cached_result = True
return cache_result
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
while retries_performed < self.retry_count + 1:
# handle running out of api calls
if self.wait_on_rate_limit:
if self._reset_time is not None:
if self._remaining_calls is not None:
if self._remaining_calls < 1:
sleep_time = self._reset_time - int(time.time())
if sleep_time > 0:
if self.wait_on_rate_limit_notify:
print("Rate limit reached. Sleeping for:", sleep_time)
time.sleep(sleep_time + 5) # sleep for few extra sec
# if self.wait_on_rate_limit and self._reset_time is not None and \
# self._remaining_calls is not None and self._remaining_calls < 1:
# sleep_time = self._reset_time - int(time.time())
# if sleep_time > 0:
# if self.wait_on_rate_limit_notify:
# print("Rate limit reached. Sleeping for: " + str(sleep_time))
# time.sleep(sleep_time + 5) # sleep for few extra sec
# Apply authentication
if self.api.auth:
auth = self.api.auth.apply_auth()
# Request compression if configured
if self.api.compression:
self.session.headers['Accept-encoding'] = 'gzip'
# Execute request
try:
resp = self.session.request(self.method,
full_url,
data=self.post_data,
timeout=self.api.timeout,
auth=auth,
proxies=self.api.proxy)
except Exception as e:
raise TweepError('Failed to send request: %s' % e)
rem_calls = resp.headers.get('x-rate-limit-remaining')
if rem_calls is not None:
self._remaining_calls = int(rem_calls)
elif isinstance(self._remaining_calls, int):
self._remaining_calls -= 1
reset_time = resp.headers.get('x-rate-limit-reset')
if reset_time is not None:
self._reset_time = int(reset_time)
if self.wait_on_rate_limit and self._remaining_calls == 0 and (
# if ran out of calls before waiting switching retry last call
resp.status_code == 429 or resp.status_code == 420):
continue
retry_delay = self.retry_delay
# Exit request loop if non-retry error code
if resp.status_code == 200:
break
elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit:
if 'retry-after' in resp.headers:
retry_delay = float(resp.headers['retry-after'])
elif self.retry_errors and resp.status_code not in self.retry_errors:
break
# Sleep before retrying request again
time.sleep(retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
self.api.last_response = resp
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg = self.parser.parse_error(resp.text)
except Exception:
error_msg = "Twitter error response: status code = %s" % resp.status_code
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, resp)
else:
raise TweepError(error_msg, resp)
# Parse the response payload
result = self.parser.parse(self, resp.text)
# Store result into cache if one is available.
if self.use_cache and self.api.cache and self.method == 'GET' and result:
self.api.cache.store(url, result)
return result
def execute(self):
self.api.cached_result = False
# Build the request URL
url = self.api_root + self.path
full_url = 'https://' + self.host + url
# Query the cache if one is available
# and this request uses a GET method.
if self.use_cache and self.api.cache and self.method == 'GET':
cache_result = self.api.cache.get(
'%s?%s' % (url, urlencode(self.session.params)))
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
if isinstance(result, Model):
result._api = self.api
else:
if isinstance(cache_result, Model):
cache_result._api = self.api
self.api.cached_result = True
return cache_result
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
while retries_performed < self.retry_count + 1:
# If auth is RateLimitHandler, select the access token
if hasattr(self.api.auth, 'tokens'): # safe bet
key, limit, remaining, reset = \
self.api.auth.select_access_token(self.resource)
self.api.auth.set_access_token(key)
self._reset_time = reset
self._remaining_calls = remaining
# handle running out of api calls
if self._reset_time is not None and \
self._remaining_calls is not None and self._remaining_calls == 0:
sleep_time = self._reset_time - int(time.time())
if self.wait_on_rate_limit:
if sleep_time > 0:
if self.wait_on_rate_limit_notify:
log.warning(
"Rate limit reached. Sleeping for: {}, until: {}"
.format(
sleep_time,
time.strftime(
'%H:%M:%S',
time.localtime(self._reset_time))))
time.sleep(sleep_time +
5) # sleep for few extra sec
# Apply authentication
auth = None
if self.api.auth:
auth = self.api.auth.apply_auth()
# Request compression if configured
if self.api.compression:
self.session.headers['Accept-encoding'] = 'gzip'
# Execute request
try:
resp = self.session.request(self.method,
full_url,
data=self.post_data,
timeout=self.api.timeout,
auth=auth,
proxies=self.api.proxy)
except Exception as e:
six.reraise(TweepError,
TweepError('Failed to send request: %s' % e),
sys.exc_info()[2])
rem_calls = resp.headers.get('x-rate-limit-remaining')
if rem_calls is not None:
self._remaining_calls = int(rem_calls)
elif isinstance(self._remaining_calls, int):
self._remaining_calls -= 1
reset_time = resp.headers.get('x-rate-limit-reset')
if reset_time is not None:
self._reset_time = int(reset_time)
if self.wait_on_rate_limit and self._remaining_calls == 0 and (
# if ran out of calls before waiting switching retry last call
resp.status_code == 429 or resp.status_code == 420):
# As the twitter reset_time is returned incorrectly early,
# the program loops up to here until the rate_limit is reset.
# Therefore, add a small sleep to reduce load.
time.sleep(10)
continue
# If auth is RateLimitHandler, update rate limits
if hasattr(self.api.auth, 'tokens'):
self.api.auth.update_rate_limits(
self.api.auth.access_token,
self.resource,
remaining=self._remaining_calls,
reset=self._reset_time)
retry_delay = self.retry_delay
# Exit request loop if non-retry error code
if resp.status_code == 200:
break
elif (resp.status_code == 429
or resp.status_code == 420) and self.wait_on_rate_limit:
if 'retry-after' in resp.headers:
retry_delay = float(resp.headers['retry-after'])
elif self.retry_errors and resp.status_code not in self.retry_errors:
break
# Sleep before retrying request again
time.sleep(retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
self.api.last_response = resp
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg, api_error_code = \
self.parser.parse_error(resp.text)
except Exception:
error_msg = "Twitter error response: status code = %s" % resp.status_code
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, resp)
else:
raise TweepError(error_msg, resp, api_code=api_error_code)
# Parse the response payload
result = self.parser.parse(self, resp.text)
# Store result into cache if one is available.
if self.use_cache and self.api.cache and self.method == 'GET' and result:
self.api.cache.store(
'%s?%s' % (url, urlencode(self.session.params)), result)
return result
def execute(api,
method,
path,
*args,
allowed_param=[],
params=None,
headers=None,
json_payload=None,
parser=None,
payload_list=False,
payload_type=None,
post_data=None,
require_auth=False,
return_cursors=False,
upload_api=False,
use_cache=True,
**kwargs):
# If authentication is required and no credentials
# are provided, throw an error.
if require_auth and not api.auth:
raise TweepError('Authentication required!')
api.cached_result = False
# Build the request URL
if upload_api:
url = api.upload_root + path
full_url = 'https://' + api.upload_host + url
else:
url = api.api_root + path
full_url = 'https://' + api.host + url
if params is None:
params = {}
for idx, arg in enumerate(args):
if arg is None:
continue
try:
params[allowed_param[idx]] = str(arg)
except IndexError:
raise TweepError('Too many parameters supplied!')
for k, arg in kwargs.items():
if arg is None:
continue
if k in params:
raise TweepError(f'Multiple values for parameter {k} supplied!')
params[k] = str(arg)
log.debug("PARAMS: %r", params)
# Query the cache if one is available
# and this request uses a GET method.
if use_cache and api.cache and method == 'GET':
cache_result = api.cache.get(f'{url}?{urlencode(params)}')
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
if isinstance(result, Model):
result._api = api
else:
if isinstance(cache_result, Model):
cache_result._api = api
api.cached_result = True
return cache_result
# Monitoring rate limits
remaining_calls = None
reset_time = None
session = requests.Session()
if parser is None:
parser = api.parser
try:
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
while retries_performed <= api.retry_count:
if (api.wait_on_rate_limit and reset_time is not None
and remaining_calls is not None and remaining_calls < 1):
# Handle running out of API calls
sleep_time = reset_time - int(time.time())
if sleep_time > 0:
log.warning(
f"Rate limit reached. Sleeping for: {sleep_time}")
time.sleep(sleep_time + 1) # Sleep for extra sec
# Apply authentication
auth = None
if api.auth:
auth = api.auth.apply_auth()
# Execute request
try:
resp = session.request(method,
full_url,
params=params,
headers=headers,
data=post_data,
json=json_payload,
timeout=api.timeout,
auth=auth,
proxies=api.proxy)
except Exception as e:
raise TweepError(
f'Failed to send request: {e}').with_traceback(
sys.exc_info()[2])
if 200 <= resp.status_code < 300:
break
rem_calls = resp.headers.get('x-rate-limit-remaining')
if rem_calls is not None:
remaining_calls = int(rem_calls)
elif remaining_calls is not None:
remaining_calls -= 1
reset_time = resp.headers.get('x-rate-limit-reset')
if reset_time is not None:
reset_time = int(reset_time)
retry_delay = api.retry_delay
if resp.status_code in (420, 429) and api.wait_on_rate_limit:
if remaining_calls == 0:
# If ran out of calls before waiting switching retry last call
continue
if 'retry-after' in resp.headers:
retry_delay = float(resp.headers['retry-after'])
elif api.retry_errors and resp.status_code not in api.retry_errors:
# Exit request loop if non-retry error code
break
# Sleep before retrying request again
time.sleep(retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
api.last_response = resp
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg, api_error_code = parser.parse_error(resp.text)
except Exception:
error_msg = f"Twitter error response: status code = {resp.status_code}"
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, resp)
else:
raise TweepError(error_msg, resp, api_code=api_error_code)
# Parse the response payload
return_cursors = return_cursors or 'cursor' in params or 'next' in params
result = parser.parse(resp.text,
api=api,
payload_list=payload_list,
payload_type=payload_type,
return_cursors=return_cursors)
# Store result into cache if one is available.
if use_cache and api.cache and method == 'GET' and result:
api.cache.store(f'{url}?{urlencode(params)}', result)
return result
finally:
session.close()
def execute(self,
method,
path,
*,
headers=None,
json_payload=None,
post_data=None,
require_auth=False,
return_cursors=False,
upload_api=False,
use_cache=True):
# If authentication is required and no credentials
# are provided, throw an error.
if require_auth and not self.api.auth:
raise TweepError('Authentication required!')
self.api.cached_result = False
# Pick correct URL root to use
if upload_api:
api_root = self.api.upload_root
else:
api_root = self.api.api_root
if upload_api:
host = self.api.upload_host
else:
host = self.api.host
# Build the request URL
url = api_root + path
full_url = 'https://' + host + url
# Query the cache if one is available
# and this request uses a GET method.
if use_cache and self.api.cache and method == 'GET':
cache_result = self.api.cache.get(
f'{url}?{urlencode(self.session.params)}')
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
if isinstance(result, Model):
result._api = self.api
else:
if isinstance(cache_result, Model):
cache_result._api = self.api
self.api.cached_result = True
return cache_result
# Continue attempting request until successful
# or maximum number of retries is reached.
retries_performed = 0
while retries_performed < self.api.retry_count + 1:
if (self.api.wait_on_rate_limit and self._reset_time is not None
and self._remaining_calls is not None
and self._remaining_calls < 1):
# Handle running out of API calls
sleep_time = self._reset_time - int(time.time())
if sleep_time > 0:
if self.api.wait_on_rate_limit_notify:
log.warning(
f"Rate limit reached. Sleeping for: {sleep_time}")
time.sleep(sleep_time + 1) # Sleep for extra sec
# Apply authentication
auth = None
if self.api.auth:
auth = self.api.auth.apply_auth()
# Execute request
try:
resp = self.session.request(method,
full_url,
headers=headers,
data=post_data,
json=json_payload,
timeout=self.api.timeout,
auth=auth,
proxies=self.api.proxy)
except Exception as e:
raise TweepError(
f'Failed to send request: {e}').with_traceback(
sys.exc_info()[2])
if 200 <= resp.status_code < 300:
break
rem_calls = resp.headers.get('x-rate-limit-remaining')
if rem_calls is not None:
self._remaining_calls = int(rem_calls)
elif self._remaining_calls is not None:
self._remaining_calls -= 1
reset_time = resp.headers.get('x-rate-limit-reset')
if reset_time is not None:
self._reset_time = int(reset_time)
retry_delay = self.api.retry_delay
if resp.status_code in (420, 429) and self.api.wait_on_rate_limit:
if self._remaining_calls == 0:
# If ran out of calls before waiting switching retry last call
continue
if 'retry-after' in resp.headers:
retry_delay = float(resp.headers['retry-after'])
elif self.api.retry_errors and resp.status_code not in self.api.retry_errors:
# Exit request loop if non-retry error code
break
# Sleep before retrying request again
time.sleep(retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
self.api.last_response = resp
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg, api_error_code = self.parser.parse_error(resp.text)
except Exception:
error_msg = f"Twitter error response: status code = {resp.status_code}"
api_error_code = None
if is_rate_limit_error_message(error_msg):
raise RateLimitError(error_msg, resp)
else:
raise TweepError(error_msg, resp, api_code=api_error_code)
# Parse the response payload
return_cursors = (return_cursors or 'cursor' in self.session.params
or 'next' in self.session.params)
result = self.parser.parse(self,
resp.text,
return_cursors=return_cursors)
# Store result into cache if one is available.
if use_cache and self.api.cache and method == 'GET' and result:
self.api.cache.store(f'{url}?{urlencode(self.session.params)}',
result)
return result
