def rest_call(endpoint, auth, params, method="get"):
"""
Call a Twitter rest api endpoint.
"""
## DEBUG
# print(endpoint)
args = {"auth": auth.oauth, "session": auth.session, "timeout": 60.0}
tries = 0
while tries < const.API_RETRY_MAX:
if method == "get":
r = req.get(endpoint, params=params, **args)
elif method == "post":
r = req.post(endpoint, data=params, **args)
else:
raise ValueError("Invalid value for parameter 'method'")
# Proper receive
if 200 <= r.status_code < 300:
auth.check_limit(r.headers)
try:
data = r.json()
except ValueError:
log.info(u"Try L1 {}: Falied to decode JSON - {}\n{}",
tries, r.status_code, r.text)
tries += 1
continue
return (data, r.status_code, 0)
log.info(u"Try L1 {}: Received error", tries)
todo, status_code, error_code = ec.get_error_todo(r)
if todo is ec.RETRY:
auth.check_limit(r.headers)
tries += 1
continue
elif todo is ec.SKIP_AND_RETRY:
auth.skip_key()
tries += 1
continue
elif todo is ec.GIVEUP:
try:
data = r.json()
except ValueError:
data = {"response_text": r.text}
return (data, status_code, error_code)
else:
raise RuntimeError("This should not be reached!")
raise Error("Tries exhausted: %d" % tries)
def rest_call(endpoint, auth, params, method="get"):
"""
Call a Twitter rest api endpoint.
"""
## DEBUG
# print(endpoint)
args = {"auth": auth.oauth, "session": auth.session, "timeout": 60.0}
tries = 0
while tries < const.API_RETRY_MAX:
if method == "get":
r = req.get(endpoint, params=params, **args)
elif method == "post":
r = req.post(endpoint, data=params, **args)
else:
raise ValueError("Invalid value for parameter 'method'")
# Proper receive
if 200 <= r.status_code < 300:
auth.check_limit(r.headers)
try:
data = r.json()
except ValueError:
log.info(u"Try L1 {}: Falied to decode JSON - {}\n{}", tries,
r.status_code, r.text)
tries += 1
continue
return (data, r.status_code, 0)
log.info(u"Try L1 {}: Received error", tries)
todo, status_code, error_code = ec.get_error_todo(r)
if todo is ec.RETRY:
auth.check_limit(r.headers)
tries += 1
continue
elif todo is ec.SKIP_AND_RETRY:
auth.skip_key()
tries += 1
continue
elif todo is ec.GIVEUP:
try:
data = r.json()
except ValueError:
data = {"response_text": r.text}
return (data, status_code, error_code)
else:
raise RuntimeError("This should not be reached!")
raise Error("Tries exhausted: %d" % tries)
def rest_call(endpoint, auth, accept_codes, params, method="get"):
"""
Call a Twitter rest api endpoint.
"""
## DEBUG
# print(endpoint)
args = {"auth": auth.oauth, "session": auth.session, "timeout": 60.0}
tries = 0
while tries < const.API_RETRY_MAX:
if method == "get":
r = req.get(endpoint, params=params, **args)
elif method == "post":
r = req.post(endpoint, data=params, **args)
else:
raise ValueError("Invalid value for parameter 'method'")
# Proper receive
if 200 <= r.status_code < 300 or r.status_code in accept_codes:
auth.check_limit(r.headers)
try:
data = r.json()
except ValueError:
log.info(u"Try L1 {}: Falied to decode Json - {}\n{}", tries,
r.status_code, r.text)
tries += 1
continue
return (data, r.status_code)
# Check if rate limited
if r.status_code in (431, 429):
log.info(u"Try L1 {}: Being throttled - {}\n{}", tries,
r.status_code, r.text)
auth.check_limit(r.headers)
tries += 1
continue
# Server side error; Retry after delay
if 500 <= r.status_code < 600:
log.info(u"Try L1 {}: Server side error {}\n{}", tries,
r.status_code, r.text)
auth.check_limit(r.headers)
tries += 1
continue
# Some other error; Break out of loop
break
raise TwitterRestAPIError(r, tries)
def stream_call(endpoint, auth, params, method):
"""
Do the streaming api.
"""
auth = OAuth1(signature_type="auth_header", **auth.token)
# Enter the infinite loop
while True:
if method == "get":
r = req.get(endpoint,
params=params,
auth=auth,
timeout=60.0,
stream=True)
elif method == "post":
r = req.post(endpoint,
data=params,
auth=auth,
timeout=60.0,
stream=True)
else:
raise ValueError("Invalid value for parameter 'method'")
if r.status_code == 200:
# Loop over the lines
try:
for line in r.iter_lines():
if line:
yield line
except ssl.SSLError as e:
log.info(u"ssl.SSLError - {}", e)
except httplib.IncompleteRead as e:
log.info(u"httplib.IncompleteRead - {}", e)
except Exception: # pylint: disable=broad-except
log.warn(u"Unexepectd exception", exc_info=True)
else: # Dont expect anything else
msg = u"Unexepectd response - {0}".format(r.status_code)
log.warn(msg)
try:
for line in r.iter_lines():
log.info(line.strip())
except Exception: # pylint: disable=broad-except
msg = (u"Unexepectd exception "
u"while processing unexpected response.")
log.warn(msg, exc_info=True)
# Try to sleep over the problem
sleep(const.API_RETRY_AFTER)
def rest_api_call(endpoint, auth, accept_codes, params):
"""
Call the rest api endpoint.
"""
# Add version info into the code
params.setdefault("client_id", auth["client_id"])
params.setdefault("client_secret", auth["client_secret"])
params.setdefault("v", VERSION)
params.setdefault("m", MODE)
tries = 0
while tries < const.API_RETRY_MAX:
r = req.get(endpoint, params=params, timeout=60.0)
# Proper receive
if 200 <= r.status_code < 300 or r.status_code in accept_codes:
time.sleep(check_rate_limit(r.headers))
try:
data = r.json()
except ValueError:
log.info(u"Try L1 {}: Falied to decode Json - {}\n{}",
tries, r.status_code, r.text)
tries += 1
continue
return (data, r.status_code)
# Check if rate limited
if r.status_code in 403:
log.info(u"Try L1 {}: Being throttled - {}\n{}",
tries, r.status_code, r.text)
time.sleep(check_rate_limit(r.headers))
tries += 1
continue
# Server side error; Retry after delay
if 500 <= r.status_code < 600:
log.info(u"Try L1 {}: Server side error {}\n{}",
tries, r.status_code, r.text)
time.sleep(check_rate_limit(r.headers))
tries += 1
continue
# Some other error; Break out of loop
break
# Give up
raise FoursquareError(r, tries)
def stream_call(endpoint, auth, params, method):
"""
Do the streaming api.
"""
auth = OAuth1(signature_type="auth_header", **auth.token)
# Enter the infinite loop
while True:
if method == "get":
r = req.get(endpoint, params=params, auth=auth,
timeout=60.0, stream=True)
elif method == "post":
r = req.post(endpoint, data=params, auth=auth,
timeout=60.0, stream=True)
else:
raise ValueError("Invalid value for parameter 'method'")
if r.status_code == 200:
# Loop over the lines
try:
for line in r.iter_lines():
if line:
yield line
except ssl.SSLError as e:
log.info(u"ssl.SSLError - {}", e)
except httplib.IncompleteRead as e:
log.info(u"httplib.IncompleteRead - {}", e)
except Exception: # pylint: disable=broad-except
log.warn(u"Unexepectd exception", exc_info=True)
else: # Dont expect anything else
msg = u"Unexepectd response - {0}".format(r.status_code)
log.warn(msg)
try:
for line in r.iter_lines():
log.info(line.strip())
except Exception: # pylint: disable=broad-except
msg = (u"Unexepectd exception "
u"while processing unexpected response.")
log.warn(msg, exc_info=True)
# Try to sleep over the problem
sleep(const.API_RETRY_AFTER)
def search(auth, query, **params):
"""
Return the results web search query from Bing.
"""
auth = "Basic " + b64encode(":" + auth)
auth_header = {"Authorization": auth}
params.setdefault("$format", "json")
params.setdefault("Query", "'%s'" % query)
tries = 0
while tries < const.API_RETRY_MAX:
r = req.get(ENDPOINT, params=params, headers=auth_header, timeout=60.0)
# Proper receive
if 200 <= r.status_code < 300:
try:
data = r.json()
except ValueError:
log.info(u"Try L1 {}: Falied to decode JSON - {}\n{}", tries,
r.status_code, r.text)
tries += 1
continue
return (data, r.status_code)
# Server side error; Retry after delay
if 500 <= r.status_code < 600:
log.info(u"Try L1 {}: Server side error {}\n{}", tries,
r.status_code, r.text)
time.sleep(const.API_RETRY_AFTER)
tries += 1
continue
# Some other error; Break out of loop
break
# Give up
raise AzureError(r, tries)
def search(auth, query, **params):
"""
Return the results web search query from Bing.
"""
auth = "Basic " + b64encode(":" + auth)
auth_header = {"Authorization": auth}
params.setdefault("$format", "json")
params.setdefault("Query", "'%s'" % query)
tries = 0
while tries < const.API_RETRY_MAX:
r = req.get(ENDPOINT, params=params, headers=auth_header, timeout=60.0)
# Proper receive
if 200 <= r.status_code < 300:
try:
data = r.json()
except ValueError:
log.info(u"Try L1 {}: Falied to decode JSON - {}\n{}",
tries, r.status_code, r.text)
tries += 1
continue
return (data, r.status_code)
# Server side error; Retry after delay
if 500 <= r.status_code < 600:
log.info(u"Try L1 {}: Server side error {}\n{}",
tries, r.status_code, r.text)
time.sleep(const.API_RETRY_AFTER)
tries += 1
continue
# Some other error; Break out of loop
break
# Give up
raise AzureError(r, tries)
