def do_lookup(auth, urls, data): """ Do the actual call. """ # Generate the get params params = {"client": CLIENT, "appver": 0.1, "apikey": auth, "pver": PVER} # Make the request tries = 0 while tries < const.API_RETRY_MAX: r = req.post(ENDPOINT, params=params, data=data, timeout=60.0) # We have at least one match if r.status_code == 200: ret = r.text ret = ret.split("\n") ret = dict(zip(urls, ret)) return ret, r.status_code # All the urls are good if r.status_code == 204: return {u: "ok" for u in urls}, r.status_code # Server side error Retry if 500 <= r.status_code < 600: log.info(u"Try L1 {}: Server side error {} {}", tries, r.status_code, r.text) time.sleep(const.API_RETRY_AFTER) # Some other error break raise GoogleSafeBrowsingError(r, tries)
def rest_call(endpoint, auth, params, method="get"): """ Call a Twitter rest api endpoint. """ ## DEBUG # print(endpoint) args = {"auth": auth.oauth, "session": auth.session, "timeout": 60.0} tries = 0 while tries < const.API_RETRY_MAX: if method == "get": r = req.get(endpoint, params=params, **args) elif method == "post": r = req.post(endpoint, data=params, **args) else: raise ValueError("Invalid value for parameter 'method'") # Proper receive if 200 <= r.status_code < 300: auth.check_limit(r.headers) try: data = r.json() except ValueError: log.info(u"Try L1 {}: Falied to decode JSON - {}\n{}", tries, r.status_code, r.text) tries += 1 continue return (data, r.status_code, 0) log.info(u"Try L1 {}: Received error", tries) todo, status_code, error_code = ec.get_error_todo(r) if todo is ec.RETRY: auth.check_limit(r.headers) tries += 1 continue elif todo is ec.SKIP_AND_RETRY: auth.skip_key() tries += 1 continue elif todo is ec.GIVEUP: try: data = r.json() except ValueError: data = {"response_text": r.text} return (data, status_code, error_code) else: raise RuntimeError("This should not be reached!") raise Error("Tries exhausted: %d" % tries)
def rest_call(endpoint, auth, accept_codes, params, method="get"): """ Call a Twitter rest api endpoint. """ ## DEBUG # print(endpoint) args = {"auth": auth.oauth, "session": auth.session, "timeout": 60.0} tries = 0 while tries < const.API_RETRY_MAX: if method == "get": r = req.get(endpoint, params=params, **args) elif method == "post": r = req.post(endpoint, data=params, **args) else: raise ValueError("Invalid value for parameter 'method'") # Proper receive if 200 <= r.status_code < 300 or r.status_code in accept_codes: auth.check_limit(r.headers) try: data = r.json() except ValueError: log.info(u"Try L1 {}: Falied to decode Json - {}\n{}", tries, r.status_code, r.text) tries += 1 continue return (data, r.status_code) # Check if rate limited if r.status_code in (431, 429): log.info(u"Try L1 {}: Being throttled - {}\n{}", tries, r.status_code, r.text) auth.check_limit(r.headers) tries += 1 continue # Server side error; Retry after delay if 500 <= r.status_code < 600: log.info(u"Try L1 {}: Server side error {}\n{}", tries, r.status_code, r.text) auth.check_limit(r.headers) tries += 1 continue # Some other error; Break out of loop break raise TwitterRestAPIError(r, tries)
def stream_call(endpoint, auth, params, method): """ Do the streaming api. """ auth = OAuth1(signature_type="auth_header", **auth.token) # Enter the infinite loop while True: if method == "get": r = req.get(endpoint, params=params, auth=auth, timeout=60.0, stream=True) elif method == "post": r = req.post(endpoint, data=params, auth=auth, timeout=60.0, stream=True) else: raise ValueError("Invalid value for parameter 'method'") if r.status_code == 200: # Loop over the lines try: for line in r.iter_lines(): if line: yield line except ssl.SSLError as e: log.info(u"ssl.SSLError - {}", e) except httplib.IncompleteRead as e: log.info(u"httplib.IncompleteRead - {}", e) except Exception: # pylint: disable=broad-except log.warn(u"Unexepectd exception", exc_info=True) else: # Dont expect anything else msg = u"Unexepectd response - {0}".format(r.status_code) log.warn(msg) try: for line in r.iter_lines(): log.info(line.strip()) except Exception: # pylint: disable=broad-except msg = (u"Unexepectd exception " u"while processing unexpected response.") log.warn(msg, exc_info=True) # Try to sleep over the problem sleep(const.API_RETRY_AFTER)