def rest_api_call(endpoint, auth, accept_codes, params): """ Call the rest api endpoint. """ # Add version info into the code params.setdefault("client_id", auth["client_id"]) params.setdefault("client_secret", auth["client_secret"]) params.setdefault("v", VERSION) params.setdefault("m", MODE) tries = 0 while tries < const.API_RETRY_MAX: r = req.get(endpoint, params=params, timeout=60.0) # Proper receive if 200 <= r.status_code < 300 or r.status_code in accept_codes: time.sleep(check_rate_limit(r.headers)) try: data = r.json() except ValueError: log.info(u"Try L1 {}: Falied to decode Json - {}\n{}", tries, r.status_code, r.text) tries += 1 continue return (data, r.status_code) # Check if rate limited if r.status_code in 403: log.info(u"Try L1 {}: Being throttled - {}\n{}", tries, r.status_code, r.text) time.sleep(check_rate_limit(r.headers)) tries += 1 continue # Server side error; Retry after delay if 500 <= r.status_code < 600: log.info(u"Try L1 {}: Server side error {}\n{}", tries, r.status_code, r.text) time.sleep(check_rate_limit(r.headers)) tries += 1 continue # Some other error; Break out of loop break # Give up raise FoursquareError(r, tries)
def check_limit(self, headers): """ Check if rate limit is hit for the current key. """ now = int(time.time()) sleep_time = check_rate_limit(headers) if sleep_time: log.debug("Key {} hit rate limit ...", self.idx) # Save the reset time self.reset[self.idx] = now + sleep_time # Move on to the next key self.idx = (self.idx + 1) % len(self.keys) # If the next key is also under ratelimit, # sleep off the rate limit window if self.reset[self.idx] > now: log.debug("Key {} still in rate limit ...", self.idx) time.sleep(self.reset[self.idx] - now)