def check_rate_limit(r): """ Check the rate limit and sleep off if it is hit. r - The response object from requests. """ try: # If we hit the rate limit sleep remain = r.headers["x-rate-limit-remaining"] remain = int(remain) if remain <= RATE_LIMIT_BUFFER: log.debug("Hit rate limit - {}", remain) now = r.headers["date"] now = times.parse(now) now = times.to_unix(now) reset = r.headers["x-rate-limit-reset"] reset = int(reset) # Sleep past the reset time log.debug("Rate limit reset in {} seconds", reset - now) sleep(reset - now + RESET_BUFFER) except KeyError as e: # We dont have the proper headers log.error("Header not found - {}", e) sleep(FAILURE_RETRY)
def forwards(self, orm): "Write your forwards methods here." for item in orm.ContentItem.objects.all(): item.displayed_from = item.source_posted_at if item.feed.source_type.lower() == 'ics': content = json.loads(item.source_content) item.displayed_until = times.parse(content['DTEND']) item.save()
def rest_rate_limit(r): """ Check the rate limit and sleep it off if hit. """ try: #limit = int(r.headers["X-Rate-Limit-Limit"]) remain = int(r.headers["X-Rate-Limit-Remaining"]) reset = int(r.headers["X-Rate-Limit-Reset"]) curtime = times.to_unix(times.parse(r.headers["date"])) except KeyError as e: # We dont have the proper headers log.error("Header not found - {}", e) sleep(RETRY_AFTER) return if remain <= RATE_LIMIT_BUFFER: log.debug("Hit rate limit - {}", remain) log.debug("Rate limit reset in {} seconds", reset - curtime) sleep(reset - curtime + RESET_BUFFER)
def make_local_timestamp(timestamp, timezone='Europe/Madrid'): if not timestamp: return None if isinstance(timestamp, basestring): timestamp = times.parse(timestamp.replace('Z', '')) return times.to_local(timestamp, timezone).strftime('%Y-%m-%d %H:%M:%S')