def test_writes(self):
cache = JsonCache('stuff')
cache.update(dict(pi=289/92))
with open(self._root.format('stuff'), 'r') as fd:
empty = fd.read()
self.assertEqual(empty, '{}')
cache.write()
with open(self._root.format('stuff'), 'r') as fd:
result = fd.read()
self.assertEqual(result, '{"pi": 3.141304347826087}')
class RateLimiter(BaseRateLimiter):
"""Twitter rate limiter."""
def __init__(self):
self._limits = JsonCache('twitter-ratelimiter')
def _sanitize_url(self, uri):
# Cache the URL sans any query parameters.
return uri.host + uri.path
def wait(self, message):
# If we haven't seen this URL, default to no wait.
seconds = self._limits.pop(self._sanitize_url(message.get_uri()), 0)
log.debug('Sleeping for {} seconds!'.format(seconds))
time.sleep(seconds)
# Don't sleep the same length of time more than once!
self._limits.write()
def update(self, message):
info = message.response_headers
url = self._sanitize_url(message.get_uri())
# This is time in the future, in UTC epoch seconds, at which the
# current rate limiting window expires.
rate_reset = info.get('X-Rate-Limit-Reset')
# This is the number of calls still allowed in this window.
rate_count = info.get('X-Rate-Limit-Remaining')
if None not in (rate_reset, rate_count):
rate_reset = int(rate_reset)
rate_count = int(rate_count)
rate_delta = abs(rate_reset - time.time())
if rate_count > 5:
# If there are more than 5 calls allowed in this window, then
# do no rate limiting.
pass
elif rate_count < 1:
# There are no calls remaining, so wait until the close of the
# current window.
self._limits[url] = rate_delta
else:
wait_secs = rate_delta / rate_count
self._limits[url] = wait_secs
log.debug('Next access to {} must wait {} seconds!'.format(
url, self._limits.get(url, 0)))