def iter_content(self, chunk_size=1, decode_unicode=False):
"""Request's default response object will read 10k bytes at a time.
It used to read only 1 byte at a time. Instead of patching requests
to change the behavior, we hardcode the chunk size here to 500k."""
# get a chunk_size from the settings value "transfer_chunk_size";
chunk_size = settings.get("transfer_chunk_size", 500*1024)
return super(Response, self).iter_content(chunk_size, decode_unicode)
self.key = md5(resource).hexdigest()
limits[resource] = self
def token(self):
for limit, (rate, interval) in self.limits.iteritems():
timestamps = window(interval)
keys = [self.key+timestamp for timestamp in timestamps]
results = ratelimit_cache.get_multi(*keys)
gets = sum(map(int, results.values()))
if gets >= rate:
logger.error("RateLimit %s exceeded for %s" % (limit, self.resource))
return False
if self.key+timestamps[0] not in results:
ratelimit_cache.add(self.key+timestamps[0], '0')
ratelimit_cache.incr(self.key+timestamps[0], 1)
return True
if not settings.get("disable_ratelimit", False):
ratelimit_cache = Memcached(**settings.like("ratelimit_cache"))
else:
ratelimit_cache = None
def enable():
global ratelimit_cache
ratelimit_cache = Memcached(**settings.like("ratelimit_cache"))
def disable():
global ratelimit_cache
ratelimit_cache = None
def token(self):
for limit, (rate, interval) in self.limits.iteritems():
timestamps = window(interval)
keys = [self.key + timestamp for timestamp in timestamps]
results = ratelimit_cache.get_multi(*keys)
gets = sum(map(int, results.values()))
if gets >= rate:
logger.error("RateLimit %s exceeded for %s" %
(limit, self.resource))
return False
if self.key + timestamps[0] not in results:
ratelimit_cache.add(self.key + timestamps[0], '0')
ratelimit_cache.incr(self.key + timestamps[0], 1)
return True
if not settings.get("disable_ratelimit", False):
ratelimit_cache = Memcached(**settings.like("ratelimit_cache"))
else:
ratelimit_cache = None
def enable():
global ratelimit_cache
ratelimit_cache = Memcached(**settings.like("ratelimit_cache"))
def disable():
global ratelimit_cache
ratelimit_cache = None