class Worker(object):
def __init__(self, namespace, id):
self.namespace = namespace
self.throttle = Throttle(namespace, "60000/m", time_unit=1)
self.id = id
def work(self):
log("worker %s start work" % self.id)
namespace = self.namespace
redis = redis_conn.redis
while redis.llen(namespace):
tasks_count = self.throttle.throttle_task_size()
log("worker %s task count %s" % (self.id, tasks_count))
if tasks_count == 0:
idle = self.throttle.idle_seconds()
log("sleep for %s seconds" % idle)
time.sleep(idle)
else:
tokens = []
while tasks_count > 0:
token = redis_conn.redis.lpop(namespace)
tokens.append(token)
tasks_count -= 1
log("worker %s served %s tokens" % (self.id, len(tokens)))
self.throttle.finish_throttling()
def run(self):
log("run worker %s" % self.id)
return gevent.spawn(self.work)
def test_throttle_should_tell_how_long_it_should_be_idle(self):
rate_limit = "60000/m"
t = Throttle(self.throttle_namespace, rate_limit)
self.assertEquals(60, t.idle_seconds(600))
self.assertEquals(59, t.idle_seconds(601))
self.assertEquals(1, t.idle_seconds(599))
def test_throttle_should_tell_how_long_it_should_be_idle(self):
rate_limit = "60000/m"
t = Throttle(self.throttle_namespace, rate_limit)
self.assertEquals(60, t.idle_seconds(600))
self.assertEquals(59, t.idle_seconds(601))
self.assertEquals(1, t.idle_seconds(599))
