class Worker(object):
def __init__(self, namespace, id):
self.namespace = namespace
self.throttle = Throttle(namespace, "60000/m", time_unit=1)
self.id = id
def work(self):
log("worker %s start work" % self.id)
namespace = self.namespace
redis = redis_conn.redis
while redis.llen(namespace):
tasks_count = self.throttle.throttle_task_size()
log("worker %s task count %s" % (self.id, tasks_count))
if tasks_count == 0:
idle = self.throttle.idle_seconds()
log("sleep for %s seconds" % idle)
time.sleep(idle)
else:
tokens = []
while tasks_count > 0:
token = redis_conn.redis.lpop(namespace)
tokens.append(token)
tasks_count -= 1
log("worker %s served %s tokens" % (self.id, len(tokens)))
self.throttle.finish_throttling()
def run(self):
log("run worker %s" % self.id)
return gevent.spawn(self.work)
def test_asking_for_tasks_should_give_a_chunk_and_change_current_rate_count_when_rate_is_small(
self):
rate_limit = "1/m"
t = Throttle(self.throttle_namespace, rate_limit)
# chunk size is 600
self.assertEquals(1, t.throttle_task_size())
def test_asking_for_tasks_when_it_exceed_rate_limit(self):
rate_limit = "60000/m"
t = Throttle(self.throttle_namespace, rate_limit)
timestamp = 600
redis_conn.redis.set(t.current_time_bucket_key(timestamp), 60001)
self.assertEquals(0, t.throttle_task_size(timestamp))
def test_asking_for_tasks_should_give_a_chunk_and_change_current_rate_count_when_rate_is_small(self):
rate_limit = "1/m"
t = Throttle(self.throttle_namespace, rate_limit)
# chunk size is 600
self.assertEquals(1, t.throttle_task_size())
def test_asking_for_tasks_when_it_exceed_rate_limit(self):
rate_limit = "60000/m"
t = Throttle(self.throttle_namespace, rate_limit)
timestamp = 600
redis_conn.redis.set(t.current_time_bucket_key(timestamp), 60001)
self.assertEquals(0, t.throttle_task_size(timestamp))
