def __init__(self, name, db_conf=None, timeout=5):
self.name = name
self.waitting_queue = C_queue(self.make_name("waitting_queue"), db_conf)
self.running_queue = C_queue(self.make_name("running_queue"), db_conf)
self.result_store = C_kvstore(self.make_name("result"), db_conf)
self.finished_task = C_set(self.make_name("finished_task"), db_conf)
gevent.spawn(self.cheanup_let, timeout)
def __init__(self, name, db_conf=None, timeout=5):
self.name = name
self.waitting_queue = C_queue(self.make_name("waitting_queue"),
db_conf)
self.running_queue = C_queue(self.make_name("running_queue"), db_conf)
self.result_store = C_kvstore(self.make_name("result"), db_conf)
self.finished_task = C_set(self.make_name("finished_task"), db_conf)
gevent.spawn(self.cheanup_let, timeout)
class TaskQueue(object):
def __init__(self, name, db_conf=None, timeout=5):
self.name = name
self.waitting_queue = C_queue(self.make_name("waitting_queue"),
db_conf)
self.running_queue = C_queue(self.make_name("running_queue"), db_conf)
self.result_store = C_kvstore(self.make_name("result"), db_conf)
self.finished_task = C_set(self.make_name("finished_task"), db_conf)
gevent.spawn(self.cheanup_let, timeout)
def make_name(self, name):
return "%s_%s" % (self.name, name)
def push_task(self, task, timeout=60):
key = gen_key()
if self.waitting_queue.push((time(), timeout, (key, task))):
return key
def pop_task(self):
while True:
put_time, timeout, item = self.waitting_queue.pop()
if time() - put_time < timeout:
break
self.running_queue.push((time(), item))
return item
def put_result(self, key, result, ttl=None):
return self.result_store.put(key, result, ttl)
def get_result(self, key):
return self.result_store.get(key)
def finish_task(self, key):
self.finished_task.add(key)
def cheanup_let(self, timeout):
while True:
start_time, item = self.running_queue.rpop()
delta = time() - start_time
if delta < timeout:
sleep(timeout - delta)
k, _ = item
if k not in self.finished_task:
self.waitting_queue.lpush(item)
else:
self.finished_task.remove(k)
class TaskQueue(object):
def __init__(self, name, db_conf=None, timeout=5):
self.name = name
self.waitting_queue = C_queue(self.make_name("waitting_queue"), db_conf)
self.running_queue = C_queue(self.make_name("running_queue"), db_conf)
self.result_store = C_kvstore(self.make_name("result"), db_conf)
self.finished_task = C_set(self.make_name("finished_task"), db_conf)
gevent.spawn(self.cheanup_let, timeout)
def make_name(self, name):
return "%s_%s" % (self.name, name)
def push_task(self, task, timeout=60):
key = gen_key()
if self.waitting_queue.push((time(), timeout, (key, task))):
return key
def pop_task(self):
while True:
put_time, timeout, item = self.waitting_queue.pop()
if time()-put_time < timeout:
break
self.running_queue.push((time(), item))
return item
def put_result(self, key, result, ttl=None):
return self.result_store.put(key, result, ttl)
def get_result(self, key):
return self.result_store.get(key)
def finish_task(self, key):
self.finished_task.add(key)
def cheanup_let(self, timeout):
while True:
start_time, item = self.running_queue.rpop()
delta = time()-start_time
if delta < timeout:
sleep(timeout-delta)
k,_ = item
if k not in self.finished_task:
self.waitting_queue.lpush(item)
else:
self.finished_task.remove(k)
