def finish_task(self, task):
try:
conn.lrem(self.processing.format(self.name), task, 1)
conn.rpush(self.dones.format(self.name), task)
conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))
except:
pass
def add_tasks(self, *tasktuples):
if tasktuples:
tasks = [
pack((caller, args, kwargs))
for caller, args, kwargs in tasktuples
]
conn.rpush(self.tasks.format(self.name), *tasks)
def finish_task(self, task):
try:
conn.lrem(self.processing.format(self.name), task, 1)
conn.rpush(self.dones.format(self.name), task)
conn.set(self.updated_at.format(self.name),
time.mktime(time.gmtime()))
except:
pass
def work(self):
time.sleep(random.random())
while True:
task = None
# check status, sleep longer if not processing, break if finished
status = self.status()
#print self.name, status
if status == 'F':
break
elif status in ['?']:
time.sleep(random.randint(15, 30))
continue
try:
if self.max_workers is not None and \
conn.llen(self.processing.format(self.name)) >= self.max_workers:
time.sleep(1)
continue
result = conn.lpop(self.tasks.format(self.name))
if result is None:
time.sleep(3)
continue
task = result
caller, args, kwargs = unpack(task)
conn.rpush(self.processing.format(self.name), task)
conn.set(self.updated_at.format(self.name),
time.mktime(time.gmtime()))
print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
if '.' in caller:
module, method = caller.rsplit('.', 1)
module = __import__(module, fromlist=[method])
caller = getattr(module, method)
else:
method = caller
caller = sys.modules['__builtin__'].__dict__[method]
except:
print("can't obtain caller, locals: {}".format(locals()))
traceback.print_exc()
if task is not None:
self.finish_task(task)
continue
try:
caller(*args, **kwargs)
except:
traceback.print_exc()
finally:
self.finish_task(task)
def work(self):
time.sleep(random.random())
while True:
task = None
# check status, sleep longer if not processing, break if finished
status = self.status()
#print self.name, status
if status == 'F':
break
elif status in ['?']:
time.sleep(random.randint(15, 30))
continue
try:
if self.max_workers is not None and \
conn.llen(self.processing.format(self.name)) >= self.max_workers:
time.sleep(1)
continue
result = conn.lpop(self.tasks.format(self.name))
if result is None:
time.sleep(3)
continue
task = result
caller, args, kwargs = unpack(task)
conn.rpush(self.processing.format(self.name), task)
conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))
print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
if '.' in caller:
module, method = caller.rsplit('.', 1)
module = __import__(module, fromlist=[method])
caller = getattr(module, method)
else:
method = caller
caller = sys.modules['__builtin__'].__dict__[method]
except:
print("can't obtain caller, locals: {}".format(locals()))
traceback.print_exc()
if task is not None:
self.finish_task(task)
continue
try:
caller(*args, **kwargs)
except:
traceback.print_exc()
finally:
self.finish_task(task)
def check_zombie(self):
try:
updated_at = conn.get(self.updated_at.format(self.name))
if updated_at:
updated_at = float(updated_at)
if (time.mktime(time.gmtime()) - updated_at) > 600:
while True:
task = conn.lpop(self.processing.format(self.name))
if task is None:
break
conn.rpush(self.dones.format(self.name), task)
except:
pass
def check_zombie(self):
try:
updated_at = conn.get(self.updated_at.format(self.name))
if updated_at:
updated_at = float(updated_at)
if (time.mktime(time.gmtime()) - updated_at) > 600:
while True:
task = conn.lpop(self.processing.format(self.name))
if task is None:
break
conn.rpush(self.dones.format(self.name), task)
except:
pass
def add_task(self, caller, *args, **kwargs):
print caller, args[:5], kwargs
conn.rpush(self.tasks.format(self.name), pack((caller, args, kwargs)))
def add_task(self, caller, *args, **kwargs):
print caller, args[:5], kwargs
conn.rpush(self.tasks.format(self.name), pack((caller, args, kwargs)))
def add_tasks(self, *tasktuples):
if tasktuples:
tasks = [ pack((caller, args, kwargs)) for caller, args, kwargs in tasktuples ]
conn.rpush(self.tasks.format(self.name), *tasks)