def finish_task(self, task):
try:
conn.lrem(self.processing.format(self.name), task, 1)
conn.rpush(self.dones.format(self.name), task)
conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))
except:
pass
def finish_task(self, task):
try:
conn.lrem(self.processing.format(self.name), task, 1)
conn.rpush(self.dones.format(self.name), task)
conn.set(self.updated_at.format(self.name),
time.mktime(time.gmtime()))
except:
pass
def work(self):
time.sleep(random.random())
while True:
task = None
# check status, sleep longer if not processing, break if finished
status = self.status()
#print self.name, status
if status == 'F':
break
elif status in ['?']:
time.sleep(random.randint(15, 30))
continue
try:
if self.max_workers is not None and \
conn.llen(self.processing.format(self.name)) >= self.max_workers:
time.sleep(1)
continue
result = conn.lpop(self.tasks.format(self.name))
if result is None:
time.sleep(3)
continue
task = result
caller, args, kwargs = unpack(task)
conn.rpush(self.processing.format(self.name), task)
conn.set(self.updated_at.format(self.name),
time.mktime(time.gmtime()))
print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
if '.' in caller:
module, method = caller.rsplit('.', 1)
module = __import__(module, fromlist=[method])
caller = getattr(module, method)
else:
method = caller
caller = sys.modules['__builtin__'].__dict__[method]
except:
print("can't obtain caller, locals: {}".format(locals()))
traceback.print_exc()
if task is not None:
self.finish_task(task)
continue
try:
caller(*args, **kwargs)
except:
traceback.print_exc()
finally:
self.finish_task(task)
def work(self):
time.sleep(random.random())
while True:
task = None
# check status, sleep longer if not processing, break if finished
status = self.status()
#print self.name, status
if status == 'F':
break
elif status in ['?']:
time.sleep(random.randint(15, 30))
continue
try:
if self.max_workers is not None and \
conn.llen(self.processing.format(self.name)) >= self.max_workers:
time.sleep(1)
continue
result = conn.lpop(self.tasks.format(self.name))
if result is None:
time.sleep(3)
continue
task = result
caller, args, kwargs = unpack(task)
conn.rpush(self.processing.format(self.name), task)
conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))
print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
if '.' in caller:
module, method = caller.rsplit('.', 1)
module = __import__(module, fromlist=[method])
caller = getattr(module, method)
else:
method = caller
caller = sys.modules['__builtin__'].__dict__[method]
except:
print("can't obtain caller, locals: {}".format(locals()))
traceback.print_exc()
if task is not None:
self.finish_task(task)
continue
try:
caller(*args, **kwargs)
except:
traceback.print_exc()
finally:
self.finish_task(task)
def finish_generation(self):
conn.set(self.generated.format(self.name), 'true')
conn.set(self.started_at.format(self.name), time.mktime(time.gmtime()))
def finish_generation(self):
conn.set(self.generated.format(self.name), 'true')
conn.set(self.started_at.format(self.name), time.mktime(time.gmtime()))