Esempio n. 1
0
 def task_all(self):
     try:
         return conn.llen(self.tasks.format(self.name)) +\
                 conn.llen(self.processing.format(self.name)) +\
                 conn.llen(self.dones.format(self.name))
     except:
         return 0
Esempio n. 2
0
 def task_all(self):
     try:
         return conn.llen(self.tasks.format(self.name)) +\
                 conn.llen(self.processing.format(self.name)) +\
                 conn.llen(self.dones.format(self.name))
     except:
         return 0
Esempio n. 3
0
    def work(self):
        time.sleep(random.random())
        while True:
            task = None

            # check status, sleep longer if not processing, break if finished
            status = self.status()
            #print self.name, status
            if status == 'F':
                break
            elif status in ['?']:
                time.sleep(random.randint(15, 30))
                continue

            try:
                if self.max_workers is not None and \
                    conn.llen(self.processing.format(self.name)) >= self.max_workers:
                    time.sleep(1)
                    continue

                result = conn.lpop(self.tasks.format(self.name))
                if result is None:
                    time.sleep(3)
                    continue

                task = result
                caller, args, kwargs = unpack(task)
                conn.rpush(self.processing.format(self.name), task)
                conn.set(self.updated_at.format(self.name),
                         time.mktime(time.gmtime()))

                print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
                if '.' in caller:
                    module, method = caller.rsplit('.', 1)
                    module = __import__(module, fromlist=[method])
                    caller = getattr(module, method)
                else:
                    method = caller
                    caller = sys.modules['__builtin__'].__dict__[method]
            except:
                print("can't obtain caller, locals: {}".format(locals()))
                traceback.print_exc()
                if task is not None:
                    self.finish_task(task)

                continue

            try:
                caller(*args, **kwargs)
            except:
                traceback.print_exc()
            finally:
                self.finish_task(task)
Esempio n. 4
0
    def work(self):
        time.sleep(random.random())
        while True:
            task = None

            # check status, sleep longer if not processing, break if finished
            status = self.status()
            #print self.name, status
            if status == 'F':
                break
            elif status in ['?']:
                time.sleep(random.randint(15, 30))
                continue

            try:
                if self.max_workers is not None and \
                    conn.llen(self.processing.format(self.name)) >= self.max_workers:
                    time.sleep(1)
                    continue

                result = conn.lpop(self.tasks.format(self.name))
                if result is None:
                    time.sleep(3)
                    continue

                task = result
                caller, args, kwargs = unpack(task)
                conn.rpush(self.processing.format(self.name), task)
                conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))

                print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
                if '.' in caller:
                    module, method = caller.rsplit('.', 1)
                    module = __import__(module, fromlist=[method])
                    caller = getattr(module, method)
                else:
                    method = caller
                    caller = sys.modules['__builtin__'].__dict__[method]
            except:
                print("can't obtain caller, locals: {}".format(locals()))
                traceback.print_exc()
                if task is not None:
                    self.finish_task(task)

                continue

            try:
                caller(*args, **kwargs)
            except:
                traceback.print_exc()
            finally:
                self.finish_task(task)
Esempio n. 5
0
 def task_left(self):
     try:
         return conn.llen(self.tasks.format(self.name)) + conn.llen(
             self.processing.format(self.name))
     except:
         return 0
Esempio n. 6
0
 def task_left(self):
     try:
         return conn.llen(self.tasks.format(self.name)) + conn.llen(self.processing.format(self.name))
     except:
         return 0