Пример #1
0
 def finish_task(self, task):
     try:
         conn.lrem(self.processing.format(self.name), task, 1)
         conn.rpush(self.dones.format(self.name), task)
         conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))
     except:
         pass
Пример #2
0
 def add_tasks(self, *tasktuples):
     if tasktuples:
         tasks = [
             pack((caller, args, kwargs))
             for caller, args, kwargs in tasktuples
         ]
         conn.rpush(self.tasks.format(self.name), *tasks)
Пример #3
0
 def finish_task(self, task):
     try:
         conn.lrem(self.processing.format(self.name), task, 1)
         conn.rpush(self.dones.format(self.name), task)
         conn.set(self.updated_at.format(self.name),
                  time.mktime(time.gmtime()))
     except:
         pass
Пример #4
0
    def work(self):
        time.sleep(random.random())
        while True:
            task = None

            # check status, sleep longer if not processing, break if finished
            status = self.status()
            #print self.name, status
            if status == 'F':
                break
            elif status in ['?']:
                time.sleep(random.randint(15, 30))
                continue

            try:
                if self.max_workers is not None and \
                    conn.llen(self.processing.format(self.name)) >= self.max_workers:
                    time.sleep(1)
                    continue

                result = conn.lpop(self.tasks.format(self.name))
                if result is None:
                    time.sleep(3)
                    continue

                task = result
                caller, args, kwargs = unpack(task)
                conn.rpush(self.processing.format(self.name), task)
                conn.set(self.updated_at.format(self.name),
                         time.mktime(time.gmtime()))

                print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
                if '.' in caller:
                    module, method = caller.rsplit('.', 1)
                    module = __import__(module, fromlist=[method])
                    caller = getattr(module, method)
                else:
                    method = caller
                    caller = sys.modules['__builtin__'].__dict__[method]
            except:
                print("can't obtain caller, locals: {}".format(locals()))
                traceback.print_exc()
                if task is not None:
                    self.finish_task(task)

                continue

            try:
                caller(*args, **kwargs)
            except:
                traceback.print_exc()
            finally:
                self.finish_task(task)
Пример #5
0
    def work(self):
        time.sleep(random.random())
        while True:
            task = None

            # check status, sleep longer if not processing, break if finished
            status = self.status()
            #print self.name, status
            if status == 'F':
                break
            elif status in ['?']:
                time.sleep(random.randint(15, 30))
                continue

            try:
                if self.max_workers is not None and \
                    conn.llen(self.processing.format(self.name)) >= self.max_workers:
                    time.sleep(1)
                    continue

                result = conn.lpop(self.tasks.format(self.name))
                if result is None:
                    time.sleep(3)
                    continue

                task = result
                caller, args, kwargs = unpack(task)
                conn.rpush(self.processing.format(self.name), task)
                conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))

                print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
                if '.' in caller:
                    module, method = caller.rsplit('.', 1)
                    module = __import__(module, fromlist=[method])
                    caller = getattr(module, method)
                else:
                    method = caller
                    caller = sys.modules['__builtin__'].__dict__[method]
            except:
                print("can't obtain caller, locals: {}".format(locals()))
                traceback.print_exc()
                if task is not None:
                    self.finish_task(task)

                continue

            try:
                caller(*args, **kwargs)
            except:
                traceback.print_exc()
            finally:
                self.finish_task(task)
Пример #6
0
 def check_zombie(self):
     try:
         updated_at = conn.get(self.updated_at.format(self.name))
         if updated_at:
             updated_at = float(updated_at)
             if (time.mktime(time.gmtime()) - updated_at) > 600:
                 while True:
                     task = conn.lpop(self.processing.format(self.name))
                     if task is None:
                         break
                     conn.rpush(self.dones.format(self.name), task)
     except:
         pass
Пример #7
0
 def check_zombie(self):
     try:
         updated_at = conn.get(self.updated_at.format(self.name))
         if updated_at:
             updated_at = float(updated_at)
             if (time.mktime(time.gmtime()) - updated_at) > 600:
                 while True:
                     task = conn.lpop(self.processing.format(self.name))
                     if task is None:
                         break
                     conn.rpush(self.dones.format(self.name), task)
     except:
         pass
Пример #8
0
 def add_task(self, caller, *args, **kwargs):
     print caller, args[:5], kwargs
     conn.rpush(self.tasks.format(self.name), pack((caller, args, kwargs)))
Пример #9
0
 def add_task(self, caller, *args, **kwargs):
     print caller, args[:5], kwargs
     conn.rpush(self.tasks.format(self.name), pack((caller, args, kwargs)))
Пример #10
0
 def add_tasks(self, *tasktuples):
     if tasktuples:
         tasks = [ pack((caller, args, kwargs)) for caller, args, kwargs in tasktuples ]
         conn.rpush(self.tasks.format(self.name), *tasks)