コード例 #1
0
ファイル: processes.py プロジェクト: mobishift2011/data007
 def finish_task(self, task):
     try:
         conn.lrem(self.processing.format(self.name), task, 1)
         conn.rpush(self.dones.format(self.name), task)
         conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))
     except:
         pass
コード例 #2
0
ファイル: processes.py プロジェクト: yankaics/data007
 def finish_task(self, task):
     try:
         conn.lrem(self.processing.format(self.name), task, 1)
         conn.rpush(self.dones.format(self.name), task)
         conn.set(self.updated_at.format(self.name),
                  time.mktime(time.gmtime()))
     except:
         pass
コード例 #3
0
ファイル: processes.py プロジェクト: yankaics/data007
    def work(self):
        time.sleep(random.random())
        while True:
            task = None

            # check status, sleep longer if not processing, break if finished
            status = self.status()
            #print self.name, status
            if status == 'F':
                break
            elif status in ['?']:
                time.sleep(random.randint(15, 30))
                continue

            try:
                if self.max_workers is not None and \
                    conn.llen(self.processing.format(self.name)) >= self.max_workers:
                    time.sleep(1)
                    continue

                result = conn.lpop(self.tasks.format(self.name))
                if result is None:
                    time.sleep(3)
                    continue

                task = result
                caller, args, kwargs = unpack(task)
                conn.rpush(self.processing.format(self.name), task)
                conn.set(self.updated_at.format(self.name),
                         time.mktime(time.gmtime()))

                print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
                if '.' in caller:
                    module, method = caller.rsplit('.', 1)
                    module = __import__(module, fromlist=[method])
                    caller = getattr(module, method)
                else:
                    method = caller
                    caller = sys.modules['__builtin__'].__dict__[method]
            except:
                print("can't obtain caller, locals: {}".format(locals()))
                traceback.print_exc()
                if task is not None:
                    self.finish_task(task)

                continue

            try:
                caller(*args, **kwargs)
            except:
                traceback.print_exc()
            finally:
                self.finish_task(task)
コード例 #4
0
ファイル: processes.py プロジェクト: mobishift2011/data007
    def work(self):
        time.sleep(random.random())
        while True:
            task = None

            # check status, sleep longer if not processing, break if finished
            status = self.status()
            #print self.name, status
            if status == 'F':
                break
            elif status in ['?']:
                time.sleep(random.randint(15, 30))
                continue

            try:
                if self.max_workers is not None and \
                    conn.llen(self.processing.format(self.name)) >= self.max_workers:
                    time.sleep(1)
                    continue

                result = conn.lpop(self.tasks.format(self.name))
                if result is None:
                    time.sleep(3)
                    continue

                task = result
                caller, args, kwargs = unpack(task)
                conn.rpush(self.processing.format(self.name), task)
                conn.set(self.updated_at.format(self.name), time.mktime(time.gmtime()))

                print('work on {}, {}, {}'.format(caller, args[:5], kwargs))
                if '.' in caller:
                    module, method = caller.rsplit('.', 1)
                    module = __import__(module, fromlist=[method])
                    caller = getattr(module, method)
                else:
                    method = caller
                    caller = sys.modules['__builtin__'].__dict__[method]
            except:
                print("can't obtain caller, locals: {}".format(locals()))
                traceback.print_exc()
                if task is not None:
                    self.finish_task(task)

                continue

            try:
                caller(*args, **kwargs)
            except:
                traceback.print_exc()
            finally:
                self.finish_task(task)
コード例 #5
0
ファイル: processes.py プロジェクト: yankaics/data007
 def finish_generation(self):
     conn.set(self.generated.format(self.name), 'true')
     conn.set(self.started_at.format(self.name), time.mktime(time.gmtime()))
コード例 #6
0
ファイル: processes.py プロジェクト: mobishift2011/data007
 def finish_generation(self):
     conn.set(self.generated.format(self.name), 'true')
     conn.set(self.started_at.format(self.name), time.mktime(time.gmtime()))