def run(self): print 'WorkerThread(%s) started!'%self._name logger = getattr(__builtin__,'print') if not self._process else (lambda s:(getattr(__builtin__,'print')(s),self.errorQueue.put(s))) while not self.stopEvent.is_set(): try: target,value = self.inQueue.get(True,timeout=self.wait),None if self.stopEvent.is_set(): break if target is not None: try: model = target #To avoid original target to be overriden in process() value = self.process(target) try: pickle.dumps(value) except pickle.PickleError: print traceback.format_exc() raise WorkerException('UnpickableValue') self.outQueue.put((model,value)) except Exception,e: msg = 'Exception in WorkerThread(%s).run()\n%s'%(self._name,except2str()) print( msg) self.outQueue.put((target,e)) finally: if not self._process: self.inQueue.task_done() if self.hook is not None: try: self.hook() except: print('Exception in WorkerThread(%s).hook()\n%s'%(self._name,except2str()))
def _receive_data(self): """ Main loop of the thread receiving data from the background Process (and launching callbacks when needed) """ while not self._threading_event.is_set(): try: if self._pipe1.poll(): key,query = self._pipe1.recv() if key not in self.data: self.trace('.Thread: received %s data; pending: %s'%(key,self.callbacks.keys())) pass if key in self.keys(): self.data[key].set(query) if self.data[key].callback: try: self.trace('.Thread:\tlaunching %s callback %s'%(key,callback)) self.data[key].callback(query) except: self.trace('.Thread:\tError in %s callback %s!'%(key,callback)) self.trace(except2str()) if key in self.callbacks: for callback in self.callbacks[key]: if callback is None: continue try: self.trace('.Thread:\tlaunching callback %s'%callback) callback(query) except: self.trace('.Thread:\tError in %s callback %s!'%(key,callback)) self.trace(except2str()) self.callbacks.pop(key) except: self.trace('.Thread,Exception:%s'%traceback.format_exc()) try: if time.time() > self.last_alive+3: self._pipe1.send((self.__ALIVE,None)) self.last_alive = time.time() except: self.trace('.Thread.is_alive(),Exception:%s'%traceback.format_exc()) self._threading_event.wait(self.timewait) print '!'*80 self.trace('.Thread: exit_data_thread') self.trace('<'*80) self.trace('<'*80)
def _run_process(self,pipe,event,executor=None): """ This is the main loop of the background Process. Queries sent to Process can be executed in different ways: - Having a process(query) method given as argument. - Having an object with a key(query) method: returns (key,result) - If none of this, passing query to an evalX call. Executor will be a callable or an object with 'target' methods """ last_alive,idle = time.time(),0 #Using NCycles count instead of raw time to avoid CPU influence key,paused = None,0 scheduled = {} #It will be a {key:[data,period,last_read]} dictionary locals_,modules,instances = {'executor':executor},{},{} key = None self.trace('.Process(%s) started'%str(executor or '')) while not event.is_set() and (pipe.poll() or idle<(self.timeout/self.timewait)): #time.time()<(last_alive+self.timeout)): try: idle+=1 now = time.time() if pipe.poll(): t = pipe.recv() #May be (key,) ; (key,args=None) ; (key,command,args) key,target,args = [(None,None,None),(t[0],None,None),(t[0],t[1],None),t][len(t)] if key!=self.__ALIVE: self.trace(shortstr('.Process: Received: %s => (%s,%s,%s)'%(str(t),key,target,args))) last_alive,idle = time.time(),0 #Keep Alive Thread elif scheduled and time.time()>paused: data = first(sorted((v.date+v.period,v) for n,v in scheduled.items()))[-1] if key not in scheduled and key is not None: self.trace('Nothing in queue, checking scheduled tasks ...') if (data.date+data.period)<=now: data.date = now key,target,args = data.name,data.target,data.args else: #print '%s > %s - %s' % (time2str(now),time2str(next)) key = None if key == self.__PAUSE: # should delay scheduled commands but not freeze those synchronous (like ADDKEY or COMMAND) paused = target self.trace('.Process: Scheduled keys will be paused %s seconds.'%(paused-time.time())) elif key == self.__ADDKEY: #(__ADDKEY,(args for ProcessedData(*))) #Done here to evaluate not-periodic keys in the same turn that they are received data = ProcessedData(*target) if data.period>0: scheduled[data.name]=data self.trace('.Process: Added key: %s'%str(data)) else: if data.name in scheduled: self.trace('.Process: Removing %s key'%data.name) scheduled.pop(data.name) if time.time()>paused: key,target,args = data.name,data.target,data.args #Added data will be immediately read if key is not None: try: if key == self.__REMOVEKEY: #(__REMOVEKEY,key) if target in scheduled: scheduled.pop(target) self.trace(scheduled) elif key == self.__BIND: # Setting a new executor object if isCallable(target): executor = target elif isinstance(target,basestring): executor = evalX(target,locals_,modules,instances) else: executor = target if isCallable(executor) and args is not None: if isMapping(args): executor = executor(**args) elif isSequence(args): executor = executor(*args) else: executor = executor(args) locals_['executor'] = executor self.trace('.Process: Bound executor object to %s'%(executor)) elif key!=self.__ALIVE: if args is None and executor is not None and (isCallable(executor) or getattr(executor,'process',None)): # Target is a set of arguments to Executor object exec_ = getattr(executor,'process',executor) args = target elif isinstance(target,basestring): # Target is a member of executor or an string to be evaluated # e.g. getattr(Reader,'get_attribute_values')(*(attr,start,stop)) if hasattr(executor,target): exec_ = getattr(executor,target) else: exec_ = evalX(target,locals_,modules,instances) #Executor bypassed if both target and args are sent else: #Target is a value or callable exec_ = target if isCallable(exec_): # Executing if key not in scheduled: self.trace(shortstr('.Process: [%s] = %s(%s)(*%s)'%(key,exec_,target,args))) if args is None: value = exec_() elif isDictionary(args): value = exec_(**args) elif isSequence(args): value = exec_(*args) else: value = exec_(args) else: #target can be a an object member or eval(target) result if key not in scheduled: self.trace(shortstr('.Process: [%s] = %s(*%s)'%(key,target,args))) value = exec_ pipe.send((key,getPickable(value))) except Exception,e: self.trace('.Process:\tError in %s process!\n%s\n%s\n%s'%(key,target,args,except2str(e))) #print traceback.format_exc() #print e pipe.send((key,getPickable(e))) except Exception,e: self.trace('.Process:\tUnknown Error in process!\n%s'%traceback.format_exc()) key = None event.wait(self.timewait)
except Exception,e: msg = 'Exception in WorkerThread(%s).run()\n%s'%(self._name,except2str()) print( msg) self.outQueue.put((target,e)) finally: if not self._process: self.inQueue.task_done() if self.hook is not None: try: self.hook() except: print('Exception in WorkerThread(%s).hook()\n%s'%(self._name,except2str())) except Queue.Empty: pass except: print 'FATAL Exception in WorkerThread(%s).run()'%self._name print except2str() print 'WorkerThread(%s) finished!'%self._name import objects class SingletonWorker(WorkerThread,objects.Singleton): """ Usage:: # ... same like WorkerThread, but command is required to get the result value command = "tc.execute(feedback='status',args=['ver\r\\n'])" sw.put(command) sw.get(command) """ def put(self,target): if not hasattr(self,'_queued'): self._queued = [] self._queued.append(target)
except Exception,e: msg = 'Exception in WorkerThread(%s).run()\n%s'%(self._name,except2str()) print( msg) self.outQueue.put((target,e)) finally: if not self._process: self.inQueue.task_done() if self.hook is not None: try: self.hook() except: print('Exception in WorkerThread(%s).hook()\n%s'%(self._name,except2str())) except Queue.Empty: pass except: print 'FATAL Exception in WorkerThread(%s).run()'%self._name print except2str() print 'WorkerThread(%s) finished!'%self._name import objects class SingletonWorker(WorkerThread,objects.Singleton): """ Usage:: # ... same like WorkerThread, but command is required to get the result value command = "tc.execute(feedback='status',args=['ver\r\\n'])" sw.put(command) sw.get(command) """ def put(self,target): if not hasattr(self,'_queued'): self._queued = [] self._queued.append(target) #<< saving a timestamp would be useful here