def slice(self, size, timeout=None): ''' Get a slice of the next items from the queue. Args: size (int): Maximum number of items to get from the queue. timeout (int): Duration, in seconds, to wait for items to be available to the queue before returning. Examples: Return up to 3 items on a 30 second timeout from the queue:: items = q.slice(3, timeout=30) Notes: This will block if the queue is empty and no timeout value is specified, or .done() has not been called on the Queue. Returns: list: A list of items from the queue. This will return None on fini() or timeout. Raises: synapse.exc.IsFini: Once the queue is fini synapse.exc.TimeOut: If timeout it specified and has passed. ''' while not self.isfini: with self.lock: ret = [] while len(ret) < size and self.deq: ret.append(self.deq.popleft()) if ret: return ret if self._que_done and not self.deq: self.fini() raise s_common.IsFini() self.event.clear() if not self.event.wait(timeout=timeout): if self.isfini: raise s_common.IsFini() raise s_common.TimeOut() raise s_common.IsFini()
def _que_work(self, work): with self._pool_lock: if self.isfini: raise s_common.IsFini(self.__class__.__name__) # we're about to put work into the queue # lets see if we should also fire another worker # if there are available threads, no need to fire if self._pool_avail != 0: self.workq.put(work) return # got any breathing room? if self._pool_maxsize > len(self._pool_threads): self._fire_thread(self._run_work) self.workq.put(work) return # got *all* the breathing room? if self._pool_maxsize == -1: self._fire_thread(self._run_work) self.workq.put(work) return self.workq.put(work)
def get(self, timeout=None): ''' Get the next item from the queue. Args: timeout (int): Duration, in seconds, to wait for items to be available to the queue before returning. Notes: This will block if the queue is empty and no timeout value is specified, or .done() has not been called on the Queue. Examples: Get an item and do stuff with it:: item = q.get(timeout=30) dostuff(item) Returns: Item from the queue, or None if the queue is fini() or timeout occurs. ''' while not self.isfini: # for perf, try taking a lockless shot at it try: return self.deq.popleft() except IndexError as e: pass with self.lock: try: return self.deq.popleft() except IndexError as e: if self._que_done: self.fini() raise s_common.IsFini() self.event.clear() if not self.event.wait(timeout=timeout): if self.isfini: raise s_common.IsFini() raise s_common.TimeOut() raise s_common.IsFini()
def wrlock(self): ''' Acquire the pool write lock for the remainder of this transaction. ''' while not self.lockd: if self.isfini: raise s_common.IsFini() self.lockd = self.pool.wlock.acquire(timeout=1)
def initJob(self, jid=None, **info): ''' Initialize and return a new job tufo. Example: jid = jobid() task = (meth,args,kwargs) job = boss.initJob(jid,task=task,timeout=60) Notes: Info Conventions: * task=(meth,args,kwargs) * timeout=<sec> # max runtime * ondone=<meth> # def ondone(job): ''' if self.isfini: raise s_common.IsFini() if jid is None: jid = s_common.guid() info['done'] = False info['times'] = [] job = (jid, info) self._boss_jobs[jid] = job # setup our per job local storage # ( for non-serializables ) joblocal = { 'ondone': info.pop('ondone', None), } self.joblocal[jid] = joblocal self._addJobTime(job, 'init') if self.pool is not None: self.pool.call(self._runJob, job) # if we have a timeout, setup a sched callback timeout = job[1].get('timeout') if timeout is not None: def hitmax(): joblocal.pop('schedevt', None) self.fire('job:done', jid=jid, err='HitMaxTime') joblocal['schedevt'] = s_glob.sched.insec(timeout, hitmax) self.fire('job:init', job=job) return job
def select(self, *args): ''' Read-only optimized cursor execute method. Args: *args (list): The args to hand to cursor execute(). Returns: Cursor: see DB API ''' if self.isfini: raise s_common.IsFini() return self.curs.execute(*args)
def pump(self, iden, func): ''' Fire a new pump thread to call the given function. Example: pdir.firePumpThread( iden, core.sync ) ''' if self.isfini: raise s_common.IsFini() with self.lock: if self.pumps.get(iden): raise Exception('Duplicate Pump Iden: %s' % (iden, )) self._runPumpThread(iden, func)
def task(self, task, jid=None): ''' Run the given task in the pool. Example: task = newtask( x.getFooByBar, bar ) pool.task(task, jid=None) Notes: * Specify jid=<iden> to generate job:done events. ''' work = s_common.tufo(task, jid=jid) with self._pool_lock: if self.isfini: raise s_common.IsFini(self.__class__.__name__) # we're about to put work into the queue # lets see if we should also fire another worker # if there are available threads, no need to fire if self._pool_avail != 0: self.workq.put(work) return # got any breathing room? if self._pool_maxsize > len(self._pool_threads): self._fire_thread(self._run_work) self.workq.put(work) return # got *all* the breathing room? if self._pool_maxsize == -1: self._fire_thread(self._run_work) self.workq.put(work) return self.workq.put(work)
def add(self, item): ''' Add an item to the persistance storage. ''' byts = s_msgpack.en(item) size = len(byts) with self.fdlock: if self.isfini: raise s_common.IsFini() if self.fdoff != self.size: self.fd.seek(0, os.SEEK_END) off = self.size self.fd.write(byts) self.size += len(byts) self.fdoff = self.size return (off, size)
def _getTeleSock(self): if self.isfini: raise s_common.IsFini() return self._tele_sock