def slice(self, size, timeout=None): ''' Get a slice of the next items from the queue. Args: size (int): Maximum number of items to get from the queue. timeout (int): Duration, in seconds, to wait for items to be available to the queue before returning. Examples: Return up to 3 items on a 30 second timeout from the queue:: items = q.slice(3, timeout=30) Notes: This will block if the queue is empty and no timeout value is specified, or .done() has not been called on the Queue. Returns: list: A list of items from the queue. This will return None on fini() or timeout. Raises: synapse.exc.IsFini: Once the queue is fini synapse.exc.TimeOut: If timeout it specified and has passed. ''' while not self.isfini: with self.lock: ret = [] while len(ret) < size and self.deq: ret.append(self.deq.popleft()) if ret: return ret if self._que_done and not self.deq: self.fini() raise s_exc.IsFini() self.event.clear() if not self.event.wait(timeout=timeout): if self.isfini: raise s_exc.IsFini() raise s_exc.TimeOut() raise s_exc.IsFini()
async def task(self, todo, name=None): if self.isfini: raise s_exc.IsFini(mesg='Telepath Proxy isfini') if self.sess is not None: return await self.taskv2(todo, name=name) task = Task() mesg = ('task:init', { 'task': task.iden, 'todo': todo, 'name': name, }) self.tasks[task.iden] = task try: await self.link.tx(mesg) retn = await task.result() return s_common.result(retn) finally: self.tasks.pop(task.iden, None)
def iternext(self): try: while True: yield self.atitem if self.bumped: if self.slab.isfini: raise s_exc.IsFini() self.bumped = False self.curs = self.slab.xact.cursor(db=self.db) if self.dupsort: self.curs.set_range_dup(*self.atitem) else: self.curs.set_range(self.atitem[0]) self.genr = self.iterfunc(self.curs) next(self.genr) self.atitem = next(self.genr) except StopIteration: return
def iternext(self): try: while True: yield self.atitem if self.bumped: if self.slab.isfini: raise s_exc.IsFini() self.bumped = False self.curs = self.slab.xact.cursor(db=self.db) if not self.resume(): raise StopIteration self.genr = self.iterfunc() if self.isatitem(): next(self.genr) self.atitem = next(self.genr) except StopIteration: return
def get(self, timeout=None): ''' Get the next item from the queue. Args: timeout (int): Duration, in seconds, to wait for items to be available to the queue before returning. Notes: This will block if the queue is empty and no timeout value is specified, or .done() has not been called on the Queue. Examples: Get an item and do stuff with it:: item = q.get(timeout=30) dostuff(item) Returns: Item from the queue, or None if the queue is fini() or timeout occurs. ''' while not self.isfini: # for perf, try taking a lockless shot at it try: return self.deq.popleft() except IndexError as e: pass with self.lock: try: return self.deq.popleft() except IndexError as e: if self._que_done: self.fini() raise s_exc.IsFini() self.event.clear() if not self.event.wait(timeout=timeout): if self.isfini: raise s_exc.IsFini() raise s_exc.TimeOut() raise s_exc.IsFini()
def wrlock(self): ''' Acquire the pool write lock for the remainder of this transaction. ''' while not self.lockd: if self.isfini: raise s_exc.IsFini() self.lockd = self.pool.wlock.acquire(timeout=1)
def _acqXactForReading(self): if self.isfini: # pragma: no cover raise s_exc.IsFini() if not self.readonly: return self.xact if not self.txnrefcount: self._initCoXact() self.txnrefcount += 1 return self.xact
def iternext(self): try: while True: yield self.atitem if self.slab.isfini: raise s_exc.IsFini() if not self.bumped: self.atitem = next(self.genr) continue self.bumped = False advance = True self.curs = self.slab.xact.cursor(db=self.db) if self.dupsort: # Grab the >= val with the exact key ret = self.curs.set_range_dup(*self.atitem) if not ret: # Grab >= key key = self.atitem[0] ret = self.curs.set_range(key) # If we got the same key, advance to the last val if ret and self.curs.key() == key: self.curs.last_dup() # We're already guaranteed to be < self.atitem because set_range_dup returned False advance = False elif self.singlekey: # if set_range failed or we didn't get the same key, we're done raise StopIteration else: ret = self.curs.set_range(self.atitem[0]) if not ret: if not self.curs.last(): raise StopIteration self.genr = self.iterfunc(self.curs) if ret and advance: next(self.genr) self.atitem = next(self.genr) except StopIteration: return
def genr(): self.fd.seek(0) while True: if self.isfini: raise s_exc.IsFini() byts = self.fd.read(CHUNK_SIZE) if not byts: return yield byts
async def iter(self, offs: int) -> AsyncIterator[Any]: ''' Returns an iterator of change entries in the log ''' if not self.donexslog: return if self.isfini: raise s_exc.IsFini() maxoffs = offs for item in self.nexslog.iter(offs): if self.isfini: # pragma: no cover raise s_exc.IsFini() maxoffs = item[0] + 1 yield item async with self.getChangeDist(maxoffs) as dist: async for item in dist: if self.isfini: raise s_exc.IsFini() yield item
def select(self, *args): ''' Read-only optimized cursor execute method. Args: *args (list): The args to hand to cursor execute(). Returns: Cursor: see DB API ''' if self.isfini: raise s_exc.IsFini() return self.curs.execute(*args)
def iternext(self): try: while True: yield self.atitem if self.bumped: if self.slab.isfini: raise s_exc.IsFini() self.bumped = False self.curs = self.slab.xact.cursor(db=self.db) if self.dupsort: ret = self.curs.set_range_dup(*self.atitem) if not ret: if self.singlekey: raise StopIteration key = self.atitem[0] ret = self.curs.set_range(key) if ret and self.curs.key() == key: ret = self.curs.next_nodup() else: ret = self.curs.set_range(self.atitem[0]) if not ret: raise StopIteration self.genr = self.iterfunc(self.curs) if self.curs.item() == self.atitem: next(self.genr) self.atitem = next(self.genr) except StopIteration: return
async def task(self, todo, name=None): # implement the main workhorse method for a proxy to allow Method # objects to use us as the proxy. while not self.isfini: try: await self.waitready() # there is a small race where the daemon may fini the proxy # account for that here... if self._t_proxy is None or self._t_proxy.isfini: self._t_ready.clear() continue return await self._t_proxy.task(todo, name=name) except s_exc.TeleRedir as e: url = e.errinfo.get('url') self._setNextUrl(url) logger.warning(f'telepath task redirected: ({s_urlhelp.sanitizeUrl(url)})') await self._t_proxy.fini() raise s_exc.IsFini(mesg='Telepath Client isfini')
async def get(self): ''' Get a SpawnProc instance; either from the pool or a new process. Returns: SpawnProc: Yields a SpawnProc. This is placed back into the pool if no exceptions occur. ''' if self.isfini: # pragma: no cover raise s_exc.IsFini() proc = None if self.spawnq: proc = self.spawnq.popleft() if proc is None: proc = await self._new() yield proc await self._put(proc)
async def tx(self, mesg): ''' Async transmit routine which will wait for writer drain(). ''' if self.isfini: raise s_exc.IsFini() byts = s_msgpack.en(mesg) try: self.writer.write(byts) # Avoid Python bug. See https://bugs.python.org/issue29930 async with self._drain_lock: await self.writer.drain() except Exception as e: await self.fini() einfo = s_common.retnexc(e) logger.debug('link.tx connection trouble %s', einfo) raise
async def proxy(self, timeout=10): await self.waitready(timeout=timeout) ret = self._t_proxy if ret is None or ret.isfini is True: raise s_exc.IsFini(mesg='Telepath Client Proxy is not available.') return ret