def slice(self, size, timeout=None):
'''
Get a slice of the next items from the queue.
Args:
size (int): Maximum number of items to get from the queue.
timeout (int): Duration, in seconds, to wait for items to be available
to the queue before returning.
Examples:
Return up to 3 items on a 30 second timeout from the queue::
items = q.slice(3, timeout=30)
Notes:
This will block if the queue is empty and no timeout value is
specified, or .done() has not been called on the Queue.
Returns:
list: A list of items from the queue. This will return None on
fini() or timeout.
Raises:
synapse.exc.IsFini: Once the queue is fini
synapse.exc.TimeOut: If timeout it specified and has passed.
'''
while not self.isfini:
with self.lock:
ret = []
while len(ret) < size and self.deq:
ret.append(self.deq.popleft())
if ret:
return ret
if self._que_done and not self.deq:
self.fini()
raise s_exc.IsFini()
self.event.clear()
if not self.event.wait(timeout=timeout):
if self.isfini:
raise s_exc.IsFini()
raise s_exc.TimeOut()
raise s_exc.IsFini()
async def task(self, todo, name=None):
if self.isfini:
raise s_exc.IsFini(mesg='Telepath Proxy isfini')
if self.sess is not None:
return await self.taskv2(todo, name=name)
task = Task()
mesg = ('task:init', {
'task': task.iden,
'todo': todo,
'name': name, })
self.tasks[task.iden] = task
try:
await self.link.tx(mesg)
retn = await task.result()
return s_common.result(retn)
finally:
self.tasks.pop(task.iden, None)
def iternext(self):
try:
while True:
yield self.atitem
if self.bumped:
if self.slab.isfini:
raise s_exc.IsFini()
self.bumped = False
self.curs = self.slab.xact.cursor(db=self.db)
if self.dupsort:
self.curs.set_range_dup(*self.atitem)
else:
self.curs.set_range(self.atitem[0])
self.genr = self.iterfunc(self.curs)
next(self.genr)
self.atitem = next(self.genr)
except StopIteration:
return
def iternext(self):
try:
while True:
yield self.atitem
if self.bumped:
if self.slab.isfini:
raise s_exc.IsFini()
self.bumped = False
self.curs = self.slab.xact.cursor(db=self.db)
if not self.resume():
raise StopIteration
self.genr = self.iterfunc()
if self.isatitem():
next(self.genr)
self.atitem = next(self.genr)
except StopIteration:
return
def get(self, timeout=None):
'''
Get the next item from the queue.
Args:
timeout (int): Duration, in seconds, to wait for items to be available
to the queue before returning.
Notes:
This will block if the queue is empty and no timeout value is
specified, or .done() has not been called on the Queue.
Examples:
Get an item and do stuff with it::
item = q.get(timeout=30)
dostuff(item)
Returns:
Item from the queue, or None if the queue is fini() or timeout occurs.
'''
while not self.isfini:
# for perf, try taking a lockless shot at it
try:
return self.deq.popleft()
except IndexError as e:
pass
with self.lock:
try:
return self.deq.popleft()
except IndexError as e:
if self._que_done:
self.fini()
raise s_exc.IsFini()
self.event.clear()
if not self.event.wait(timeout=timeout):
if self.isfini:
raise s_exc.IsFini()
raise s_exc.TimeOut()
raise s_exc.IsFini()
def wrlock(self):
'''
Acquire the pool write lock for the remainder of this transaction.
'''
while not self.lockd:
if self.isfini:
raise s_exc.IsFini()
self.lockd = self.pool.wlock.acquire(timeout=1)
def _acqXactForReading(self):
if self.isfini: # pragma: no cover
raise s_exc.IsFini()
if not self.readonly:
return self.xact
if not self.txnrefcount:
self._initCoXact()
self.txnrefcount += 1
return self.xact
def iternext(self):
try:
while True:
yield self.atitem
if self.slab.isfini:
raise s_exc.IsFini()
if not self.bumped:
self.atitem = next(self.genr)
continue
self.bumped = False
advance = True
self.curs = self.slab.xact.cursor(db=self.db)
if self.dupsort:
# Grab the >= val with the exact key
ret = self.curs.set_range_dup(*self.atitem)
if not ret:
# Grab >= key
key = self.atitem[0]
ret = self.curs.set_range(key)
# If we got the same key, advance to the last val
if ret and self.curs.key() == key:
self.curs.last_dup()
# We're already guaranteed to be < self.atitem because set_range_dup returned False
advance = False
elif self.singlekey: # if set_range failed or we didn't get the same key, we're done
raise StopIteration
else:
ret = self.curs.set_range(self.atitem[0])
if not ret:
if not self.curs.last():
raise StopIteration
self.genr = self.iterfunc(self.curs)
if ret and advance:
next(self.genr)
self.atitem = next(self.genr)
except StopIteration:
return
def genr():
self.fd.seek(0)
while True:
if self.isfini:
raise s_exc.IsFini()
byts = self.fd.read(CHUNK_SIZE)
if not byts:
return
yield byts
async def iter(self, offs: int) -> AsyncIterator[Any]:
'''
Returns an iterator of change entries in the log
'''
if not self.donexslog:
return
if self.isfini:
raise s_exc.IsFini()
maxoffs = offs
for item in self.nexslog.iter(offs):
if self.isfini: # pragma: no cover
raise s_exc.IsFini()
maxoffs = item[0] + 1
yield item
async with self.getChangeDist(maxoffs) as dist:
async for item in dist:
if self.isfini:
raise s_exc.IsFini()
yield item
def select(self, *args):
'''
Read-only optimized cursor execute method.
Args:
*args (list): The args to hand to cursor execute().
Returns:
Cursor: see DB API
'''
if self.isfini:
raise s_exc.IsFini()
return self.curs.execute(*args)
def iternext(self):
try:
while True:
yield self.atitem
if self.bumped:
if self.slab.isfini:
raise s_exc.IsFini()
self.bumped = False
self.curs = self.slab.xact.cursor(db=self.db)
if self.dupsort:
ret = self.curs.set_range_dup(*self.atitem)
if not ret:
if self.singlekey:
raise StopIteration
key = self.atitem[0]
ret = self.curs.set_range(key)
if ret and self.curs.key() == key:
ret = self.curs.next_nodup()
else:
ret = self.curs.set_range(self.atitem[0])
if not ret:
raise StopIteration
self.genr = self.iterfunc(self.curs)
if self.curs.item() == self.atitem:
next(self.genr)
self.atitem = next(self.genr)
except StopIteration:
return
async def task(self, todo, name=None):
# implement the main workhorse method for a proxy to allow Method
# objects to use us as the proxy.
while not self.isfini:
try:
await self.waitready()
# there is a small race where the daemon may fini the proxy
# account for that here...
if self._t_proxy is None or self._t_proxy.isfini:
self._t_ready.clear()
continue
return await self._t_proxy.task(todo, name=name)
except s_exc.TeleRedir as e:
url = e.errinfo.get('url')
self._setNextUrl(url)
logger.warning(f'telepath task redirected: ({s_urlhelp.sanitizeUrl(url)})')
await self._t_proxy.fini()
raise s_exc.IsFini(mesg='Telepath Client isfini')
async def get(self):
'''
Get a SpawnProc instance; either from the pool or a new process.
Returns:
SpawnProc: Yields a SpawnProc. This is placed back into the pool if no exceptions occur.
'''
if self.isfini: # pragma: no cover
raise s_exc.IsFini()
proc = None
if self.spawnq:
proc = self.spawnq.popleft()
if proc is None:
proc = await self._new()
yield proc
await self._put(proc)
async def tx(self, mesg):
'''
Async transmit routine which will wait for writer drain().
'''
if self.isfini:
raise s_exc.IsFini()
byts = s_msgpack.en(mesg)
try:
self.writer.write(byts)
# Avoid Python bug. See https://bugs.python.org/issue29930
async with self._drain_lock:
await self.writer.drain()
except Exception as e:
await self.fini()
einfo = s_common.retnexc(e)
logger.debug('link.tx connection trouble %s', einfo)
raise
async def proxy(self, timeout=10):
await self.waitready(timeout=timeout)
ret = self._t_proxy
if ret is None or ret.isfini is True:
raise s_exc.IsFini(mesg='Telepath Client Proxy is not available.')
return ret