def slice(self, size, timeout=None):
'''
Get a slice of the next items from the queue.
Args:
size (int): Maximum number of items to get from the queue.
timeout (int): Duration, in seconds, to wait for items to be available
to the queue before returning.
Examples:
Return up to 3 items on a 30 second timeout from the queue::
items = q.slice(3, timeout=30)
Notes:
This will block if the queue is empty and no timeout value is
specified, or .done() has not been called on the Queue.
Returns:
list: A list of items from the queue. This will return None on
fini() or timeout.
Raises:
synapse.exc.IsFini: Once the queue is fini
synapse.exc.TimeOut: If timeout it specified and has passed.
'''
while not self.isfini:
with self.lock:
ret = []
while len(ret) < size and self.deq:
ret.append(self.deq.popleft())
if ret:
return ret
if self._que_done and not self.deq:
self.fini()
raise s_common.IsFini()
self.event.clear()
if not self.event.wait(timeout=timeout):
if self.isfini:
raise s_common.IsFini()
raise s_common.TimeOut()
raise s_common.IsFini()
def _que_work(self, work):
with self._pool_lock:
if self.isfini:
raise s_common.IsFini(self.__class__.__name__)
# we're about to put work into the queue
# lets see if we should also fire another worker
# if there are available threads, no need to fire
if self._pool_avail != 0:
self.workq.put(work)
return
# got any breathing room?
if self._pool_maxsize > len(self._pool_threads):
self._fire_thread(self._run_work)
self.workq.put(work)
return
# got *all* the breathing room?
if self._pool_maxsize == -1:
self._fire_thread(self._run_work)
self.workq.put(work)
return
self.workq.put(work)
def get(self, timeout=None):
'''
Get the next item from the queue.
Args:
timeout (int): Duration, in seconds, to wait for items to be available
to the queue before returning.
Notes:
This will block if the queue is empty and no timeout value is
specified, or .done() has not been called on the Queue.
Examples:
Get an item and do stuff with it::
item = q.get(timeout=30)
dostuff(item)
Returns:
Item from the queue, or None if the queue is fini() or timeout occurs.
'''
while not self.isfini:
# for perf, try taking a lockless shot at it
try:
return self.deq.popleft()
except IndexError as e:
pass
with self.lock:
try:
return self.deq.popleft()
except IndexError as e:
if self._que_done:
self.fini()
raise s_common.IsFini()
self.event.clear()
if not self.event.wait(timeout=timeout):
if self.isfini:
raise s_common.IsFini()
raise s_common.TimeOut()
raise s_common.IsFini()
def wrlock(self):
'''
Acquire the pool write lock for the remainder of this transaction.
'''
while not self.lockd:
if self.isfini:
raise s_common.IsFini()
self.lockd = self.pool.wlock.acquire(timeout=1)
def initJob(self, jid=None, **info):
'''
Initialize and return a new job tufo.
Example:
jid = jobid()
task = (meth,args,kwargs)
job = boss.initJob(jid,task=task,timeout=60)
Notes:
Info Conventions:
* task=(meth,args,kwargs)
* timeout=<sec> # max runtime
* ondone=<meth> # def ondone(job):
'''
if self.isfini:
raise s_common.IsFini()
if jid is None:
jid = s_common.guid()
info['done'] = False
info['times'] = []
job = (jid, info)
self._boss_jobs[jid] = job
# setup our per job local storage
# ( for non-serializables )
joblocal = {
'ondone': info.pop('ondone', None),
}
self.joblocal[jid] = joblocal
self._addJobTime(job, 'init')
if self.pool is not None:
self.pool.call(self._runJob, job)
# if we have a timeout, setup a sched callback
timeout = job[1].get('timeout')
if timeout is not None:
def hitmax():
joblocal.pop('schedevt', None)
self.fire('job:done', jid=jid, err='HitMaxTime')
joblocal['schedevt'] = s_glob.sched.insec(timeout, hitmax)
self.fire('job:init', job=job)
return job
def select(self, *args):
'''
Read-only optimized cursor execute method.
Args:
*args (list): The args to hand to cursor execute().
Returns:
Cursor: see DB API
'''
if self.isfini:
raise s_common.IsFini()
return self.curs.execute(*args)
def pump(self, iden, func):
'''
Fire a new pump thread to call the given function.
Example:
pdir.firePumpThread( iden, core.sync )
'''
if self.isfini:
raise s_common.IsFini()
with self.lock:
if self.pumps.get(iden):
raise Exception('Duplicate Pump Iden: %s' % (iden, ))
self._runPumpThread(iden, func)
def task(self, task, jid=None):
'''
Run the given task in the pool.
Example:
task = newtask( x.getFooByBar, bar )
pool.task(task, jid=None)
Notes:
* Specify jid=<iden> to generate job:done events.
'''
work = s_common.tufo(task, jid=jid)
with self._pool_lock:
if self.isfini:
raise s_common.IsFini(self.__class__.__name__)
# we're about to put work into the queue
# lets see if we should also fire another worker
# if there are available threads, no need to fire
if self._pool_avail != 0:
self.workq.put(work)
return
# got any breathing room?
if self._pool_maxsize > len(self._pool_threads):
self._fire_thread(self._run_work)
self.workq.put(work)
return
# got *all* the breathing room?
if self._pool_maxsize == -1:
self._fire_thread(self._run_work)
self.workq.put(work)
return
self.workq.put(work)
def add(self, item):
'''
Add an item to the persistance storage.
'''
byts = s_msgpack.en(item)
size = len(byts)
with self.fdlock:
if self.isfini:
raise s_common.IsFini()
if self.fdoff != self.size:
self.fd.seek(0, os.SEEK_END)
off = self.size
self.fd.write(byts)
self.size += len(byts)
self.fdoff = self.size
return (off, size)
def _getTeleSock(self):
if self.isfini:
raise s_common.IsFini()
return self._tele_sock
