def __init__(self, q, name=None):
if name is None:
name = "QueueIterator-%d" % (seq(),)
self.q = q
self.name = name
self.finalise_later = True
# count of non-sentinel items
self._item_count = 0
def setUp(self):
dbpath = 'test-%d.tch' % (seq(), )
self.dbpath = dbpath
if os.path.exists(dbpath):
os.remove(dbpath)
with open("/dev/tty", "w") as tty:
tty.write("SETUP test %s\n" % (self, ))
self.backend = Backend_TokyoCabinet(dbpath)
self.db = NodeDB(backend=self.backend)
def newkey(self):
''' Allocate a new key.
'''
now = time.time()
secs = int(now)
subsecs = now - secs
key = '%d.#%dM%dP%d' % (secs, seq(), subsecs * 1e6, self.pid)
assert self.validkey(key), "invalid new key: %s" % (key, )
return key
def _submit(
self,
func,
priority=None,
delay=None,
when=None,
name=None,
pfx=None,
LF=None,
retry_delay=None
):
if delay is not None and when is not None:
raise ValueError(
"you can't specify both delay= and when= (%s, %s)" % (delay, when)
)
if priority is None:
priority = self._priority
elif isinstance(priority, int):
priority = (priority,)
if pfx is not None:
func = pfx.partial(func)
if LF is None:
LF = LateFunction(func, name=name, retry_delay=retry_delay)
pri_entry = list(priority)
pri_entry.append(seq()) # ensure FIFO servicing of equal priorities
pri_entry.append(LF)
now = time.time()
if delay is not None:
when = now + delay
if when is None or when <= now:
# queue the request now
self.debug("queuing %s", LF)
heappush(self.pending, pri_entry)
self._try_dispatch()
else:
# queue the request at a later time
def queueFunc():
LF = pri_entry[-1]
self.debug("queuing %s after delay", LF)
heappush(self.pending, pri_entry)
self._try_dispatch()
with self._lock:
if self._timerQ is None:
self._timerQ = TimerQueue(
name="<TimerQueue %s._timerQ>" % (self.name,)
)
self.debug("delay %s until %s", LF, when)
self._timerQ.add(when, queueFunc)
# record the function as outstanding and attach a notification
# to remove it from the outstanding set on completion
self.outstanding.add(LF)
LF.notify(self.outstanding.remove)
return LF
def __init__(self, name=None):
if name is None:
name = 'TimerQueue-%d' % (seq(),)
self.name = name
self.Q = PriorityQueue() # queue of waiting jobs
self.pending = None # or (Timer, when, func)
self.closed = False
self._lock = Lock()
self.mainRunning = False
self.mainThread = Thread(target=self._main)
self.mainThread.start()
def mkbasename(self):
now=time.time()
secs=int(now)
subsecs=now-secs
left=str(secs)
if self.__hostname is None:
self.__hostname=socket.gethostname()
right=self.__hostname.replace('/','\057').replace(':','\072')
middle='#'+str(seq())+'M'+str(subsecs*1e6)+'P'+str(os.getpid())+'Q'+str(_nextDelivered())
return string.join((left,middle,right),'.')
def __init__(self, blocking=False, name=None):
''' Initialise the `NullQueue`.
Parameters:
* `blocking`: optional; if true, calls to `.get()` block until
`.shutdown()`; default: `False`.
* `name`: optional name for this `NullQueue`.
'''
if name is None:
name = "%s%d" % (self.__class__.__name__, seq())
self.name = name
self._lock = RLock()
MultiOpenMixin.__init__(self)
self.blocking = blocking
def __init__(self, func, name=None, retry_delay=None):
''' Initialise a `LateFunction`.
Parameters:
* `func` is the callable for later execution.
* `name`, if supplied, specifies an identifying name for the `LateFunction`.
* `retry_local`: time delay before retry of this function on RetryError.
Default from `later.retry_delay`.
'''
Result.__init__(self)
self.func = func
if name is None:
name = "LF-%d[%s]" % (seq(), funcname(func))
if retry_delay is None:
retry_delay = DEFAULT_RETRY_DELAY
self.name = name
self.retry_delay = retry_delay
def dispatch(self, func, retq=None, deliver=None, pfx=None, daemon=None):
''' Dispatch the callable `func` in a separate thread.
On completion the result is the sequence
`func_result, None, None, None`.
On an exception the result is the sequence
`None, exec_type, exc_value, exc_traceback`.
If `retq` is not None, the result is .put() on retq.
If `deliver` is not None, deliver(result) is called.
If the parameter `pfx` is not None, submit pfx.partial(func);
see the cs.logutils.Pfx.partial method for details.
If `daemon` is not None, set the .daemon attribute of the Thread to `daemon`.
TODO: high water mark for idle Threads.
'''
if self.closed:
raise ValueError("%s: closed, but dispatch() called" % (self,))
if pfx is not None:
func = pfx.partial(func)
if daemon is None:
daemon = current_thread().daemon
idle = self.idle_daemon if daemon else self.idle_fg
with self._lock:
debug("dispatch: idle = %s", idle)
if idle:
# use an idle thread
entry = idle.pop()
debug("dispatch: reuse %s", entry)
else:
debug("dispatch: need new thread")
# no available threads - make one
Targs = []
T = Thread(
target=self._handler,
args=Targs,
name=("%s:worker" % (self.name,))
)
T.daemon = daemon
Q = IterableQueue(name="%s:IQ%d" % (self.name, seq()))
entry = WTPoolEntry(T, Q)
self.all.add(entry)
Targs.append(entry)
debug("%s: start new worker thread (daemon=%s)", self, T.daemon)
T.start()
entry.queue.put((func, retq, deliver))
def __init__(self, name=None, max_spare=4):
''' Initialise the WorkerThreadPool.
Parameters:
* `name`: optional name for the pool
* `max_spare`: maximum size of each idle pool (daemon and non-daemon)
'''
if name is None:
name = "WorkerThreadPool-%d" % (seq(),)
if max_spare < 1:
raise ValueError("max_spare(%s) must be >= 1" % (max_spare,))
MultiOpenMixin.__init__(self)
self.name = name
self.max_spare = max_spare
self.idle_fg = deque() # nondaemon Threads
self.idle_daemon = deque() # daemon Threads
self.all = set()
self._lock = Lock()
def __init__(self, name, functor, outQ):
''' Initialise the PushQueue with the callable `functor`
and the output queue `outQ`.
Parameters:
* `functor` is a one-to-many function which accepts a single
item of input and returns an iterable of outputs; it may be a
generator. These outputs are passed to `outQ.put` individually as
received.
* `outQ` is a `MultiOpenMixin` which accepts via its `.put()` method.
'''
if name is None:
name = "%s%d-%s" % (self.__class__.__name__, seq(), functor)
self.name = name
self._lock = RLock()
MultiOpenMixin.__init__(self)
self.functor = functor
self.outQ = outQ
def __init__(self, capacity, name=None, inboundCapacity=0, retry_delay=None):
''' Initialise the Later instance.
Parameters:
* `capacity`: resource contraint on this Later; if an int, it is used
to size a Semaphore to constrain the number of dispatched functions
which may be in play at a time; if not an int it is presumed to be a
suitable Semaphore-like object, perhaps shared with other subsystems.
* `name`: optional identifying name for this instance.
* `inboundCapacity`: if >0, used as a limit on the number of
undispatched functions that may be queued up; the default is 0 (no
limit). Calls to submit functions when the inbound limit is reached
block until some functions are dispatched.
* `retry_delay`: time delay for requeued functions.
Default: `DEFAULT_RETRY_DELAY`.
'''
if name is None:
name = "Later-%d" % (seq(),)
if ifdebug():
import inspect # pylint: disable=import-outside-toplevel
filename, lineno = inspect.stack()[1][1:3]
name = "%s[%s:%d]" % (name, filename, lineno)
debug(
"Later.__init__(capacity=%s, inboundCapacity=%s, name=%s)", capacity,
inboundCapacity, name
)
if retry_delay is None:
retry_delay = DEFAULT_RETRY_DELAY
self.capacity = capacity
self.inboundCapacity = inboundCapacity
self.retry_delay = retry_delay
self.name = name
self._lock = Lock()
self.outstanding = set() # dispatched but uncompleted LateFunctions
self.delayed = set() # unqueued, delayed until specific time
self.pending = [] # undispatched LateFunctions, a heap
self.running = set() # running LateFunctions
# counter tracking jobs queued or active
self._state = ""
self.logger = None # reporting; see logTo() method
self._priority = (0,)
self._timerQ = None # queue for delayed requests; instantiated at need
# inbound requests queue
self._finished = None
def after(Rs, R, func, *a, **kw):
''' After the completion of `Rs` call `func(*a,**kw)` and return
its result via `R`; return the `Result` object.
Parameters:
* `Rs`: an iterable of Results.
* `R`: a `Result` to collect to result of calling `func`.
If `None`, one will be created.
* `func`, `a`, `kw`: a callable and its arguments.
'''
if R is None:
R = Result("after-%d" % (seq(), ))
elif not isinstance(R, Result):
raise TypeError(
"after(Rs, R, func, ...): expected Result for R, got %r" % (R, ))
lock = Lock()
Rs = list(Rs)
count = len(Rs)
if count == 0:
R.call(func, *a, **kw)
else:
countery = [
count
] # to stop "count" looking like a local var inside the closure
def count_down(_):
''' Notification function to submit `func` after sufficient invocations.
'''
with lock:
countery[0] -= 1
count = countery[0]
if count > 0:
# not ready yet
return
if count == 0:
R.call(func, *a, **kw)
else:
raise RuntimeError("count < 0: %d" % (count, ))
# submit the notifications
for subR in Rs:
subR.notify(count_down)
return R
def addmsg(self, msg):
if 'from' not in msg:
pw = pwd.getpwuid(os.geteuid())
gecos = pw[4]
cpos = gecos.find(',')
if cpos >= 0: gecos = gecos[:cpos]
msg['From'] = "%s <%s>" % (gecos, pw[0])
if 'date' not in msg:
# RFC822 time
# FIXME: locale likely to break this?
msg['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S +0000",
time.gmtime())
if 'message-id' not in msg:
msg['Message-ID'] = '<%d.%d.%d@%s>' % (time.time(), os.getpid(),
seq(), socket.gethostname())
self.add(msg)
def inner(*a, **kw):
if not force and not ifdebug():
return f(*a, **kw)
filename, lineno = inspect.stack()[1][1:3]
n = seq()
R = Result()
T = threading.Thread(target=_debug_watcher,
args=(filename, lineno, n, f.__name__, R))
T.daemon = True
T.start()
debug("%s:%d: [%d] call %s(*%r, **%r)", filename, lineno, n,
f.__name__, a, kw)
start = time.time()
try:
retval = f(*a, **kw)
except Exception as e:
error("EXCEPTION from %s(*%s, **%s): %s", f, a, kw, e)
raise
end = time.time()
debug("%s:%d: [%d] called %s, elapsed %gs, got %r", filename, lineno,
n, f.__name__, end - start, retval)
R.put(retval)
return retval
def __init__(self, name=None, start_time=None, units_scale=None):
''' Initialise a progress instance.
Parameters:
* `name`: optional name
* `start_time`: optional UNIX epoch start time, default from `time.time()`
* `units_scale`: a scale for use with `cs.units.transcribe`,
default `BINARY_BYTES_SCALE`
'''
now = time.time()
if name is None:
name = '-'.join((type(self).__name__, str(seq())))
if start_time is None:
start_time = now
elif start_time > now:
raise ValueError("start_time(%s) > now(%s)" % (start_time, now))
if units_scale is None:
units_scale = BINARY_BYTES_SCALE
self.name = name
self.start_time = start_time
self.units_scale = units_scale
self.notify_update = set()
self._warned = set()
self._lock = RLock()
def __init__(self,
recv,
send,
request_handler=None,
name=None,
packet_grace=None,
tick=None):
''' Initialise the PacketConnection.
Parameters:
* `recv`: inbound binary stream.
If this is an `int` it is taken to be an OS file descriptor,
otherwise it should be a `cs.buffer.CornuCopyBuffer`
or a file like object with a `read1` or `read` method.
* `send`: outbound binary stream.
If this is an `int` it is taken to be an OS file descriptor,
otherwise it should be a file like object with `.write(bytes)`
and `.flush()` methods.
For a file descriptor sending is done via an os.dup() of
the supplied descriptor, so the caller remains responsible
for closing the original descriptor.
* `packet_grace`:
default pause in the packet sending worker
to allow another packet to be queued
before flushing the output stream.
Default: `DEFAULT_PACKET_GRACE`s.
A value of `0` will flush immediately if the queue is empty.
* `request_handler`: an optional callable accepting
(`rq_type`, `flags`, `payload`).
The request_handler may return one of 5 values on success:
* `None`: response will be 0 flags and an empty payload.
* `int`: flags only. Response will be the flags and an empty payload.
* `bytes`: payload only. Response will be 0 flags and the payload.
* `str`: payload only. Response will be 0 flags and the str
encoded as bytes using UTF-8.
* `(int, bytes)`: Specify flags and payload for response.
An unsuccessful request should raise an exception, which
will cause a failure response packet.
* `tick`: optional tick parameter, default `None`.
If `None`, do nothing.
If a Boolean, call `tick_fd_2` if true, otherwise do nothing.
Otherwise `tick` should be a callable accepting a byteslike value.
'''
if name is None:
name = str(seq())
self.name = name
if isinstance(recv, int):
self._recv = CornuCopyBuffer.from_fd(recv)
elif isinstance(recv, CornuCopyBuffer):
self._recv = recv
else:
self._recv = CornuCopyBuffer.from_file(recv)
if isinstance(send, int):
self._send = os.fdopen(os.dup(send), 'wb')
else:
self._send = send
if packet_grace is None:
packet_grace = DEFAULT_PACKET_GRACE
if tick is None:
tick = lambda bs: None
elif isinstance(tick, bool):
if tick:
tick = tick_fd_2
else:
tick = lambda bs: None
self.packet_grace = packet_grace
self.request_handler = request_handler
self.tick = tick
# tags of requests in play against the local system
self._channel_request_tags = {0: set()}
self.notify_recv_eof = set()
self.notify_send_eof = set()
# LateFunctions for the requests we are performing for the remote system
self._running = set()
# requests we have outstanding against the remote system
self._pending = {0: {}}
# sequence of tag numbers
# TODO: later, reuse old tags to prevent monotonic growth of tag field
self._tag_seq = Seq(1)
# work queue for local requests
self._later = Later(4, name="%s:Later" % (self, ))
self._later.open()
# dispatch queue of Packets to send
self._sendQ = IterableQueue(16)
self._lock = Lock()
self.closed = False
# debugging: check for reuse of (channel,tag) etc
self.__sent = set()
self.__send_queued = set()
# dispatch Thread to process received packets
self._recv_thread = bg_thread(self._receive_loop,
name="%s[_receive_loop]" % (self.name, ))
# dispatch Thread to send data
# primary purpose is to bundle output by deferring flushes
self._send_thread = bg_thread(self._send_loop,
name="%s[_send]" % (self.name, ))
def add(self, when, func):
''' Queue a new job to be called at 'when'.
'func' is the job function, typically made with `functools.partial`.
'''
assert not self.closed, "add() on closed TimerQueue"
self.Q.put((when, seq(), func))
