def __init__(self, q, name=None): if name is None: name = "QueueIterator-%d" % (seq(),) self.q = q self.name = name self.finalise_later = True # count of non-sentinel items self._item_count = 0
def setUp(self): dbpath = 'test-%d.tch' % (seq(), ) self.dbpath = dbpath if os.path.exists(dbpath): os.remove(dbpath) with open("/dev/tty", "w") as tty: tty.write("SETUP test %s\n" % (self, )) self.backend = Backend_TokyoCabinet(dbpath) self.db = NodeDB(backend=self.backend)
def newkey(self): ''' Allocate a new key. ''' now = time.time() secs = int(now) subsecs = now - secs key = '%d.#%dM%dP%d' % (secs, seq(), subsecs * 1e6, self.pid) assert self.validkey(key), "invalid new key: %s" % (key, ) return key
def _submit( self, func, priority=None, delay=None, when=None, name=None, pfx=None, LF=None, retry_delay=None ): if delay is not None and when is not None: raise ValueError( "you can't specify both delay= and when= (%s, %s)" % (delay, when) ) if priority is None: priority = self._priority elif isinstance(priority, int): priority = (priority,) if pfx is not None: func = pfx.partial(func) if LF is None: LF = LateFunction(func, name=name, retry_delay=retry_delay) pri_entry = list(priority) pri_entry.append(seq()) # ensure FIFO servicing of equal priorities pri_entry.append(LF) now = time.time() if delay is not None: when = now + delay if when is None or when <= now: # queue the request now self.debug("queuing %s", LF) heappush(self.pending, pri_entry) self._try_dispatch() else: # queue the request at a later time def queueFunc(): LF = pri_entry[-1] self.debug("queuing %s after delay", LF) heappush(self.pending, pri_entry) self._try_dispatch() with self._lock: if self._timerQ is None: self._timerQ = TimerQueue( name="<TimerQueue %s._timerQ>" % (self.name,) ) self.debug("delay %s until %s", LF, when) self._timerQ.add(when, queueFunc) # record the function as outstanding and attach a notification # to remove it from the outstanding set on completion self.outstanding.add(LF) LF.notify(self.outstanding.remove) return LF
def __init__(self, name=None): if name is None: name = 'TimerQueue-%d' % (seq(),) self.name = name self.Q = PriorityQueue() # queue of waiting jobs self.pending = None # or (Timer, when, func) self.closed = False self._lock = Lock() self.mainRunning = False self.mainThread = Thread(target=self._main) self.mainThread.start()
def mkbasename(self): now=time.time() secs=int(now) subsecs=now-secs left=str(secs) if self.__hostname is None: self.__hostname=socket.gethostname() right=self.__hostname.replace('/','\057').replace(':','\072') middle='#'+str(seq())+'M'+str(subsecs*1e6)+'P'+str(os.getpid())+'Q'+str(_nextDelivered()) return string.join((left,middle,right),'.')
def __init__(self, blocking=False, name=None): ''' Initialise the `NullQueue`. Parameters: * `blocking`: optional; if true, calls to `.get()` block until `.shutdown()`; default: `False`. * `name`: optional name for this `NullQueue`. ''' if name is None: name = "%s%d" % (self.__class__.__name__, seq()) self.name = name self._lock = RLock() MultiOpenMixin.__init__(self) self.blocking = blocking
def __init__(self, func, name=None, retry_delay=None): ''' Initialise a `LateFunction`. Parameters: * `func` is the callable for later execution. * `name`, if supplied, specifies an identifying name for the `LateFunction`. * `retry_local`: time delay before retry of this function on RetryError. Default from `later.retry_delay`. ''' Result.__init__(self) self.func = func if name is None: name = "LF-%d[%s]" % (seq(), funcname(func)) if retry_delay is None: retry_delay = DEFAULT_RETRY_DELAY self.name = name self.retry_delay = retry_delay
def dispatch(self, func, retq=None, deliver=None, pfx=None, daemon=None): ''' Dispatch the callable `func` in a separate thread. On completion the result is the sequence `func_result, None, None, None`. On an exception the result is the sequence `None, exec_type, exc_value, exc_traceback`. If `retq` is not None, the result is .put() on retq. If `deliver` is not None, deliver(result) is called. If the parameter `pfx` is not None, submit pfx.partial(func); see the cs.logutils.Pfx.partial method for details. If `daemon` is not None, set the .daemon attribute of the Thread to `daemon`. TODO: high water mark for idle Threads. ''' if self.closed: raise ValueError("%s: closed, but dispatch() called" % (self,)) if pfx is not None: func = pfx.partial(func) if daemon is None: daemon = current_thread().daemon idle = self.idle_daemon if daemon else self.idle_fg with self._lock: debug("dispatch: idle = %s", idle) if idle: # use an idle thread entry = idle.pop() debug("dispatch: reuse %s", entry) else: debug("dispatch: need new thread") # no available threads - make one Targs = [] T = Thread( target=self._handler, args=Targs, name=("%s:worker" % (self.name,)) ) T.daemon = daemon Q = IterableQueue(name="%s:IQ%d" % (self.name, seq())) entry = WTPoolEntry(T, Q) self.all.add(entry) Targs.append(entry) debug("%s: start new worker thread (daemon=%s)", self, T.daemon) T.start() entry.queue.put((func, retq, deliver))
def __init__(self, name=None, max_spare=4): ''' Initialise the WorkerThreadPool. Parameters: * `name`: optional name for the pool * `max_spare`: maximum size of each idle pool (daemon and non-daemon) ''' if name is None: name = "WorkerThreadPool-%d" % (seq(),) if max_spare < 1: raise ValueError("max_spare(%s) must be >= 1" % (max_spare,)) MultiOpenMixin.__init__(self) self.name = name self.max_spare = max_spare self.idle_fg = deque() # nondaemon Threads self.idle_daemon = deque() # daemon Threads self.all = set() self._lock = Lock()
def __init__(self, name, functor, outQ): ''' Initialise the PushQueue with the callable `functor` and the output queue `outQ`. Parameters: * `functor` is a one-to-many function which accepts a single item of input and returns an iterable of outputs; it may be a generator. These outputs are passed to `outQ.put` individually as received. * `outQ` is a `MultiOpenMixin` which accepts via its `.put()` method. ''' if name is None: name = "%s%d-%s" % (self.__class__.__name__, seq(), functor) self.name = name self._lock = RLock() MultiOpenMixin.__init__(self) self.functor = functor self.outQ = outQ
def __init__(self, capacity, name=None, inboundCapacity=0, retry_delay=None): ''' Initialise the Later instance. Parameters: * `capacity`: resource contraint on this Later; if an int, it is used to size a Semaphore to constrain the number of dispatched functions which may be in play at a time; if not an int it is presumed to be a suitable Semaphore-like object, perhaps shared with other subsystems. * `name`: optional identifying name for this instance. * `inboundCapacity`: if >0, used as a limit on the number of undispatched functions that may be queued up; the default is 0 (no limit). Calls to submit functions when the inbound limit is reached block until some functions are dispatched. * `retry_delay`: time delay for requeued functions. Default: `DEFAULT_RETRY_DELAY`. ''' if name is None: name = "Later-%d" % (seq(),) if ifdebug(): import inspect # pylint: disable=import-outside-toplevel filename, lineno = inspect.stack()[1][1:3] name = "%s[%s:%d]" % (name, filename, lineno) debug( "Later.__init__(capacity=%s, inboundCapacity=%s, name=%s)", capacity, inboundCapacity, name ) if retry_delay is None: retry_delay = DEFAULT_RETRY_DELAY self.capacity = capacity self.inboundCapacity = inboundCapacity self.retry_delay = retry_delay self.name = name self._lock = Lock() self.outstanding = set() # dispatched but uncompleted LateFunctions self.delayed = set() # unqueued, delayed until specific time self.pending = [] # undispatched LateFunctions, a heap self.running = set() # running LateFunctions # counter tracking jobs queued or active self._state = "" self.logger = None # reporting; see logTo() method self._priority = (0,) self._timerQ = None # queue for delayed requests; instantiated at need # inbound requests queue self._finished = None
def after(Rs, R, func, *a, **kw): ''' After the completion of `Rs` call `func(*a,**kw)` and return its result via `R`; return the `Result` object. Parameters: * `Rs`: an iterable of Results. * `R`: a `Result` to collect to result of calling `func`. If `None`, one will be created. * `func`, `a`, `kw`: a callable and its arguments. ''' if R is None: R = Result("after-%d" % (seq(), )) elif not isinstance(R, Result): raise TypeError( "after(Rs, R, func, ...): expected Result for R, got %r" % (R, )) lock = Lock() Rs = list(Rs) count = len(Rs) if count == 0: R.call(func, *a, **kw) else: countery = [ count ] # to stop "count" looking like a local var inside the closure def count_down(_): ''' Notification function to submit `func` after sufficient invocations. ''' with lock: countery[0] -= 1 count = countery[0] if count > 0: # not ready yet return if count == 0: R.call(func, *a, **kw) else: raise RuntimeError("count < 0: %d" % (count, )) # submit the notifications for subR in Rs: subR.notify(count_down) return R
def addmsg(self, msg): if 'from' not in msg: pw = pwd.getpwuid(os.geteuid()) gecos = pw[4] cpos = gecos.find(',') if cpos >= 0: gecos = gecos[:cpos] msg['From'] = "%s <%s>" % (gecos, pw[0]) if 'date' not in msg: # RFC822 time # FIXME: locale likely to break this? msg['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) if 'message-id' not in msg: msg['Message-ID'] = '<%d.%d.%d@%s>' % (time.time(), os.getpid(), seq(), socket.gethostname()) self.add(msg)
def inner(*a, **kw): if not force and not ifdebug(): return f(*a, **kw) filename, lineno = inspect.stack()[1][1:3] n = seq() R = Result() T = threading.Thread(target=_debug_watcher, args=(filename, lineno, n, f.__name__, R)) T.daemon = True T.start() debug("%s:%d: [%d] call %s(*%r, **%r)", filename, lineno, n, f.__name__, a, kw) start = time.time() try: retval = f(*a, **kw) except Exception as e: error("EXCEPTION from %s(*%s, **%s): %s", f, a, kw, e) raise end = time.time() debug("%s:%d: [%d] called %s, elapsed %gs, got %r", filename, lineno, n, f.__name__, end - start, retval) R.put(retval) return retval
def __init__(self, name=None, start_time=None, units_scale=None): ''' Initialise a progress instance. Parameters: * `name`: optional name * `start_time`: optional UNIX epoch start time, default from `time.time()` * `units_scale`: a scale for use with `cs.units.transcribe`, default `BINARY_BYTES_SCALE` ''' now = time.time() if name is None: name = '-'.join((type(self).__name__, str(seq()))) if start_time is None: start_time = now elif start_time > now: raise ValueError("start_time(%s) > now(%s)" % (start_time, now)) if units_scale is None: units_scale = BINARY_BYTES_SCALE self.name = name self.start_time = start_time self.units_scale = units_scale self.notify_update = set() self._warned = set() self._lock = RLock()
def __init__(self, recv, send, request_handler=None, name=None, packet_grace=None, tick=None): ''' Initialise the PacketConnection. Parameters: * `recv`: inbound binary stream. If this is an `int` it is taken to be an OS file descriptor, otherwise it should be a `cs.buffer.CornuCopyBuffer` or a file like object with a `read1` or `read` method. * `send`: outbound binary stream. If this is an `int` it is taken to be an OS file descriptor, otherwise it should be a file like object with `.write(bytes)` and `.flush()` methods. For a file descriptor sending is done via an os.dup() of the supplied descriptor, so the caller remains responsible for closing the original descriptor. * `packet_grace`: default pause in the packet sending worker to allow another packet to be queued before flushing the output stream. Default: `DEFAULT_PACKET_GRACE`s. A value of `0` will flush immediately if the queue is empty. * `request_handler`: an optional callable accepting (`rq_type`, `flags`, `payload`). The request_handler may return one of 5 values on success: * `None`: response will be 0 flags and an empty payload. * `int`: flags only. Response will be the flags and an empty payload. * `bytes`: payload only. Response will be 0 flags and the payload. * `str`: payload only. Response will be 0 flags and the str encoded as bytes using UTF-8. * `(int, bytes)`: Specify flags and payload for response. An unsuccessful request should raise an exception, which will cause a failure response packet. * `tick`: optional tick parameter, default `None`. If `None`, do nothing. If a Boolean, call `tick_fd_2` if true, otherwise do nothing. Otherwise `tick` should be a callable accepting a byteslike value. ''' if name is None: name = str(seq()) self.name = name if isinstance(recv, int): self._recv = CornuCopyBuffer.from_fd(recv) elif isinstance(recv, CornuCopyBuffer): self._recv = recv else: self._recv = CornuCopyBuffer.from_file(recv) if isinstance(send, int): self._send = os.fdopen(os.dup(send), 'wb') else: self._send = send if packet_grace is None: packet_grace = DEFAULT_PACKET_GRACE if tick is None: tick = lambda bs: None elif isinstance(tick, bool): if tick: tick = tick_fd_2 else: tick = lambda bs: None self.packet_grace = packet_grace self.request_handler = request_handler self.tick = tick # tags of requests in play against the local system self._channel_request_tags = {0: set()} self.notify_recv_eof = set() self.notify_send_eof = set() # LateFunctions for the requests we are performing for the remote system self._running = set() # requests we have outstanding against the remote system self._pending = {0: {}} # sequence of tag numbers # TODO: later, reuse old tags to prevent monotonic growth of tag field self._tag_seq = Seq(1) # work queue for local requests self._later = Later(4, name="%s:Later" % (self, )) self._later.open() # dispatch queue of Packets to send self._sendQ = IterableQueue(16) self._lock = Lock() self.closed = False # debugging: check for reuse of (channel,tag) etc self.__sent = set() self.__send_queued = set() # dispatch Thread to process received packets self._recv_thread = bg_thread(self._receive_loop, name="%s[_receive_loop]" % (self.name, )) # dispatch Thread to send data # primary purpose is to bundle output by deferring flushes self._send_thread = bg_thread(self._send_loop, name="%s[_send]" % (self.name, ))
def add(self, when, func): ''' Queue a new job to be called at 'when'. 'func' is the job function, typically made with `functools.partial`. ''' assert not self.closed, "add() on closed TimerQueue" self.Q.put((when, seq(), func))