def open(self, E, flags): ''' Open a regular file `E`, allocate FileHandle, return FileHandle index. Increments the kernel reference count. ''' for_read = (flags & O_RDONLY) == O_RDONLY or (flags & O_RDWR) == O_RDWR for_write = (flags & O_WRONLY) == O_WRONLY or (flags & O_RDWR) == O_RDWR for_append = (flags & O_APPEND) == O_APPEND for_trunc = (flags & O_TRUNC) == O_TRUNC debug( "for_read=%s, for_write=%s, for_append=%s", for_read, for_write, for_append ) if for_trunc and not for_write: OS_EINVAL("O_TRUNC requires O_WRONLY or O_RDWR") if for_append and not for_write: OS_EINVAL("O_APPEND requires O_WRONLY or O_RDWR") if (for_write and not for_append) and self.append_only: OS_EINVAL("fs is append_only but no O_APPEND") if for_trunc and self.append_only: OS_EINVAL("fs is append_only but O_TRUNC") if (for_write or for_append) and self.readonly: error("fs is readonly") OS_EROFS("fs is readonly") if E.issym: if flags & O_NOFOLLOW: OS_ELOOP("open symlink with O_NOFOLLOW") OS_EINVAL("open(%s)" % (E,)) elif not E.isfile: OS_EINVAL("open of nonfile: %s" % (E,)) FH = FileHandle(self, E, for_read, for_write, for_append, lock=self._lock) if flags & O_TRUNC: FH.truncate(0) return self._new_file_handle_index(FH)
def iterate_once(): ''' Call `iterate`. Place the result on outQ. Close the queue at end of iteration or other exception. Otherwise, requeue ourself to collect the next iteration value. ''' if test_ready is not None and not test_ready(): raise RetryError("iterate_once: not ready yet") try: item = iterate() except StopIteration: outQ.close() R.result = iterationss[0] except Exception as e: # pylint: disable=broad-except exception( "defer_iterable: iterate_once: exception during iteration: %s", e ) outQ.close() R.exc_info = sys.exc_info() else: iterationss[0] += 1 # put the item onto the output queue # this may itself defer various tasks (eg in a pipeline) debug("L.defer_iterable: iterate_once: %s.put(%r)", outQ, item) outQ.put(item) # now queue another iteration to run after those defered tasks self._defer(iterate_once)
def acquire(self, *a): ''' Acquire the lock. ''' # quietly support Python 3 arguments after blocking parameter blocking = True if a: blocking = a[0] a = a[1:] filename, lineno = inspect.stack()[1][1:3] debug("%s:%d: acquire(blocking=%s)", filename, lineno, blocking) if blocking: # blocking # try non-blocking first # if successful, good # otherwise spawn a monitoring thread to report on slow acquisition # and block taken = self.lock.acquire(False) if not taken: Q = Queue() T = Thread(target=self._timed_acquire, args=(Q, filename, lineno)) T.daemon = True T.start() taken = self.lock.acquire(blocking, *a) Q.put(taken) else: # non-blocking: do ordinary lock acquisition taken = self.lock.acquire(blocking, *a) if taken: self.held = (filename, lineno) return taken
def release(self): ''' Release the lock. ''' filename, lineno = inspect.stack()[0][1:3] debug("%s:%d: release()", filename, lineno) self.held = None self.lock.release()
def __rskipto(self, rpos): assert rpos >= 0, "rpos(%d) < 0" % rpos pos = self.__offset ndx = self.__ndx suboff = self.__suboff hops = 0 nskip = rpos while nskip > 0: hops += 1 s = self.__strs[ndx] taillen = len(s) - suboff if taillen <= nskip: nskip -= taillen ndx += 1 suboff = 0 else: suboff += nskip nskip = 0 break debug("__rskipto(%d)@pos=%d took %d hops", rpos, self.__offset, hops) self.__offset += rpos self.__ndx = ndx self.__suboff = suboff
def close(self): ''' Close the socket. ''' with Pfx("%s.close", self): if self._sock is None: ##warning("close when _sock=None") return if self._for_write: shut_mode = socket.SHUT_WR shut_mode_s = 'SHUT_WR' self.flush() else: shut_mode = socket.SHUT_RD shut_mode_s = 'SHUT_RD' with Pfx("_sock.shutdown(%s)", shut_mode_s): try: self._sock.shutdown(shut_mode) except (socket.error, OSError) as e: if e.errno == errno.ENOTCONN: # client end went away ##info("%s", e) pass elif e.errno == errno.EBADF: debug("%s", e) else: raise finally: self._close()
def __exit__(self, exc_type, exc_val, exc_tb): ''' Exit handler: release the "complete" lock; the placeholder function is blocking on this, and will return on its release. ''' global default # pylint: disable=global-statement debug("%s: __exit__: exc_type=%s", self, exc_type) default.pop() return False
def remove(self, key): subdir, msgbase = self.msgmap[key] msgpath = os.path.join(self.path, subdir, msgbase) debug("%s: remove key %s: %s", self, key, msgpath) try: os.remove(msgpath) except OSError as e: warning("%s: remove key %s: %s: %s", self, key, msgpath, e) del self.msgmap[key]
def save_file(self, fp, key=None, flags=''): ''' Save the contents of the file-like object `fp` into the Maildir. Return the key for the saved message. ''' with NamedTemporaryFile('w', dir=os.path.join(self.path, 'tmp')) as T: debug("create new file %s for key %s", T.name, key) T.write(fp.read()) T.flush() return self.save_filepath(T.name, key=key, flags=flags)
def find_user_password(self, realm, authuri): user, password = HTTPPasswordMgrWithDefaultRealm.find_user_password(self, realm, authuri) if user is None: U = URL(authuri, None) netauth = self._netrc.authenticators(U.hostname) if netauth is not None: user, account, password = netauth debug("find_user_password(%r, %r): netrc: user=%r password=%r", realm, authuri, user, password) return user, password
def getChunk(self, ndx): debug("getChunk", ndx) p = self.findChunkIndex(ndx) if p >= len(self.__cache) or ndx < self.__cache[p][0]: # not cached - fetch and relocate self.preload(ndx) p = self.findChunkIndex(ndx) return self.__cache[p]
def _send_loop(self): ''' Send packets upstream. Write every packet directly to self._send. Flush whenever the queue is empty. ''' XX = self.tick ##with Pfx("%s._send", self): with PrePfx("_SEND [%s]", self): with post_condition(("_send is None", lambda: self._send is None)): fp = self._send Q = self._sendQ grace = self.packet_grace for P in Q: sig = (P.channel, P.tag, P.is_request) if sig in self.__sent: raise RuntimeError("second send of %s" % (P, )) self.__sent.add(sig) try: XX(b'>') for bs in P.transcribe_flat(): fp.write(bs) if Q.empty(): # no immediately ready further packets: flush the output buffer if grace > 0: # allow a little time for further Packets to queue XX(b'Sg') sleep(grace) if Q.empty(): # still nothing XX(b'F') fp.flush() else: XX(b'F') fp.flush() except OSError as e: if e.errno == errno.EPIPE: warning("remote end closed") break raise try: XX(b'>EOF') for bs in self.EOF_Packet.transcribe_flat(): fp.write(bs) fp.close() except (OSError, IOError) as e: if e.errno == errno.EPIPE: debug("remote end closed: %s", e) elif e.errno == errno.EBADF: warning("local end closed: %s", e) else: raise except Exception as e: error("(_SEND) UNEXPECTED EXCEPTION: %s %s", e, e.__class__) raise self._send = None
def _packet_disconnect(self, conn): debug("PacketConnection DISCONNECT notification: %s", conn) oconn = self._conn if oconn is conn: self._conn = None self._conn_attempt_last = time.time() else: debug( "disconnect of %s, but that is not the current connection, ignoring", conn)
def callif(self): ''' Trigger a call to `func(self,*self.func_args,**self.func_kwargsw)` if we're pending and not blocked or cancelled. ''' with self._lock: if not self.ready: try: self.call() except (BlockedError, CancellationError) as e: debug("%s.callif: %s", self, e)
def hrefs(self, absolute=False): ''' All 'href=' values from the content HTML 'A' tags. If `absolute`, resolve the sources with respect to our URL. ''' for A in self.As: try: href = strip_whitespace(A['href']) except KeyError: debug("no href, skip %r", A) continue yield URL( (urljoin(self.baseurl, href) if absolute else href), self )
def __setitem__(self, key, value): ''' Set a frame text to `value`. ''' debug("%s: SET TO %r", key, value) if self._valid_frameid(key): self._update_frame(key, value) else: frameid = ID3.names_to_frameids.get(key) if frameid is None: raise KeyError(".%s: no mapping to a frameid" % (key, )) self[frameid] = value
def pipef(*argv, **kw): ''' Context manager returning the standard output of a command. ''' debug("+ %r |", argv) P = pipefrom(argv, **kw) yield P.stdout if P.wait() != 0: pipecmd = ' '.join(argv) raise ValueError("%s: exit status %d" % ( pipecmd, P.returncode, ))
def _act(self, R, target): ''' Perform this Action on behalf of the Target `target`. Arrange to put the result onto `R`. ''' with Pfx("%s.act(target=%s)", self, target): try: debug("start act...") M = target.maker mdebug = M.debug_make v = self.variant if v == 'shell': debug("shell command") shcmd = self.mexpr(self.context, target.namespaces) if M.no_action or not self.silent: print(shcmd) if M.no_action: mdebug("OK (maker.no_action)") R.put(True) return R.put(self._shcmd(target, shcmd)) return if v == 'make': subtargets = self.mexpr(self.context, target.namespaces).split() mdebug("targets = %s", subtargets) subTs = [M[subtarget] for subtarget in subtargets] def _act_after_make(): # analyse success of targets, update R ok = True mdebug = M.debug_make for T in subTs: if T.result: mdebug("submake \"%s\" OK", T) else: ok = False mdebug("submake \"%s\" FAIL", T) R.put(ok) for T in subTs: mdebug("submake \"%s\"", T) T.require() M.after(subTs, _act_after_make) return raise NotImplementedError("unsupported variant: %s" % (self.variant, )) except Exception as e: error("action failed: %s", e) R.put(False)
def message_addresses(M, header_names): ''' Yield (realname, address) pairs from all the named headers. ''' for header_name in header_names: hdrs = M.get_all(header_name, ()) for hdr in hdrs: for realname, address in getaddresses((hdr, )): if not address: debug( "message_addresses(M, %r): header_name %r: hdr=%r: getaddresses() => (%r, %r): DISCARDED", header_names, header_name, hdr, realname, address) else: yield realname, address
def platonic_Store( self, store_name, clause_name, *, path=None, basedir=None, follow_symlinks=False, meta=None, archive=None, hashclass=None, ): ''' Construct a PlatonicStore from a "datadir" clause. ''' if basedir is None: basedir = self.get_default('basedir') if path is None: path = clause_name debug("path from clausename: %r", path) path = longpath(path) debug("longpath(path) ==> %r", path) if not isabspath(path): if path.startswith('./'): path = abspath(path) debug("abspath ==> %r", path) else: if basedir is None: raise ValueError('relative path %r but no basedir' % (path, )) basedir = longpath(basedir) debug("longpath(basedir) ==> %r", basedir) path = joinpath(basedir, path) debug("path ==> %r", path) if follow_symlinks is None: follow_symlinks = False if meta is None: meta_store = None elif isinstance(meta, str): meta_store = Store(meta, self) if isinstance(archive, str): archive = longpath(archive) return PlatonicStore( store_name, path, hashclass=hashclass, indexclass=None, follow_symlinks=follow_symlinks, meta_store=meta_store, archive=archive, flags_prefix='VT_' + clause_name, )
def _debug_watcher(filename, lineno, n, funcname, R): slow = 2 sofar = 0 slowness = 0 while not R.ready: if slowness >= slow: debug("%s:%d: [%d] calling %s, %gs elapsed so far...", filename, lineno, n, funcname, sofar) # reset report time and complain more slowly next time slowness = 0 slow += 1 time.sleep(DEBUG_POLL_RATE) sofar += DEBUG_POLL_RATE slowness += DEBUG_POLL_RATE
def __getitem__(self, key): ''' Fetch the text of the specified frame. ''' if self._valid_frameid(key): frameid = key frame = self.get_frame(frameid) if frame is None: raise KeyError(".%s: no such frame" % (frameid, )) return frame['text'] frameid = ID3.names_to_frameids.get(key) debug("names_to_frameids.get(%r) ==> %r", key, frameid) if frameid is None: raise KeyError(".%s: no mapping to a frameid" % (key, )) return self[frameid]
def pipeline(later, actions, inputs=None, outQ=None, name=None): ''' Construct a function pipeline to be mediated by this Later queue. Return: `input, output` where `input`` is a closeable queue on which more data items can be put and `output` is an iterable from which result can be collected. Parameters: * `actions`: an iterable of filter functions accepting single items from the iterable `inputs`, returning an iterable output. * `inputs`: the initial iterable inputs; this may be None. If missing or None, it is expected that the caller will be supplying input items via `input.put()`. * `outQ`: the optional output queue; if None, an IterableQueue() will be allocated. * `name`: name for the PushQueue implementing this pipeline. If `inputs` is None or `open` is true, the returned `input` requires a call to `input.close()` when no further inputs are to be supplied. Example use with presupplied Later `L`: input, output = L.pipeline( [ ls, filter_ls, ( FUNC_MANY_TO_MANY, lambda items: sorted(list(items)) ), ], ('.', '..', '../..'), ) for item in output: print(item) ''' filter_funcs = list(actions) if not filter_funcs: raise ValueError("no actions") if outQ is None: outQ = IterableQueue(name="pipelineIQ") if name is None: name = "pipelinePQ" pipeline = Pipeline(name, later, filter_funcs, outQ) inQ = pipeline.inQ if inputs is not None: later.defer_iterable(inputs, inQ) else: debug( "%s._pipeline: no inputs, NOT setting up _defer_iterable( inputs, inQ=%r)", later, inQ) return pipeline
def filecache_Store( self, store_name, clause_name, *, path=None, max_files=None, max_file_size=None, basedir=None, backend=None, hashclass=None, ): ''' Construct a FileCacheStore from a "filecache" clause. ''' if basedir is None: basedir = self.get_default('basedir') if path is None: path = clause_name debug("path from clausename: %r", path) path = longpath(path) debug("longpath(path) ==> %r", path) if isinstance(max_files, str): max_files = scaled_value(max_files) if isinstance(max_file_size, str): max_file_size = scaled_value(max_file_size) if backend is None: backend_store = None else: backend_store = self.Store_from_spec(backend) if not isabspath(path): if path.startswith('./'): path = abspath(path) debug("abspath ==> %r", path) else: if basedir is None: raise ValueError('relative path %r but no basedir' % (path, )) basedir = longpath(basedir) debug("longpath(basedir) ==> %r", basedir) path = joinpath(basedir, path) debug("path ==> %r", path) return FileCacheStore( store_name, backend_store, path, max_cachefile_size=max_file_size, max_cachefiles=max_files, hashclass=hashclass, )
def srcs(self, *a, **kw): ''' All 'src=' values from the content HTML. If `absolute`, resolve the sources with respect to our URL. ''' absolute = False if 'absolute' in kw: absolute = kw['absolute'] del kw['absolute'] for A in self.find_all(*a, **kw): try: src = strip_whitespace(A['src']) except KeyError: debug("no src, skip %r", A) continue yield URL( (urljoin(self.baseurl, src) if absolute else src), self )
def doit(self): # pull off our pending task and untick it Tfunc = None with self._lock: if self.pending: T, Twhen, Tfunc = self.pending self.pending = None # run it if we haven't been told not to if Tfunc: try: retval = Tfunc() except Exception as e: # pylint: disable=broad-except exception("func %s threw exception: %s", Tfunc, e) else: debug("func %s returns %s", Tfunc, retval)
def msgmap(self): ''' Scan the maildir, return key->message-info mapping. ''' debug("compute msgmap for %s", self.path) msgmap = {} for subdir in 'new', 'cur': subdirpath = os.path.join(self.path, subdir) for msgbase in os.listdir(subdirpath): if msgbase.startswith('.'): continue try: key, _ = msgbase.split(':', 1) except ValueError: key = msgbase msgmap[key] = (subdir, msgbase) return msgmap
def skip_errs(iterable): ''' Iterate over `iterable` and yield its values. If it raises URLError or HTTPError, report the error and skip the result. ''' debug("skip_errs...") I = iter(iterable) while True: try: i = next(I) except StopIteration: break except (URLError, HTTPError) as e: warning("%s", e) else: debug("skip_errs: yield %r", i) yield i
def sleep(delay): ''' time.sleep() sometimes sleeps significantly less that requested. This function calls time.sleep() until at least `delay` seconds have elapsed, trying to be precise. ''' if delay < 0: raise ValueError("cs.timeutils.sleep: delay should be >= 0, given %g" % (delay, )) t0 = time.time() end = t0 + delay while t0 < end: delay = end - t0 time.sleep(delay) elapsed = time.time() - t0 if elapsed < delay: from cs.logutils import debug debug("time.sleep(%ss) took only %ss", delay, elapsed) t0 = time.time()
def __init__(self, capacity, name=None, inboundCapacity=0, retry_delay=None): ''' Initialise the Later instance. Parameters: * `capacity`: resource contraint on this Later; if an int, it is used to size a Semaphore to constrain the number of dispatched functions which may be in play at a time; if not an int it is presumed to be a suitable Semaphore-like object, perhaps shared with other subsystems. * `name`: optional identifying name for this instance. * `inboundCapacity`: if >0, used as a limit on the number of undispatched functions that may be queued up; the default is 0 (no limit). Calls to submit functions when the inbound limit is reached block until some functions are dispatched. * `retry_delay`: time delay for requeued functions. Default: `DEFAULT_RETRY_DELAY`. ''' if name is None: name = "Later-%d" % (seq(),) if ifdebug(): import inspect # pylint: disable=import-outside-toplevel filename, lineno = inspect.stack()[1][1:3] name = "%s[%s:%d]" % (name, filename, lineno) debug( "Later.__init__(capacity=%s, inboundCapacity=%s, name=%s)", capacity, inboundCapacity, name ) if retry_delay is None: retry_delay = DEFAULT_RETRY_DELAY self.capacity = capacity self.inboundCapacity = inboundCapacity self.retry_delay = retry_delay self.name = name self._lock = Lock() self.outstanding = set() # dispatched but uncompleted LateFunctions self.delayed = set() # unqueued, delayed until specific time self.pending = [] # undispatched LateFunctions, a heap self.running = set() # running LateFunctions # counter tracking jobs queued or active self._state = "" self.logger = None # reporting; see logTo() method self._priority = (0,) self._timerQ = None # queue for delayed requests; instantiated at need # inbound requests queue self._finished = None