def open(self, E, flags):
''' Open a regular file `E`, allocate FileHandle, return FileHandle index.
Increments the kernel reference count.
'''
for_read = (flags & O_RDONLY) == O_RDONLY or (flags & O_RDWR) == O_RDWR
for_write = (flags & O_WRONLY) == O_WRONLY or (flags & O_RDWR) == O_RDWR
for_append = (flags & O_APPEND) == O_APPEND
for_trunc = (flags & O_TRUNC) == O_TRUNC
debug(
"for_read=%s, for_write=%s, for_append=%s", for_read, for_write,
for_append
)
if for_trunc and not for_write:
OS_EINVAL("O_TRUNC requires O_WRONLY or O_RDWR")
if for_append and not for_write:
OS_EINVAL("O_APPEND requires O_WRONLY or O_RDWR")
if (for_write and not for_append) and self.append_only:
OS_EINVAL("fs is append_only but no O_APPEND")
if for_trunc and self.append_only:
OS_EINVAL("fs is append_only but O_TRUNC")
if (for_write or for_append) and self.readonly:
error("fs is readonly")
OS_EROFS("fs is readonly")
if E.issym:
if flags & O_NOFOLLOW:
OS_ELOOP("open symlink with O_NOFOLLOW")
OS_EINVAL("open(%s)" % (E,))
elif not E.isfile:
OS_EINVAL("open of nonfile: %s" % (E,))
FH = FileHandle(self, E, for_read, for_write, for_append, lock=self._lock)
if flags & O_TRUNC:
FH.truncate(0)
return self._new_file_handle_index(FH)
def iterate_once():
''' Call `iterate`. Place the result on outQ.
Close the queue at end of iteration or other exception.
Otherwise, requeue ourself to collect the next iteration value.
'''
if test_ready is not None and not test_ready():
raise RetryError("iterate_once: not ready yet")
try:
item = iterate()
except StopIteration:
outQ.close()
R.result = iterationss[0]
except Exception as e: # pylint: disable=broad-except
exception(
"defer_iterable: iterate_once: exception during iteration: %s", e
)
outQ.close()
R.exc_info = sys.exc_info()
else:
iterationss[0] += 1
# put the item onto the output queue
# this may itself defer various tasks (eg in a pipeline)
debug("L.defer_iterable: iterate_once: %s.put(%r)", outQ, item)
outQ.put(item)
# now queue another iteration to run after those defered tasks
self._defer(iterate_once)
def acquire(self, *a):
''' Acquire the lock.
'''
# quietly support Python 3 arguments after blocking parameter
blocking = True
if a:
blocking = a[0]
a = a[1:]
filename, lineno = inspect.stack()[1][1:3]
debug("%s:%d: acquire(blocking=%s)", filename, lineno, blocking)
if blocking:
# blocking
# try non-blocking first
# if successful, good
# otherwise spawn a monitoring thread to report on slow acquisition
# and block
taken = self.lock.acquire(False)
if not taken:
Q = Queue()
T = Thread(target=self._timed_acquire,
args=(Q, filename, lineno))
T.daemon = True
T.start()
taken = self.lock.acquire(blocking, *a)
Q.put(taken)
else:
# non-blocking: do ordinary lock acquisition
taken = self.lock.acquire(blocking, *a)
if taken:
self.held = (filename, lineno)
return taken
def release(self):
''' Release the lock.
'''
filename, lineno = inspect.stack()[0][1:3]
debug("%s:%d: release()", filename, lineno)
self.held = None
self.lock.release()
def __rskipto(self, rpos):
assert rpos >= 0, "rpos(%d) < 0" % rpos
pos = self.__offset
ndx = self.__ndx
suboff = self.__suboff
hops = 0
nskip = rpos
while nskip > 0:
hops += 1
s = self.__strs[ndx]
taillen = len(s) - suboff
if taillen <= nskip:
nskip -= taillen
ndx += 1
suboff = 0
else:
suboff += nskip
nskip = 0
break
debug("__rskipto(%d)@pos=%d took %d hops", rpos, self.__offset, hops)
self.__offset += rpos
self.__ndx = ndx
self.__suboff = suboff
def close(self):
''' Close the socket.
'''
with Pfx("%s.close", self):
if self._sock is None:
##warning("close when _sock=None")
return
if self._for_write:
shut_mode = socket.SHUT_WR
shut_mode_s = 'SHUT_WR'
self.flush()
else:
shut_mode = socket.SHUT_RD
shut_mode_s = 'SHUT_RD'
with Pfx("_sock.shutdown(%s)", shut_mode_s):
try:
self._sock.shutdown(shut_mode)
except (socket.error, OSError) as e:
if e.errno == errno.ENOTCONN:
# client end went away
##info("%s", e)
pass
elif e.errno == errno.EBADF:
debug("%s", e)
else:
raise
finally:
self._close()
def __exit__(self, exc_type, exc_val, exc_tb):
''' Exit handler: release the "complete" lock; the placeholder
function is blocking on this, and will return on its release.
'''
global default # pylint: disable=global-statement
debug("%s: __exit__: exc_type=%s", self, exc_type)
default.pop()
return False
def remove(self, key):
subdir, msgbase = self.msgmap[key]
msgpath = os.path.join(self.path, subdir, msgbase)
debug("%s: remove key %s: %s", self, key, msgpath)
try:
os.remove(msgpath)
except OSError as e:
warning("%s: remove key %s: %s: %s", self, key, msgpath, e)
del self.msgmap[key]
def save_file(self, fp, key=None, flags=''):
''' Save the contents of the file-like object `fp` into the Maildir.
Return the key for the saved message.
'''
with NamedTemporaryFile('w', dir=os.path.join(self.path, 'tmp')) as T:
debug("create new file %s for key %s", T.name, key)
T.write(fp.read())
T.flush()
return self.save_filepath(T.name, key=key, flags=flags)
def find_user_password(self, realm, authuri):
user, password = HTTPPasswordMgrWithDefaultRealm.find_user_password(self, realm, authuri)
if user is None:
U = URL(authuri, None)
netauth = self._netrc.authenticators(U.hostname)
if netauth is not None:
user, account, password = netauth
debug("find_user_password(%r, %r): netrc: user=%r password=%r", realm, authuri, user, password)
return user, password
def getChunk(self, ndx):
debug("getChunk", ndx)
p = self.findChunkIndex(ndx)
if p >= len(self.__cache) or ndx < self.__cache[p][0]:
# not cached - fetch and relocate
self.preload(ndx)
p = self.findChunkIndex(ndx)
return self.__cache[p]
def _send_loop(self):
''' Send packets upstream.
Write every packet directly to self._send.
Flush whenever the queue is empty.
'''
XX = self.tick
##with Pfx("%s._send", self):
with PrePfx("_SEND [%s]", self):
with post_condition(("_send is None", lambda: self._send is None)):
fp = self._send
Q = self._sendQ
grace = self.packet_grace
for P in Q:
sig = (P.channel, P.tag, P.is_request)
if sig in self.__sent:
raise RuntimeError("second send of %s" % (P, ))
self.__sent.add(sig)
try:
XX(b'>')
for bs in P.transcribe_flat():
fp.write(bs)
if Q.empty():
# no immediately ready further packets: flush the output buffer
if grace > 0:
# allow a little time for further Packets to queue
XX(b'Sg')
sleep(grace)
if Q.empty():
# still nothing
XX(b'F')
fp.flush()
else:
XX(b'F')
fp.flush()
except OSError as e:
if e.errno == errno.EPIPE:
warning("remote end closed")
break
raise
try:
XX(b'>EOF')
for bs in self.EOF_Packet.transcribe_flat():
fp.write(bs)
fp.close()
except (OSError, IOError) as e:
if e.errno == errno.EPIPE:
debug("remote end closed: %s", e)
elif e.errno == errno.EBADF:
warning("local end closed: %s", e)
else:
raise
except Exception as e:
error("(_SEND) UNEXPECTED EXCEPTION: %s %s", e,
e.__class__)
raise
self._send = None
def _packet_disconnect(self, conn):
debug("PacketConnection DISCONNECT notification: %s", conn)
oconn = self._conn
if oconn is conn:
self._conn = None
self._conn_attempt_last = time.time()
else:
debug(
"disconnect of %s, but that is not the current connection, ignoring",
conn)
def callif(self):
''' Trigger a call to `func(self,*self.func_args,**self.func_kwargsw)`
if we're pending and not blocked or cancelled.
'''
with self._lock:
if not self.ready:
try:
self.call()
except (BlockedError, CancellationError) as e:
debug("%s.callif: %s", self, e)
def hrefs(self, absolute=False):
''' All 'href=' values from the content HTML 'A' tags.
If `absolute`, resolve the sources with respect to our URL.
'''
for A in self.As:
try:
href = strip_whitespace(A['href'])
except KeyError:
debug("no href, skip %r", A)
continue
yield URL( (urljoin(self.baseurl, href) if absolute else href), self )
def __setitem__(self, key, value):
''' Set a frame text to `value`.
'''
debug("%s: SET TO %r", key, value)
if self._valid_frameid(key):
self._update_frame(key, value)
else:
frameid = ID3.names_to_frameids.get(key)
if frameid is None:
raise KeyError(".%s: no mapping to a frameid" % (key, ))
self[frameid] = value
def pipef(*argv, **kw):
''' Context manager returning the standard output of a command.
'''
debug("+ %r |", argv)
P = pipefrom(argv, **kw)
yield P.stdout
if P.wait() != 0:
pipecmd = ' '.join(argv)
raise ValueError("%s: exit status %d" % (
pipecmd,
P.returncode,
))
def _act(self, R, target):
''' Perform this Action on behalf of the Target `target`.
Arrange to put the result onto `R`.
'''
with Pfx("%s.act(target=%s)", self, target):
try:
debug("start act...")
M = target.maker
mdebug = M.debug_make
v = self.variant
if v == 'shell':
debug("shell command")
shcmd = self.mexpr(self.context, target.namespaces)
if M.no_action or not self.silent:
print(shcmd)
if M.no_action:
mdebug("OK (maker.no_action)")
R.put(True)
return
R.put(self._shcmd(target, shcmd))
return
if v == 'make':
subtargets = self.mexpr(self.context,
target.namespaces).split()
mdebug("targets = %s", subtargets)
subTs = [M[subtarget] for subtarget in subtargets]
def _act_after_make():
# analyse success of targets, update R
ok = True
mdebug = M.debug_make
for T in subTs:
if T.result:
mdebug("submake \"%s\" OK", T)
else:
ok = False
mdebug("submake \"%s\" FAIL", T)
R.put(ok)
for T in subTs:
mdebug("submake \"%s\"", T)
T.require()
M.after(subTs, _act_after_make)
return
raise NotImplementedError("unsupported variant: %s" %
(self.variant, ))
except Exception as e:
error("action failed: %s", e)
R.put(False)
def message_addresses(M, header_names):
''' Yield (realname, address) pairs from all the named headers.
'''
for header_name in header_names:
hdrs = M.get_all(header_name, ())
for hdr in hdrs:
for realname, address in getaddresses((hdr, )):
if not address:
debug(
"message_addresses(M, %r): header_name %r: hdr=%r: getaddresses() => (%r, %r): DISCARDED",
header_names, header_name, hdr, realname, address)
else:
yield realname, address
def platonic_Store(
self,
store_name,
clause_name,
*,
path=None,
basedir=None,
follow_symlinks=False,
meta=None,
archive=None,
hashclass=None,
):
''' Construct a PlatonicStore from a "datadir" clause.
'''
if basedir is None:
basedir = self.get_default('basedir')
if path is None:
path = clause_name
debug("path from clausename: %r", path)
path = longpath(path)
debug("longpath(path) ==> %r", path)
if not isabspath(path):
if path.startswith('./'):
path = abspath(path)
debug("abspath ==> %r", path)
else:
if basedir is None:
raise ValueError('relative path %r but no basedir' %
(path, ))
basedir = longpath(basedir)
debug("longpath(basedir) ==> %r", basedir)
path = joinpath(basedir, path)
debug("path ==> %r", path)
if follow_symlinks is None:
follow_symlinks = False
if meta is None:
meta_store = None
elif isinstance(meta, str):
meta_store = Store(meta, self)
if isinstance(archive, str):
archive = longpath(archive)
return PlatonicStore(
store_name,
path,
hashclass=hashclass,
indexclass=None,
follow_symlinks=follow_symlinks,
meta_store=meta_store,
archive=archive,
flags_prefix='VT_' + clause_name,
)
def _debug_watcher(filename, lineno, n, funcname, R):
slow = 2
sofar = 0
slowness = 0
while not R.ready:
if slowness >= slow:
debug("%s:%d: [%d] calling %s, %gs elapsed so far...", filename,
lineno, n, funcname, sofar)
# reset report time and complain more slowly next time
slowness = 0
slow += 1
time.sleep(DEBUG_POLL_RATE)
sofar += DEBUG_POLL_RATE
slowness += DEBUG_POLL_RATE
def __getitem__(self, key):
''' Fetch the text of the specified frame.
'''
if self._valid_frameid(key):
frameid = key
frame = self.get_frame(frameid)
if frame is None:
raise KeyError(".%s: no such frame" % (frameid, ))
return frame['text']
frameid = ID3.names_to_frameids.get(key)
debug("names_to_frameids.get(%r) ==> %r", key, frameid)
if frameid is None:
raise KeyError(".%s: no mapping to a frameid" % (key, ))
return self[frameid]
def pipeline(later, actions, inputs=None, outQ=None, name=None):
''' Construct a function pipeline to be mediated by this Later queue.
Return: `input, output`
where `input`` is a closeable queue on which more data items can be put
and `output` is an iterable from which result can be collected.
Parameters:
* `actions`: an iterable of filter functions accepting
single items from the iterable `inputs`, returning an
iterable output.
* `inputs`: the initial iterable inputs; this may be None.
If missing or None, it is expected that the caller will
be supplying input items via `input.put()`.
* `outQ`: the optional output queue; if None, an IterableQueue() will be
allocated.
* `name`: name for the PushQueue implementing this pipeline.
If `inputs` is None or `open` is true, the returned `input` requires
a call to `input.close()` when no further inputs are to be supplied.
Example use with presupplied Later `L`:
input, output = L.pipeline(
[
ls,
filter_ls,
( FUNC_MANY_TO_MANY, lambda items: sorted(list(items)) ),
],
('.', '..', '../..'),
)
for item in output:
print(item)
'''
filter_funcs = list(actions)
if not filter_funcs:
raise ValueError("no actions")
if outQ is None:
outQ = IterableQueue(name="pipelineIQ")
if name is None:
name = "pipelinePQ"
pipeline = Pipeline(name, later, filter_funcs, outQ)
inQ = pipeline.inQ
if inputs is not None:
later.defer_iterable(inputs, inQ)
else:
debug(
"%s._pipeline: no inputs, NOT setting up _defer_iterable( inputs, inQ=%r)",
later, inQ)
return pipeline
def filecache_Store(
self,
store_name,
clause_name,
*,
path=None,
max_files=None,
max_file_size=None,
basedir=None,
backend=None,
hashclass=None,
):
''' Construct a FileCacheStore from a "filecache" clause.
'''
if basedir is None:
basedir = self.get_default('basedir')
if path is None:
path = clause_name
debug("path from clausename: %r", path)
path = longpath(path)
debug("longpath(path) ==> %r", path)
if isinstance(max_files, str):
max_files = scaled_value(max_files)
if isinstance(max_file_size, str):
max_file_size = scaled_value(max_file_size)
if backend is None:
backend_store = None
else:
backend_store = self.Store_from_spec(backend)
if not isabspath(path):
if path.startswith('./'):
path = abspath(path)
debug("abspath ==> %r", path)
else:
if basedir is None:
raise ValueError('relative path %r but no basedir' %
(path, ))
basedir = longpath(basedir)
debug("longpath(basedir) ==> %r", basedir)
path = joinpath(basedir, path)
debug("path ==> %r", path)
return FileCacheStore(
store_name,
backend_store,
path,
max_cachefile_size=max_file_size,
max_cachefiles=max_files,
hashclass=hashclass,
)
def srcs(self, *a, **kw):
''' All 'src=' values from the content HTML.
If `absolute`, resolve the sources with respect to our URL.
'''
absolute = False
if 'absolute' in kw:
absolute = kw['absolute']
del kw['absolute']
for A in self.find_all(*a, **kw):
try:
src = strip_whitespace(A['src'])
except KeyError:
debug("no src, skip %r", A)
continue
yield URL( (urljoin(self.baseurl, src) if absolute else src), self )
def doit(self):
# pull off our pending task and untick it
Tfunc = None
with self._lock:
if self.pending:
T, Twhen, Tfunc = self.pending
self.pending = None
# run it if we haven't been told not to
if Tfunc:
try:
retval = Tfunc()
except Exception as e: # pylint: disable=broad-except
exception("func %s threw exception: %s", Tfunc, e)
else:
debug("func %s returns %s", Tfunc, retval)
def msgmap(self):
''' Scan the maildir, return key->message-info mapping.
'''
debug("compute msgmap for %s", self.path)
msgmap = {}
for subdir in 'new', 'cur':
subdirpath = os.path.join(self.path, subdir)
for msgbase in os.listdir(subdirpath):
if msgbase.startswith('.'):
continue
try:
key, _ = msgbase.split(':', 1)
except ValueError:
key = msgbase
msgmap[key] = (subdir, msgbase)
return msgmap
def skip_errs(iterable):
''' Iterate over `iterable` and yield its values.
If it raises URLError or HTTPError, report the error and skip the result.
'''
debug("skip_errs...")
I = iter(iterable)
while True:
try:
i = next(I)
except StopIteration:
break
except (URLError, HTTPError) as e:
warning("%s", e)
else:
debug("skip_errs: yield %r", i)
yield i
def sleep(delay):
''' time.sleep() sometimes sleeps significantly less that requested.
This function calls time.sleep() until at least `delay` seconds have
elapsed, trying to be precise.
'''
if delay < 0:
raise ValueError("cs.timeutils.sleep: delay should be >= 0, given %g" %
(delay, ))
t0 = time.time()
end = t0 + delay
while t0 < end:
delay = end - t0
time.sleep(delay)
elapsed = time.time() - t0
if elapsed < delay:
from cs.logutils import debug
debug("time.sleep(%ss) took only %ss", delay, elapsed)
t0 = time.time()
def __init__(self, capacity, name=None, inboundCapacity=0, retry_delay=None):
''' Initialise the Later instance.
Parameters:
* `capacity`: resource contraint on this Later; if an int, it is used
to size a Semaphore to constrain the number of dispatched functions
which may be in play at a time; if not an int it is presumed to be a
suitable Semaphore-like object, perhaps shared with other subsystems.
* `name`: optional identifying name for this instance.
* `inboundCapacity`: if >0, used as a limit on the number of
undispatched functions that may be queued up; the default is 0 (no
limit). Calls to submit functions when the inbound limit is reached
block until some functions are dispatched.
* `retry_delay`: time delay for requeued functions.
Default: `DEFAULT_RETRY_DELAY`.
'''
if name is None:
name = "Later-%d" % (seq(),)
if ifdebug():
import inspect # pylint: disable=import-outside-toplevel
filename, lineno = inspect.stack()[1][1:3]
name = "%s[%s:%d]" % (name, filename, lineno)
debug(
"Later.__init__(capacity=%s, inboundCapacity=%s, name=%s)", capacity,
inboundCapacity, name
)
if retry_delay is None:
retry_delay = DEFAULT_RETRY_DELAY
self.capacity = capacity
self.inboundCapacity = inboundCapacity
self.retry_delay = retry_delay
self.name = name
self._lock = Lock()
self.outstanding = set() # dispatched but uncompleted LateFunctions
self.delayed = set() # unqueued, delayed until specific time
self.pending = [] # undispatched LateFunctions, a heap
self.running = set() # running LateFunctions
# counter tracking jobs queued or active
self._state = ""
self.logger = None # reporting; see logTo() method
self._priority = (0,)
self._timerQ = None # queue for delayed requests; instantiated at need
# inbound requests queue
self._finished = None
