def spring(commands, env=None, stdout=None, stderr=b""):
"""Execute a series of commands and accumulate their output to a single destination."""
with defer() as later:
with defer() as here:
# A spring never receives any input from stdin, i.e., we always
# want it to be redirected from /dev/null.
fds = _PipelineFileDescriptors(later, here, None, stdout, stderr)
# When running the spring we need to alternate between spawning
# new processes and polling for data. In that scenario, we do not
# want the polling to block until we started processes for all
# commands passed in.
fds.blockable(False)
# Finally execute our spring and pass in the prepared file
# descriptors to use.
pids, poller, status, failed = _spring(commands, env, fds)
# We started all processes and will wait for them to finish. From
# now on we can allow any invocation of poll to block.
fds.blockable(True)
# Poll until there is no more data.
for _ in poller:
pass
data_out, data_err = fds.data()
_wait(pids, commands, data_err, status=status, failed=failed)
return data_out, data_err
def pipeline(commands, env=None, stdin=None, stdout=None, stderr=b""):
"""Execute a pipeline, supplying the given data to stdin and reading from stdout & stderr.
This function executes a pipeline of commands and connects their
stdin and stdout file descriptors as desired. All keyword parameters
can be either None (in which case they get implicitly redirected
to/from a null device), a valid file descriptor, or some data. In
case data is given (which should be a byte-like object) it will be
fed into the standard input of the first command (in case of stdin)
or be used as the initial buffer content of data to read (stdout and
stderr) of the last command (which means all actually read data will
just be appended).
"""
with defer() as later:
with defer() as here:
# Set up the file descriptors to pass to our execution pipeline.
fds = _PipelineFileDescriptors(later, here, stdin, stdout, stderr)
# Finally execute our pipeline and pass in the prepared file
# descriptors to use.
pids = _pipeline(commands, env, fds.stdin(), fds.stdout(),
fds.stderr())
for _ in fds.poll():
pass
data_out, data_err = fds.data()
# We have read or written all data that was available, the last thing
# to do is to wait for all the processes to finish and to clean them
# up.
_wait(pids, commands, data_err)
return data_out, data_err
def testDeferNested(self):
"""Verify that defer blocks can be nested."""
with self.assertRaises(Exception):
with defer() as d1:
# Note that when this expression gets evaluated the value of
# self._counter.count() should be five.
d1.defer(lambda: self._counter.set(self._counter.count() * 3))
with defer() as d2:
# Increment d1 once more but in different block.
d1.defer(self._counter.increment)
# And now also let d2 change the value.
d2.defer(self._counter.set, 4)
raise Exception()
self.assertEqual(self._counter.count(), 15)
def testDeferWithException(self):
"""Verify that defer functionality behaves correctly in the face of exceptions."""
with self.assertRaises(Exception):
with defer() as d:
d.defer(self._counter.increment)
raise Exception()
self.assertEqual(self._counter.count(), 1)
def testDeferNested(self):
"""Verify that defer blocks can be nested."""
with self.assertRaises(Exception):
with defer() as d1:
# Note that when this expression gets evaluated the value of
# self._counter.count() should be five.
d1.defer(lambda: self._counter.set(self._counter.count() * 3))
with defer() as d2:
# Increment d1 once more but in different block.
d1.defer(self._counter.increment)
# And now also let d2 change the value.
d2.defer(self._counter.set, 4)
raise Exception()
self.assertEqual(self._counter.count(), 15)
def testDeferWithException(self):
"""Verify that defer functionality behaves correctly in the face of exceptions."""
with self.assertRaises(Exception):
with defer() as d:
d.defer(self._counter.increment)
raise Exception()
self.assertEqual(self._counter.count(), 1)
def testDeferRunEarlyRunOnce(self):
"""Verify that we can invoke a deferred function early using the returned object."""
with defer() as d:
deferer = d.defer(self._counter.increment)
deferer()
self.assertEqual(self._counter.count(), 1)
# Should not be run a second time.
self.assertEqual(self._counter.count(), 1)
def testDeferHasCorrectOrder(self):
"""Verify that deferred functions are invoked in the right order."""
with defer() as d:
d.defer(self._counter.set, 2)
d.defer(self._counter.set, 1)
# Functions should be invoked in reverse order to honor potential
# dependencies between objects.
self.assertEqual(self._counter.count(), 2)
def testDeferCorrectParameterPassing(self):
"""Verify that variable and keyword arguments can be passed to a deferred function."""
with defer() as d:
d.defer(self._counter.add, 1, 2, third=3)
# A lambda expression should work as well.
d.defer(lambda: self._counter.add(first=4, second=5, third=6))
# All parameters should have been passed to the add() invocation.
self.assertEqual(self._counter.count(), 21)
def testDeferRunEarlyRunOnce(self):
"""Verify that we can invoke a deferred function early using the returned object."""
with defer() as d:
deferer = d.defer(self._counter.increment)
deferer()
self.assertEqual(self._counter.count(), 1)
# Should not be run a second time.
self.assertEqual(self._counter.count(), 1)
def testDeferHasCorrectOrder(self):
"""Verify that deferred functions are invoked in the right order."""
with defer() as d:
d.defer(self._counter.set, 2)
d.defer(self._counter.set, 1)
# Functions should be invoked in reverse order to honor potential
# dependencies between objects.
self.assertEqual(self._counter.count(), 2)
def testDeferCorrectParameterPassing(self):
"""Verify that variable and keyword arguments can be passed to a deferred function."""
with defer() as d:
d.defer(self._counter.add, 1, 2, third=3)
# A lambda expression should work as well.
d.defer(lambda: self._counter.add(first=4, second=5, third=6))
# All parameters should have been passed to the add() invocation.
self.assertEqual(self._counter.count(), 21)
def testDefer(self):
"""Verify that with non-exceptional control flow a deferred function is invoked."""
self.assertEqual(self._counter.count(), 0)
with defer() as d:
d.defer(self._counter.increment)
# Must not be incremented immediately, only after block exit.
self.assertEqual(self._counter.count(), 0)
self.assertEqual(self._counter.count(), 1)
def testDeferReleaseAll(self):
"""Test the release functionality of defer objects."""
with defer() as d:
d.defer(self._counter.increment)
d.defer(self._counter.increment)
d.release()
# No function should have been executed.
self.assertEqual(self._counter.count(), 0)
def testDeferFunctionRelease(self):
"""Test the release functionality of deferred functions."""
with defer() as d:
f = d.defer(self._counter.increment)
_ = d.defer(self._counter.increment)
f.release()
# Only one increment should have been executed.
self.assertEqual(self._counter.count(), 1)
def testDeferReleaseAll(self):
"""Test the release functionality of defer objects."""
with defer() as d:
d.defer(self._counter.increment)
d.defer(self._counter.increment)
d.release()
# No function should have been executed.
self.assertEqual(self._counter.count(), 0)
def testDeferFunctionRelease(self):
"""Test the release functionality of deferred functions."""
with defer() as d:
f = d.defer(self._counter.increment)
_ = d.defer(self._counter.increment)
f.release()
# Only one increment should have been executed.
self.assertEqual(self._counter.count(), 1)
def testDefer(self):
"""Verify that with non-exceptional control flow a deferred function is invoked."""
self.assertEqual(self._counter.count(), 0)
with defer() as d:
d.defer(self._counter.increment)
# Must not be incremented immediately, only after block exit.
self.assertEqual(self._counter.count(), 0)
self.assertEqual(self._counter.count(), 1)
def poll(self):
"""Poll the file pipe descriptors for more data until each indicated that it is done.
There are two modes in which this method can work. In blocking
mode (the default), we will block waiting for new data to become
available for processing. In non-blocking mode we yield if no more
data is currently available but can resume polling later. The
blocking mode can be influenced via the blockable member function.
Note that this change can even happen after we yielded execution
in the non blockable case.
Note that because we require non-blocking behavior in order to
support springs, this function uses 'yield' instead of 'return'
for conveying any results to the caller (even in the blockable
case). The reason is a little Python oddity where a function that
yields anything (even in a path that is never reached), always
implicitly returns a generator rather as opposed to a "direct"
result.
"""
def pollWrite(data):
"""Conditionally set up polling for write events."""
if data:
poll_.register(data["out"], _OUT)
data["unreg"] = d.defer(poll_.unregister, data["out"])
polls[data["out"]] = data
def pollRead(data):
"""Conditionally set up polling for read events."""
if data:
poll_.register(data["in"], _IN)
data["unreg"] = d.defer(poll_.unregister, data["in"])
polls[data["in"]] = data
# We need a poll object if we want to send any data to stdin or want
# to receive any data from stdout or stderr.
if self._stdin or self._stdout or self._stderr:
poll_ = poll()
# We use a dictionary here to elegantly look up the entry (which is,
# another dictionary) for the respective file descriptor we received
# an event for and to decide if we need to poll more.
polls = {}
with defer() as d:
# Set up the polling infrastructure.
pollWrite(self._stdin)
pollRead(self._stdout)
pollRead(self._stderr)
while polls:
events = poll_.poll(self._timeout)
for fd, event in events:
close = False
data = polls[fd]
# Note that reading (POLLIN or POLLPRI) and writing (POLLOUT)
# are mutually exclusive operations on a pipe. All can be
# combined with a HUP or with other errors (POLLERR or
# POLLNVAL; even though we did not subscribe to them), though.
if event & POLLOUT:
close = _write(data)
elif event & POLLIN or event & POLLPRI:
if event & POLLHUP:
# In case we received a combination of a data-is-available
# and a HUP event we need to make sure that we flush the
# entire pipe buffer before we stop the polling. Otherwise
# we might leave data unread that was successfully sent to
# us.
# Note that from a logical point of view this problem
# occurs only in the receive case. In the write case we
# have full control over the file descriptor ourselves and
# if the remote side closes its part there is no point in
# sending any more data.
while not _read(data):
pass
else:
close = _read(data)
# We explicitly (and early, compared to the defers we
# scheduled previously) close the file descriptor on POLLHUP,
# when we received EOF (for reading), or run out of data to
# send (for writing).
if event & POLLHUP or close:
data["close"]()
data["unreg"]()
del polls[fd]
# All error codes are reported to clients such that they can
# deal with potentially incomplete data.
if event & (POLLERR | POLLNVAL):
string = eventToString(event)
error = "Error while polling for new data, event: {s} ({e})"
error = error.format(s=string, e=event)
raise ConnectionError(error)
if self._timeout is not None:
yield
yield
