def base_case(self):
queue = Queue()
t = EHThread(target=self.worker, args=[queue])
t.start()
t.join()
assert queue.get(block=False) == 7
assert queue.empty()
def base_case(self):
queue = Queue()
t = EHThread(target=self.worker, args=[queue])
t.start()
t.join()
eq_(queue.get(block=False), 7)
ok_(queue.empty())
def base_case(self):
queue = Queue()
t = EHThread(target=self.worker, args=[queue])
t.start()
t.join()
eq_(queue.get(block=False), 7)
ok_(queue.empty())
def exhibits_is_dead_flag(self):
t = EHThread(target=self.worker, args=[None])
t.start()
t.join()
ok_(t.is_dead)
t = EHThread(target=self.worker, args=[Queue()])
t.start()
t.join()
ok_(not t.is_dead)
def exhibits_is_dead_flag(self):
t = EHThread(target=self.worker, args=[None])
t.start()
t.join()
assert t.is_dead
t = EHThread(target=self.worker, args=[Queue()])
t.start()
t.join()
assert not t.is_dead
def exhibits_is_dead_flag(self):
# Spin up a thread that will except internally (can't put() on a
# None object)
t = EHThread(target=self.worker, args=[None])
t.start()
t.join()
# Excepted -> it's dead
assert t.is_dead
# Spin up a happy thread that can exit peacefully (it's not "dead",
# though...maybe we should change that terminology)
t = EHThread(target=self.worker, args=[Queue()])
t.start()
t.join()
# Not dead, just uh...sleeping?
assert not t.is_dead
def catches_exceptions(self):
# Induce exception by submitting a bad queue obj
t = EHThread(target=self.worker, args=[None])
t.start()
t.join()
wrapper = t.exception()
ok_(isinstance(wrapper, ExceptionWrapper))
eq_(wrapper.kwargs, {'args': [None], 'target': self.worker})
eq_(wrapper.type, AttributeError)
ok_(isinstance(wrapper.value, AttributeError))
def catches_exceptions(self):
# Induce exception by submitting a bad queue obj
t = EHThread(target=self.worker, args=[None])
t.start()
t.join()
wrapper = t.exception()
assert isinstance(wrapper, ExceptionWrapper)
assert wrapper.kwargs == {"args": [None], "target": self.worker}
assert wrapper.type == AttributeError
assert isinstance(wrapper.value, AttributeError)
def catches_exceptions(self):
# Induce exception by submitting a bad queue obj
t = EHThread(target=self.worker, args=[None])
t.start()
t.join()
wrapper = t.exception()
ok_(isinstance(wrapper, ExceptionWrapper))
eq_(wrapper.kwargs, {'args': [None], 'target': self.worker})
eq_(wrapper.type, AttributeError)
ok_(isinstance(wrapper.value, AttributeError))
def manual_threading_works_okay(self):
# TODO: needs https://github.com/pyinvoke/invoke/issues/438 fixed
# before it will reliably pass
skip()
# Kind of silly but a nice base case for "how would someone thread this
# stuff; and are there any bizarre gotchas lurking in default
# config/context/connection state?"
# Specifically, cut up the local (usually 100k's long) words dict into
# per-thread chunks, then read those chunks via shell command, as a
# crummy "make sure each thread isn't polluting things like stored
# stdout" sanity test
queue = Queue()
# TODO: skip test on Windows or find suitable alternative file
with codecs.open(_words, encoding='utf-8') as fd:
data = [x.strip() for x in fd.readlines()]
threads = []
num_words = len(data)
chunksize = len(data) / len(self.cxns) # will be an int, which is fine
for i, cxn in enumerate(self.cxns):
start = i * chunksize
end = max([start + chunksize, num_words])
chunk = data[start:end]
kwargs = dict(
queue=queue,
cxn=cxn,
start=start,
num_words=num_words,
count=len(chunk),
expected=chunk,
)
thread = ExceptionHandlingThread(target=_worker, kwargs=kwargs)
threads.append(thread)
for t in threads:
t.start()
for t in threads:
t.join(5) # Kinda slow, but hey, maybe the test runner is hot
while not queue.empty():
cxn, result, expected = queue.get(block=False)
for resultword, expectedword in zip_longest(result, expected):
err = u"({2!r}, {3!r}->{4!r}) {0!r} != {1!r}".format(
resultword,
expectedword,
cxn,
expected[0],
expected[-1],
)
assert resultword == expectedword, err
def run(self, *args, **kwargs):
results = GroupResult()
queue = Queue()
threads = []
for cxn in self:
my_kwargs = dict(
cxn=cxn,
queue=queue,
args=args,
kwargs=kwargs,
)
thread = ExceptionHandlingThread(
target=thread_worker,
kwargs=my_kwargs,
)
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
# TODO: configurable join timeout
# TODO: (in sudo's version) configurability around interactive
# prompting resulting in an exception instead, as in v1
thread.join()
# Get non-exception results from queue
while not queue.empty():
# TODO: io-sleep? shouldn't matter if all threads are now joined
cxn, result = queue.get(block=False)
# TODO: outstanding musings about how exactly aggregate results
# ought to ideally operate...heterogenous obj like this, multiple
# objs, ??
results[cxn] = result
# Get exceptions from the threads themselves.
# TODO: in a non-thread setup, this would differ, e.g.:
# - a queue if using multiprocessing
# - some other state-passing mechanism if using e.g. coroutines
# - ???
excepted = False
for thread in threads:
wrapper = thread.exception()
if wrapper is not None:
# Outer kwargs is Thread instantiation kwargs, inner is kwargs
# passed to thread target/body.
cxn = wrapper.kwargs['kwargs']['cxn']
results[cxn] = wrapper.value
excepted = True
if excepted:
raise GroupException(results)
return results
def _run_and_kill(self, pty):
def bg_body():
# No reliable way to detect "an exception happened in the inner
# child that wasn't KeyboardInterrupt", so best we can do is:
# * Ensure exited 130
# * Get mad if any output is seen that doesn't look like
# KeyboardInterrupt stacktrace (because it's probably some
# OTHER stacktrace).
pty_flag = "--pty" if pty else "--no-pty"
result = run(
"inv -c signal_tasks expect SIGINT {0}".format(pty_flag),
hide=True,
warn=True,
)
bad_signal = result.exited != 130
output = result.stdout + result.stderr
had_keyboardint = 'KeyboardInterrupt' in output
if bad_signal or (output and not had_keyboardint):
err = "Subprocess had output and/or bad exit:"
raise Exception("{0}\n\n{1}".format(err, result))
# Execute sub-invoke in a thread so we can talk to its subprocess
# while it's running.
# TODO: useful async API for run() which at least wraps threads for
# you, and exposes the inner PID
bg = ExceptionHandlingThread(target=bg_body)
bg.start()
# Wait a bit to ensure subprocess is in the right state & not still
# starting up (lolpython?). NOTE: if you bump this you must also
# bump the `signal.alarm` call within _support/signaling.py!
# Otherwise both tests will always fail as the ALARM fires
# (resulting in "Never got any signals!" in debug log) before this
# here sleep finishes.
time.sleep(2 if PYPY else 1)
# Send expected signal (use pty to ensure no intermediate 'sh'
# processes on Linux; is of no consequence on Darwin.)
interpreter = 'pypy' if PYPY else 'python'
cmd = "pkill -INT -f \"{0}.*inv -c signal_tasks\""
run(cmd.format(interpreter), pty=True)
# Rejoin subprocess thread & check for exceptions
bg.join()
wrapper = bg.exception()
if wrapper:
# This is an ExceptionWrapper, not an actual exception, since
# most places using ExceptionHandlingThread need access to the
# thread's arguments & such. We just want the exception here.
raise wrapper.value
def run(self, *args, **kwargs):
results = GroupResult()
queue = Queue()
threads = []
for cxn in self:
my_kwargs = dict(
cxn=cxn,
queue=queue,
args=args,
kwargs=kwargs,
)
thread = ExceptionHandlingThread(
target=thread_worker,
kwargs=my_kwargs,
)
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
# TODO: configurable join timeout
# TODO: (in sudo's version) configurability around interactive
# prompting resulting in an exception instead, as in v1
thread.join()
# Get non-exception results from queue
while not queue.empty():
# TODO: io-sleep? shouldn't matter if all threads are now joined
cxn, result = queue.get(block=False)
# TODO: outstanding musings about how exactly aggregate results
# ought to ideally operate...heterogenous obj like this, multiple
# objs, ??
results[cxn] = result
# Get exceptions from the threads themselves.
# TODO: in a non-thread setup, this would differ, e.g.:
# - a queue if using multiprocessing
# - some other state-passing mechanism if using e.g. coroutines
# - ???
excepted = False
for thread in threads:
wrapper = thread.exception()
if wrapper is not None:
# Outer kwargs is Thread instantiation kwargs, inner is kwargs
# passed to thread target/body.
cxn = wrapper.kwargs['kwargs']['cxn']
results[cxn] = wrapper.value
excepted = True
if excepted:
raise GroupException(results)
return results
def _run_and_kill(self, pty):
def bg_body():
# No reliable way to detect "an exception happened in the inner
# child that wasn't KeyboardInterrupt", so best we can do is:
# * Ensure exited 130
# * Get mad if any output is seen that doesn't look like
# KeyboardInterrupt stacktrace (because it's probably some
# OTHER stacktrace).
pty_flag = "--pty" if pty else "--no-pty"
result = run(
"inv -c signal_tasks expect SIGINT {0}".format(pty_flag),
hide=True,
warn=True,
)
bad_signal = result.exited != 130
output = result.stdout + result.stderr
had_keyboardint = 'KeyboardInterrupt' in output
if bad_signal or (output and not had_keyboardint):
err = "Subprocess had output and/or bad exit:"
raise Exception("{0}\n\n{1}".format(err, result))
# Execute sub-invoke in a thread so we can talk to its subprocess
# while it's running.
# TODO: useful async API for run() which at least wraps threads for
# you, and exposes the inner PID
bg = ExceptionHandlingThread(target=bg_body)
bg.start()
# Wait a bit to ensure subprocess is in the right state & not still
# starting up (lolpython?). NOTE: if you bump this you must also
# bump the `signal.alarm` call within _support/signaling.py!
# Otherwise both tests will always fail as the ALARM fires
# (resulting in "Never got any signals!" in debug log) before this
# here sleep finishes.
time.sleep(1)
# Send expected signal (use pty to ensure no intermediate 'sh'
# processes on Linux; is of no consequence on Darwin.)
interpreter = 'pypy' if PYPY else 'python'
cmd = "pkill -INT -f \"{0}.*inv -c signal_tasks\""
run(cmd.format(interpreter), pty=True)
# Rejoin subprocess thread & check for exceptions
bg.join()
wrapper = bg.exception()
if wrapper:
# This is an ExceptionWrapper, not an actual exception, since
# most places using ExceptionHandlingThread need access to the
# thread's arguments & such. We just want the exception here.
raise wrapper.value
def _run_and_kill(self, pty):
def bg_body():
# TODO: use 'expect exit of 130' when that's implemented Hide
# output by default, then assume an error & display stderr, if
# there's any stderr. (There's no reliable way to tell the
# subprocess raised an exception, because we'll be interrupted
# before completion, and won't have access to its exit code.)
pty_flag = "--pty" if pty else "--no-pty"
result = run(
"inv -c signal_tasks expect SIGINT {0}".format(pty_flag),
hide=True,
warn=True,
)
if result.exited != 130 or result.stdout or result.stderr:
err = "Subprocess had output and/or bad exit:"
raise Exception("{0}\n\n{1}".format(err, result))
# Execute sub-invoke in a thread so we can talk to its subprocess
# while it's running.
# TODO: useful async API for run() which at least wraps threads for
# you, and exposes the inner PID
bg = ExceptionHandlingThread(target=bg_body)
bg.start()
# Wait a bit to ensure subprocess is in the right state & not still
# starting up (lolpython?)
time.sleep(1)
# Send expected signal (use pty to ensure no intermediate 'sh'
# processes on Linux; is of no consequence on Darwin.)
run("pkill -INT -f \"python.*inv -c signal_tasks\"", pty=True)
# Rejoin subprocess thread & check for exceptions
bg.join()
wrapper = bg.exception()
if wrapper:
# This is an ExceptionWrapper, not an actual exception, since
# most places using ExceptionHandlingThread need access to the
# thread's arguments & such. We just want the exception here.
raise wrapper.value