def test_exit_with_grace():
with TestProcess(sys.executable, '-u', HELPER, 'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(0.05)
sock.connect(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, "ThreadID",
"ProcessID", ">>>")
sock.send(b"print('FOOBAR')\n")
wait_for_strings(client.read, TIMEOUT, "FOOBAR")
wait_for_strings(proc.read, TIMEOUT,
'UID:%s' % os.getuid())
sock.shutdown(socket.SHUT_WR)
select.select([sock], [], [], 5)
sock.recv(1024)
try:
sock.shutdown(socket.SHUT_RD)
except Exception as exc:
print("Failed to SHUT_RD: %s" % exc)
try:
sock.close()
except Exception as exc:
print("Failed to close socket: %s" % exc)
wait_for_strings(proc.read, TIMEOUT, 'DONE.', 'Cleaned up.',
'Waiting for new connection')
def test_trash_input():
with TestProcess(sys.executable, __file__, 'daemon',
'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '{a1}', '{b1}',
'RemotePdb session open at ')
host, port = re.findall("RemotePdb session open at (.+):(.+),",
proc.read())[0]
with TestSocket(
socket.create_connection((host, int(port)),
timeout=TIMEOUT)) as client:
with dump_on_error(client.read):
wait_for_strings(proc.read, TIMEOUT,
'accepted connection from')
wait_for_strings(client.read, TIMEOUT, "-> print('{b2}')")
for i in range(100):
client.fh.write(b'\r\n'.join(b'print("[%d]")' %
(i * 10 + j)
for j in range(10)) +
b'\r\n')
client.fh.write(b'continue\r\n')
wait_for_strings(client.read, TIMEOUT,
*['[%s]' % i for i in range(1000)])
wait_for_strings(proc.read, TIMEOUT, 'DIED.')
def test_exit_with_grace():
with TestProcess(sys.executable, '-u', HELPER, 'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(0.05)
sock.connect(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, "ThreadID", "ProcessID", ">>>")
sock.send(b"print('FOOBAR')\n")
wait_for_strings(client.read, TIMEOUT, "FOOBAR")
wait_for_strings(proc.read, TIMEOUT, 'UID:%s' % os.getuid())
sock.shutdown(socket.SHUT_WR)
select.select([sock], [], [], 5)
sock.recv(1024)
try:
sock.shutdown(socket.SHUT_RD)
except Exception as exc:
print("Failed to SHUT_RD: %s" % exc)
try:
sock.close()
except Exception as exc:
print("Failed to close socket: %s" % exc)
wait_for_strings(proc.read, TIMEOUT, 'DONE.', 'Cleaned up.', 'Waiting for new connection')
def test_path():
with TestProcess(sys.executable, HELPER, 'test_simple') as service:
with dump_on_error(service.read):
wait_for_strings(service.read, TIMEOUT, '/tmp/manhole-')
with TestProcess('manhole-cli', '/tmp/manhole-%s' % service.proc.pid, bufsize=0, stdin=subprocess.PIPE) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, '(ManholeConsole)', '>>>')
client.proc.stdin.write(b"1234+2345\n")
wait_for_strings(client.read, TIMEOUT, '3579')
def test_usr2():
with TestProcess(sys.executable, '-u', HELPER, 'test_oneshot_on_usr2') as service:
with dump_on_error(service.read):
wait_for_strings(service.read, TIMEOUT, 'Not patching os.fork and os.forkpty. Oneshot activation is done by signal')
with TestProcess('manhole-cli', '-USR2', str(service.proc.pid), bufsize=0, stdin=subprocess.PIPE) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, '(ManholeConsole)', '>>>')
client.proc.stdin.write(b"1234+2345\n")
wait_for_strings(client.read, TIMEOUT, '3579')
def test_pid():
with TestProcess(sys.executable, HELPER, 'test_simple') as service:
with dump_on_error(service.read):
wait_for_strings(service.read, TIMEOUT, '/tmp/manhole-')
with TestProcess('manhole-cli', str(service.proc.pid), bufsize=0, stdin=subprocess.PIPE) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, '(ManholeConsole)', '>>>')
client.proc.stdin.write(b"1234+2345\n")
wait_for_strings(client.read, TIMEOUT, '3579')
def test_usr2():
with TestProcess(sys.executable, '-u', HELPER, 'test_oneshot_on_usr2') as service:
with dump_on_error(service.read):
wait_for_strings(service.read, TIMEOUT,
'Not patching os.fork and os.forkpty. Oneshot activation is done by signal')
with TestProcess('manhole-cli', '-USR2', str(service.proc.pid), bufsize=0, stdin=subprocess.PIPE) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, '(ManholeConsole)', '>>>')
client.proc.stdin.write(b"1234+2345\n")
wait_for_strings(client.read, TIMEOUT, '3579')
def test_redirect_stderr_default():
with TestProcess(sys.executable, HELPER, 'test_redirect_stderr_default') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = connect_to_manhole(SOCKET_PATH)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
client.reset()
sock.send(b"import sys\n"
b"sys.stderr.write('OK')\n")
wait_for_strings(client.read, 1, "OK")
def test_redirect_stderr_default():
with TestProcess(sys.executable, HELPER,
'test_redirect_stderr_default') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = connect_to_manhole(SOCKET_PATH)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
client.reset()
sock.send(b"import sys\n" b"sys.stderr.write('OK')\n")
wait_for_strings(client.read, 1, "OK")
def test_simple():
with TestProcess(sys.executable, __file__, 'daemon', 'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT,
'{a1}',
'{b1}',
'RemotePdb session open at ')
host, port = re.findall("RemotePdb session open at (.+):(.+),", proc.read())[0]
with TestSocket(socket.create_connection((host, int(port)), timeout=TIMEOUT)) as client:
with dump_on_error(client.read):
wait_for_strings(proc.read, TIMEOUT, 'accepted connection from')
wait_for_strings(client.read, TIMEOUT, "-> print('{b2}')")
client.fh.write(b'quit\r\n')
wait_for_strings(proc.read, TIMEOUT, 'DIED.')
def test_simple():
with TestProcess(sys.executable, __file__, 'daemon', 'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT,
'{a1}',
'{b1}',
'RemotePdb session open at ')
host, port = re.findall("RemotePdb session open at (.+):(.+),", proc.read())[0]
with TestSocket(socket.create_connection((host, int(port)), timeout=TIMEOUT)) as client:
with dump_on_error(client.read):
wait_for_strings(proc.read, TIMEOUT, 'accepted connection from')
wait_for_strings(client.read, TIMEOUT, "-> print('{b2}')")
client.fh.write(b'quit\r\n')
wait_for_strings(proc.read, TIMEOUT, 'DIED.')
def test_socket_path_with_fork():
with TestProcess(sys.executable, '-u', HELPER, 'test_socket_path_with_fork') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Not patching os.fork and os.forkpty. Using user socket path')
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = connect_to_manhole(SOCKET_PATH)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, "ProcessID", "ThreadID", ">>>")
sock.send(b"print('BEFORE FORK')\n")
wait_for_strings(client.read, TIMEOUT, "BEFORE FORK")
time.sleep(2)
sock.send(b"print('AFTER FORK')\n")
wait_for_strings(client.read, TIMEOUT, "AFTER FORK")
def test_sigmask():
with TestProcess(sys.executable, HELPER, 'test_sigmask') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = connect_to_manhole(SOCKET_PATH)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
client.reset()
# Python 2.7 returns [10L], Python 3 returns [10]
sock.send(b"from __future__ import print_function\n"
b"import signalfd\n"
b"mask = signalfd.sigprocmask(signalfd.SIG_BLOCK, [])\n"
b"print([int(n) for n in mask])\n")
wait_for_strings(client.read, 1, '%s' % [signal.SIGUSR1])
def test_interrupt_on_accept():
with TestProcess(sys.executable, '-u', HELPER, 'test_interrupt_on_accept') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection', 'Sending signal to manhole thread', 'Waiting for new connection')
assert_manhole_running(proc, uds_path)
def check_print_tracebacks(uds_path):
sock = connect_to_manhole(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
sock.send(b"NO_SUCH_NAME\n")
wait_for_strings(client.read, 1, "NameError:", "name 'NO_SUCH_NAME' is not defined", ">>>")
def test_oneshot_on_usr2_error():
with TestProcess(sys.executable, '-u', HELPER,
'test_oneshot_on_usr2') as proc:
with dump_on_error(proc.read):
wait_for_strings(
proc.read, TIMEOUT,
'Not patching os.fork and os.forkpty. Oneshot activation is done by signal'
)
pytest.raises(AssertionError, wait_for_strings, proc.read, TIMEOUT,
'/tmp/manhole-')
proc.signal(signal.SIGUSR2)
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
assert_manhole_running(
proc,
uds_path,
oneshot=True,
extra=lambda client: client.sock.send(b"raise SystemExit()\n"))
proc.reset()
proc.signal(signal.SIGUSR2)
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
assert_manhole_running(proc, uds_path, oneshot=True)
def test_uwsgi():
with TestProcess(
'uwsgi',
'--master',
'--processes', '1',
'--no-orphans',
'--log-5xx',
'--single-interpreter',
'--shared-socket', ':0',
'--no-default-app',
'--manage-script-name',
'--http', '=0',
'--mount', '=wsgi:application',
*['--virtualenv', os.environ['VIRTUAL_ENV']] if 'VIRTUAL_ENV' in os.environ else []
) as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'uWSGI http bound')
port = re.findall(r"uWSGI http bound on :(\d+) fd", proc.read())[0]
assert requests.get('http://127.0.0.1:%s/' % port).text == 'OK'
wait_for_strings(proc.read, TIMEOUT, 'spawned uWSGI worker 1')
pid = re.findall(r"spawned uWSGI worker 1 \(pid: (\d+), ", proc.read())[0]
for _ in range(2):
with open('/tmp/manhole-pid', 'w') as fh:
fh.write(pid)
assert_manhole_running(proc, '/tmp/manhole-%s' % pid, oneshot=True)
def test_queue_collapse():
with TestProcess(sys.executable, helper.__file__,
'queue_collapse') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Queues =>')
clients = []
for _ in range(5):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(2)
sock.connect(UDS_PATH)
if PY3:
fh = sock.makefile("rwb", buffering=0)
else:
fh = sock.makefile(bufsize=0)
fh.write(b"queue_collapse\n")
clients.append((fh, sock))
try:
t1 = time.time()
for fh, _ in clients:
fh.readline()
delta = time.time() - t1
if delta > TIMEOUT:
raise AssertionError(
'Jobs took too much time (%0.2f sec)' % delta)
wait_for_strings(
proc.read, TIMEOUT, 'queue_collapse OK',
'%s:%s' % (pwd.getpwuid(os.getuid())[0], os.getpid()))
finally:
[(fh.close(), sock.close()) for fh, sock in clients]
def test_queue_collapse():
with TestProcess(sys.executable, helper.__file__, 'queue_collapse') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Queues =>')
clients = []
for _ in range(5):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(2)
sock.connect(UDS_PATH)
if PY3:
fh = sock.makefile("rwb", buffering=0)
else:
fh = sock.makefile(bufsize=0)
fh.write(b"queue_collapse\n")
clients.append((fh, sock))
try:
t1 = time.time()
for fh, _ in clients:
fh.readline()
delta = time.time() - t1
if delta > TIMEOUT:
raise AssertionError('Jobs took too much time (%0.2f sec)' % delta)
wait_for_strings(proc.read, TIMEOUT,
'queue_collapse OK',
'%s:%s' % (pwd.getpwuid(os.getuid())[0], os.getpid()))
finally:
[(fh.close(), sock.close()) for fh, sock in clients]
def assert_manhole_running(proc, uds_path, oneshot=False, extra=None):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(0.5)
for i in range(TIMEOUT):
try:
sock.connect(uds_path)
break
except Exception as exc:
print('Failed to connect to %s: %s' % (uds_path, exc))
time.sleep(1)
if i + 1 == TIMEOUT:
raise
try:
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, "ProcessID", "ThreadID", ">>>")
sock.send(b"print('FOOBAR')\n")
wait_for_strings(client.read, TIMEOUT, "FOOBAR")
wait_for_strings(proc.read, TIMEOUT, 'UID:%s' % os.getuid())
if extra:
extra(sock)
sock.shutdown(socket.SHUT_RDWR)
finally:
sock.close()
wait_for_strings(proc.read, TIMEOUT, 'Cleaned up.', *[] if oneshot else ['Waiting for new connection'])
def assert_manhole_running(proc, uds_path, oneshot=False, extra=None):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.settimeout(0.5)
for i in range(TIMEOUT):
try:
sock.connect(uds_path)
break
except Exception as exc:
print('Failed to connect to %s: %s' % (uds_path, exc))
time.sleep(1)
if i + 1 == TIMEOUT:
raise
try:
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, "ProcessID", "ThreadID",
">>>")
sock.send(b"print('FOOBAR')\n")
wait_for_strings(client.read, TIMEOUT, "FOOBAR")
wait_for_strings(proc.read, TIMEOUT, 'UID:%s' % os.getuid())
if extra:
extra(sock)
sock.shutdown(socket.SHUT_RDWR)
finally:
sock.close()
wait_for_strings(proc.read, TIMEOUT, 'Cleaned up.',
*[] if oneshot else ['Waiting for new connection'])
def test_sample():
with TestProcess('coverage', 'run', 'tests/nosetests.py', '--verbose',
'--with-html', '--html-file=sample.html',
'tests/test_sample.py') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Ran 9 tests in')
output = open('sample.html').read()
assert """<tr>
<td>test_sample</td>
<td class="failed">1</td>
<td class="failed">1</td>
<td>1</td>
<td>1</td>
<td>4</td>
</tr>""" in output
assert """<tr>
<td>test_sample.MainTestCase</td>
<td class="failed">1</td>
<td>0</td>
<td>0</td>
<td>1</td>
<td>2</td>
</tr>""" in output
assert """<tr>
<td>test_sample.FailedSetupTestCase</td>
<td>0</td>
<td class="failed">1</td>
<td>0</td>
<td>0</td>
<td>1</td>
</tr>""" in output
assert """<tr>
<td>test_sample.SecondTestCase</td>
<td>0</td>
<td>0</td>
<td>0</td>
<td>2</td>
<td>2</td>
</tr>""" in output
assert """<tr>
<td><strong>Total</strong></td>
<td class="failed">2</td>
<td class="failed">2</td>
<td>1</td>
<td>4</td>
<td>9</td>
</tr>""" in output
assert "<h2>test_sample.MainTestCase (1 failures, 0 errors)</h2>" in output
assert '<section id="test_sample.MainTestCase:test_b">' in output
assert '<h3>test_b: <strong>' in output
assert '<section id="test_sample:test_b">' in output
assert '<h3>test_b: <strong>' in output
assert '<li><a class="success">test_a</a></li>' in output
assert '<li><a class="failed" href="#test_sample.MainTestCase:test_b">test_b</a></li>' in output
assert '<h2>test_sample (1 failures, 1 errors)</h2>' in output
assert '<li><a class="success">test_a</a></li>' in output
assert '<li><a class="failed" href="#test_sample:test_b">test_b</a></li>' in output
assert "<h2>test_sample.FailedSetupTestCase (0 failures, 1 errors)</h2>" in output
def test_gdb():
with TestProcess('python', '-mtarget',
'manhole') as target, dump_on_error(target.read):
with TestProcess('hunter-trace', '-p', str(target.proc.pid), '--gdb',
'stdlib=False') as tracer, dump_on_error(tracer.read):
wait_for_strings(
tracer.read,
TIMEOUT,
'WARNING: Using GDB may deadlock the process or create unpredictable results!',
'Output stream active. Starting tracer',
'call => stuff()',
'line time.sleep(1)',
'return <= stuff: None',
)
wait_for_strings(target.read, TIMEOUT, 'Broken pipe',
'Stopping tracer.')
def test_no_overlap(redis_server):
"""
This test tries to simulate contention: lots of clients trying to acquire at the same time.
If there would be a bug that would allow two clients to hold the lock at the same time it
would most likely regress this test.
The code here mostly tries to parse out the pid of the process and the time when it got and
released the lock. If there's is overlap (eg: pid1.start < pid2.start < pid1.end) then we
got a very bad regression on our hands ...
The subprocess being run (check helper.py) will fork bunch of processes and will try to
syncronize them (using the builting sched) to try to acquire the lock at the same time.
"""
with TestProcess(sys.executable, HELPER, 'test_no_overlap') as proc:
with dump_on_error(proc.read):
name = 'lock:foobar'
wait_for_strings(proc.read, 10 * TIMEOUT, 'Getting %r ...' % name)
wait_for_strings(proc.read, 10 * TIMEOUT,
'Got lock for %r.' % name)
wait_for_strings(proc.read, 10 * TIMEOUT, 'Releasing %r.' % name)
wait_for_strings(proc.read, 10 * TIMEOUT,
'UNLOCK_SCRIPT not cached.')
wait_for_strings(proc.read, 10 * TIMEOUT, 'DIED.')
class Event(object):
pid = start = end = '?'
def __str__(self):
return "Event(%s; %r => %r)" % (self.pid, self.start,
self.end)
events = defaultdict(Event)
for line in proc.read().splitlines():
try:
pid, time, junk = line.split(' ', 2)
pid = int(pid)
except ValueError:
continue
if 'Got lock for' in junk:
events[pid].pid = pid
events[pid].start = time
if 'Releasing' in junk:
events[pid].pid = pid
events[pid].end = time
assert len(events) == 125
# not very smart but we don't have millions of events so it's
# ok - compare all the events with all the other events:
for event in events.values():
for other in events.values():
if other is not event:
try:
if (other.start < event.start < other.end
or other.start < event.end < other.end):
pytest.fail('%s overlaps %s' % (event, other))
except Exception:
print("[%s/%s]" % (event, other))
raise
def check_locals(uds_path):
sock = connect_to_manhole(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
sock.send(b"from __future__ import print_function\n"
b"print(k1, k2)\n")
wait_for_strings(client.read, 1, "v1 v2")
def test_no_block(conn):
with Lock(conn, "foobar"):
with TestProcess(sys.executable, HELPER, "test_no_block") as proc:
with dump_on_error(proc.read):
name = "lock:foobar"
wait_for_strings(
proc.read, TIMEOUT, "Getting %r ..." % name, "Failed to get %r." % name, "acquire=>False", "DIED."
)
def check_locals(uds_path):
sock = connect_to_manhole(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, ">>>")
sock.send(b"from __future__ import print_function\n"
b"print(k1, k2)\n")
wait_for_strings(client.read, TIMEOUT, "v1 v2")
def test_activate_on_with_oneshot_on():
with TestProcess(sys.executable, '-u', __file__, 'daemon',
'test_activate_on_with_oneshot_on') as proc:
with dump_on_error(proc.read):
wait_for_strings(
proc.read, TIMEOUT,
"RuntimeError('You cannot do activation of the Manhole thread on the same signal that you want to do oneshot activation !')"
)
def test_manhole():
with process_tests.TestProcess(sys.executable, '-mtarget', 'manhole') as target, \
process_tests.dump_on_error(target.read):
process_tests.wait_for_strings(target.read, TIMEOUT, 'Oneshot activation is done by signal')
with process_tests.TestProcess('hunter-trace', '-p', str(target.proc.pid), 'stdlib=False') as tracer,\
process_tests.dump_on_error(tracer.read):
process_tests.wait_for_strings(
tracer.read, TIMEOUT,
'Output stream active. Starting tracer',
'call => stuff()',
'line time.sleep(1)',
'return <= stuff: None',
)
process_tests.wait_for_strings(target.read, TIMEOUT, 'Broken pipe. Stopping tracer.')
def test_signalfd_weirdness_negative():
with TestProcess(sys.executable, '-u', HELPER, 'test_signalfd_weirdness_negative') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
wait_for_strings(proc.read, TIMEOUT, 'reading from signalfd failed')
assert_manhole_running(proc, uds_path)
def test_custom_exit_code():
with TestProcess(sys.executable, helper.__file__, 'custom_exit_code') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Queues =>')
with connection(3) as fh:
fh.write(b"asdf\n")
line = fh.readline()
json.loads(line.decode('ascii'))["exit_code"] == 123
def test_sigmask():
with TestProcess(sys.executable, HELPER, 'test_sigmask') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = connect_to_manhole(SOCKET_PATH)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
client.reset()
# Python 2.7 returns [10L], Python 3 returns [10]
sock.send(
b"from __future__ import print_function\n"
b"import signalfd\n"
b"mask = signalfd.sigprocmask(signalfd.SIG_BLOCK, [])\n"
b"print([int(n) for n in mask])\n")
wait_for_strings(client.read, 1,
'%s' % [int(signal.SIGUSR1)])
def check_print_tracebacks(uds_path):
sock = connect_to_manhole(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
sock.send(b"NO_SUCH_NAME\n")
wait_for_strings(client.read, 1, "NameError:",
"name 'NO_SUCH_NAME' is not defined", ">>>")
def test_activate_on_with_oneshot_on():
with TestProcess(sys.executable, '-u', HELPER,
'test_activate_on_with_oneshot_on') as proc:
with dump_on_error(proc.read):
wait_for_strings(
proc.read, TIMEOUT,
"You cannot do activation of the Manhole thread on the same signal that you want to do "
"oneshot activation !")
def test_no_overlap(redis_server):
"""
This test tries to simulate contention: lots of clients trying to acquire at the same time.
If there would be a bug that would allow two clients to hold the lock at the same time it
would most likely regress this test.
The code here mostly tries to parse out the pid of the process and the time when it got and
released the lock. If there's is overlap (eg: pid1.start < pid2.start < pid1.end) then we
got a very bad regression on our hands ...
The subprocess being run (check helper.py) will fork bunch of processes and will try to
syncronize them (using the builting sched) to try to acquire the lock at the same time.
"""
with TestProcess(sys.executable, HELPER, 'test_no_overlap') as proc:
with dump_on_error(proc.read):
name = 'lock:foobar'
wait_for_strings(proc.read, 10 * TIMEOUT, 'Getting %r ...' % name)
wait_for_strings(proc.read, 10 * TIMEOUT, 'Got lock for %r.' % name)
wait_for_strings(proc.read, 10 * TIMEOUT, 'Releasing %r.' % name)
wait_for_strings(proc.read, 10 * TIMEOUT, 'UNLOCK_SCRIPT not cached.')
wait_for_strings(proc.read, 10 * TIMEOUT, 'DIED.')
class Event(object):
pid = start = end = '?'
def __str__(self):
return "Event(%s; %r => %r)" % (self.pid, self.start, self.end)
events = defaultdict(Event)
for line in proc.read().splitlines():
try:
pid, time, junk = line.split(' ', 2)
pid = int(pid)
except ValueError:
continue
if 'Got lock for' in junk:
events[pid].pid = pid
events[pid].start = time
if 'Releasing' in junk:
events[pid].pid = pid
events[pid].end = time
assert len(events) == 125
# not very smart but we don't have millions of events so it's
# ok - compare all the events with all the other events:
for event in events.values():
for other in events.values():
if other is not event:
try:
if (
other.start < event.start < other.end or
other.start < event.end < other.end
):
pytest.fail('%s overlaps %s' % (event, other))
except Exception:
print("[%s/%s]" % (event, other))
raise
def test_install_twice_not_strict():
with TestProcess(sys.executable, HELPER, 'test_install_twice_not_strict') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Not patching os.fork and os.forkpty. Oneshot activation is done by signal')
proc.signal(signal.SIGUSR2)
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
assert_manhole_running(proc, uds_path, oneshot=True)
def test_sample():
with TestProcess(
'coverage', 'run', 'tests/nosetests.py', '--verbose', '--with-html', '--html-file=sample.html',
'tests/test_sample.py'
) as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Ran 9 tests in')
output = open('sample.html').read()
assert """<tr>
<td>test_sample</td>
<td>1</td>
<td>1</td>
<td>1</td>
<td>4</td>
</tr>""" in output
assert """<tr>
<td>test_sample.MainTestCase</td>
<td>1</td>
<td>0</td>
<td>1</td>
<td>2</td>
</tr>""" in output
assert """<tr>
<td>test_sample.FailedSetupTestCase</td>
<td>0</td>
<td>0</td>
<td>0</td>
<td>1</td>
</tr>""" in output
assert """<tr>
<td>test_sample.SecondTestCase</td>
<td>0</td>
<td>0</td>
<td>2</td>
<td>2</td>
</tr>""" in output
assert """<tr>
<td><strong>Total</strong></td>
<td>2</td>
<td>1</td>
<td>4</td>
<td>9</td>
</tr>""" in output
assert "<h2>test_sample.MainTestCase (1 failures, 0 errors)</h2>" in output
assert '<section id="test_sample.MainTestCase:test_b">' in output
assert '<h3>test_b: <strong>' in output
assert '<section id="test_sample:test_b">' in output
assert '<h3>test_b: <strong>' in output
assert '<li><a class="success">test_a</a></li>' in output
assert '<li><a class="failed" href="#test_sample.MainTestCase:test_b">test_b</a></li>' in output
assert '<h2>test_sample (1 failures, 1 errors)</h2>' in output
assert '<li><a class="success">test_a</a></li>' in output
assert '<li><a class="failed" href="#test_sample:test_b">test_b</a></li>' in output
assert "<h2>test_sample.FailedSetupTestCase (0 failures, 1 errors)</h2>" in output
def test_locals_after_fork():
with TestProcess(sys.executable, HELPER, 'test_locals_after_fork') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Fork detected')
proc.reset()
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
child_uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
check_locals(child_uds_path)
def test_connection_handler_exec(variant):
with TestProcess(sys.executable, HELPER, 'test_connection_handler_exec_' + variant) as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
for _ in range(200):
proc.reset()
sock = connect_to_manhole(uds_path)
wait_for_strings(proc.read, TIMEOUT, 'UID:%s' % os.getuid(), )
with TestSocket(sock) as client:
with dump_on_error(client.read):
sock.send(b"print('FOOBAR')\n")
wait_for_strings(proc.read, TIMEOUT, 'FOOBAR')
sock.send(b"tete()\n")
wait_for_strings(proc.read, TIMEOUT, 'TETE')
sock.send(b"exit()\n")
wait_for_strings(proc.read, TIMEOUT, 'Exiting exec loop.')
def test_custom_exit_code():
with TestProcess(sys.executable, helper.__file__,
'custom_exit_code') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Queues =>')
with connection(3) as fh:
fh.write(b"asdf\n")
line = fh.readline()
json.loads(line.decode('ascii'))["exit_code"] == 123
def test_manhole():
with TestProcess('python',
'-msamplemanhole') as target, dump_on_error(target.read):
wait_for_strings(target.read, TIMEOUT,
'Oneshot activation is done by signal')
with TestProcess('hunter-trace', '-p', str(target.proc.pid),
'stdlib=False') as tracer, dump_on_error(tracer.read):
wait_for_strings(
tracer.read,
TIMEOUT,
'Output stream active. Starting tracer',
'call => stuff()',
'line time.sleep(1)',
'return <= stuff: None',
)
wait_for_strings(target.read, TIMEOUT, 'Broken pipe',
'Stopping tracer.')
def test_environ_variable_activation():
with TestProcess(sys.executable, '-u', HELPER, 'test_environ_variable_activation', env=dict(os.environ, PYTHONMANHOLE="oneshot_on='USR2'")) as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Not patching os.fork and os.forkpty. Oneshot activation is done by signal')
proc.signal(signal.SIGUSR2)
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
assert_manhole_running(proc, uds_path, oneshot=True)
def test_simple():
with TestProcess(sys.executable, HELPER, 'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
for _ in range(20):
proc.reset()
assert_manhole_running(proc, uds_path)
def test_cover_looponfail(testdir, monkeypatch):
testdir.makepyfile(mod=MODULE)
testdir.makeconftest(CONFTEST)
script = testdir.makepyfile(BASIC_TEST)
monkeypatch.setattr(testdir, "run", lambda *args: TestProcess(*map(str, args)))
with testdir.runpytest("-v", "--cov=%s" % script.dirpath(), "--looponfail", script) as process:
with dump_on_error(process.read):
wait_for_strings(process.read, 30, "Stmts Miss Cover") # 30 seconds
def test_locmem():
with TestProcess('django-admin.py',
'runserver', '127.0.0.1:0', '--traceback', '--noreload', '--nothreading',
env=dict(os.environ, UWSGI_CACHE_FALLBACK='y')) as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '127.0.0.1')
port = get_ports(proc.proc.pid)
url = "http://127.0.0.1:%s" % port
assertions(url)
def test_locals_after_fork():
with TestProcess(sys.executable, HELPER, 'test_locals_after_fork') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Fork detected')
proc.reset()
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
child_uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
check_locals(child_uds_path)
def redis_server(scope='module'):
try:
os.unlink(UDS_PATH)
except OSError:
pass
with TestProcess('redis-server', '--port', '0', '--unixsocket', UDS_PATH) as process:
with dump_on_error(process.read):
wait_for_strings(process.read, TIMEOUT, "Running")
yield process
def test_simple():
with TestProcess(sys.executable, HELPER, 'test_simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
for _ in range(20):
proc.reset()
assert_manhole_running(proc, uds_path)
def redis_server(scope='module'):
try:
os.unlink(UDS_PATH)
except OSError:
pass
with TestProcess('redis-server', '--port', '0', '--unixsocket', UDS_PATH) as process:
with dump_on_error(process.read):
wait_for_strings(process.read, TIMEOUT, "Running")
yield process
def test_sigprocmask_negative():
with TestProcess(sys.executable, '-u', HELPER,
'test_signalfd_weirdness_negative') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
wait_for_strings(proc.read, TIMEOUT, 'signalled=True')
assert_manhole_running(proc, uds_path)
def test_socket_path_with_fork():
with TestProcess(sys.executable, '-u', HELPER,
'test_socket_path_with_fork') as proc:
with dump_on_error(proc.read):
wait_for_strings(
proc.read, TIMEOUT,
'Not patching os.fork and os.forkpty. Using user socket path')
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
sock = connect_to_manhole(SOCKET_PATH)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, TIMEOUT, "ProcessID",
"ThreadID", ">>>")
sock.send(b"print('BEFORE FORK')\n")
wait_for_strings(client.read, TIMEOUT, "BEFORE FORK")
time.sleep(2)
sock.send(b"print('AFTER FORK')\n")
wait_for_strings(client.read, TIMEOUT, "AFTER FORK")
def test_interrupt_on_accept():
with TestProcess(sys.executable, '-u', __file__, 'daemon',
'test_interrupt_on_accept') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection',
'Sending signal to manhole thread',
'Waiting for new connection')
assert_manhole_running(proc, uds_path)
def check_dump_stacktraces(uds_path):
sock = connect_to_manhole(uds_path)
with TestSocket(sock) as client:
with dump_on_error(client.read):
wait_for_strings(client.read, 1, ">>>")
sock.send(b"dump_stacktraces()\n")
# Start of dump
wait_for_strings(client.read, 1, "#########", "ThreadID=", "#########")
# End of dump
wait_for_strings(client.read, 1, "#############################################", ">>>")
def test_signalfd_weirdness():
with TestProcess(sys.executable, '-u', HELPER, 'test_signalfd_weirdness') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
wait_for_strings(proc.read, 25 * TIMEOUT, *[
'[%s] read from signalfd:' % j for j in range(200)
])
assert_manhole_running(proc, uds_path)
def test_activate_on_usr2():
with TestProcess(sys.executable, '-u', HELPER, 'test_activate_on_usr2') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Not patching os.fork and os.forkpty. Activation is done by signal')
raises(AssertionError, wait_for_strings, proc.read, TIMEOUT, '/tmp/manhole-')
proc.signal(signal.SIGUSR2)
wait_for_strings(proc.read, TIMEOUT, '/tmp/manhole-')
uds_path = re.findall(r"(/tmp/manhole-\d+)", proc.read())[0]
wait_for_strings(proc.read, TIMEOUT, 'Waiting for new connection')
assert_manhole_running(proc, uds_path)
def test_gdb_clean_exit():
with TestProcess(sys.executable, '-mtarget',
'manhole') as target, dump_on_error(target.read):
with TestProcess('hunter-trace', '-p', str(target.proc.pid),
'stdlib=False',
'--gdb') as tracer, dump_on_error(tracer.read):
wait_for_strings(
tracer.read,
TIMEOUT,
'WARNING: Using GDB may deadlock the process or create unpredictable results!',
'Output stream active. Starting tracer',
'call => stuff()',
'line time.sleep(1)',
'return <= stuff: None',
)
target.reset()
tracer.proc.send_signal(signal.SIGINT)
wait_for_strings(target.read, TIMEOUT, 'Doing stuff', 'Doing stuff',
'Doing stuff')
def test_timeout_acquired(conn):
with TestProcess(sys.executable, HELPER, 'test_timeout') as proc:
with dump_on_error(proc.read):
name = 'lock:foobar'
wait_for_strings(
proc.read, TIMEOUT,
'Getting %r ...' % name,
'Got lock for %r.' % name,
)
lock = Lock(conn, "foobar")
assert lock.acquire(timeout=2)
def test_manhole_clean_exit():
with TestProcess('python', '-mtarget',
'manhole') as target, dump_on_error(target.read):
wait_for_strings(target.read, TIMEOUT,
'Oneshot activation is done by signal')
with TestProcess('hunter-trace', '-p', str(target.proc.pid),
'stdlib=False') as tracer, dump_on_error(tracer.read):
wait_for_strings(
tracer.read,
TIMEOUT,
'Output stream active. Starting tracer',
'call => stuff()',
'line time.sleep(1)',
'return <= stuff: None',
)
target.reset()
tracer.proc.send_signal(signal.SIGINT)
wait_for_strings(target.read, TIMEOUT, 'remote.deactivate()',
'Doing stuff', 'Doing stuff', 'Doing stuff')
def test_prespawned():
with TestProcess(sys.executable, helper.__file__, 'simple') as proc:
with dump_on_error(proc.read):
wait_for_strings(proc.read, TIMEOUT, 'Queues =>')
response = client.request(helper.PATH, b"foobar")
assert response.exit_code == 0
wait_for_strings(proc.read, TIMEOUT,
'%s:%s' % (pwd.getpwuid(os.getuid())[0], os.getpid()),
'JOB foobar EXECUTED',
'completed. Passing back results to',
'Queues => 0 workspaces')
