def test_readv(self):
"""Test readv."""
global do_sleeps
def testit(family, address, block_sends, block_receives,
expected_results):
s = coro.make_socket(family, socket.SOCK_STREAM)
server.block_sends = block_sends
coro.with_timeout(5, s.connect, (address, server.port))
blocks = coro.with_timeout(5, s.readv, block_receives)
self.assertEqual(len(blocks), len(expected_results))
for block, expected_block in zip(blocks, expected_results):
self.assertEqual(block, expected_block)
to_test = [(socket.AF_INET, '127.0.0.1')]
if coro.has_ipv6():
to_test.append((socket.AF_INET6, '::1'))
else:
sys.stderr.write('Warning: No IPv6 support; skipping tests\n')
for family, address in to_test:
server = TestServer()
server_thread = coro.spawn(server.serve, family, address)
# Give the server a chance to start.
coro.yield_slice()
# Different levels of greediness.
for greediness in (1024, 1):
coro.current().set_max_selfish_acts(greediness)
# Do it once without sleeps, once with sleeps.
for sleep in (False, True):
do_sleeps = sleep
testit(family, address, (5, 19, 3, 8), (5, 19, 3, 8),
('01234', '0123456789012345678', '012', '01234567'))
testit(family, address, (5, 19, 3, 8), (24, 3, 8),
('012340123456789012345678', '012', '01234567'))
testit(family, address, (5, 19, 3, 8), (2, 3, 19, 3, 8),
('01', '234', '0123456789012345678', '012',
'01234567'))
testit(family, address, (5, 5),
(1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
('0', '1', '2', '3', '4', '0', '1', '2', '3', '4'))
testit(family, address, (1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
(5, 5), ('00000', '00000'))
testit(family, address, (10, ), (5, ), ('01234', ))
testit(family, address, (5, ), (10, ), ('01234', ))
testit(family, address, (), (), ())
testit(family, address, (), (5, 2, 8), ())
testit(family, address, (5, 9), (5, 10),
('01234', '012345678'))
testit(family, address, (5, 9), (5, 5, 3, 7),
('01234', '01234', '567', '8'))
testit(family, address, (5, 5), (5, 5, 10),
('01234', '01234'))
testit(family, address, (5, ), (6, ), ('01234', ))
testit(family, address, (512 * 1024, ), (512 * 1024, ),
(big_block[:512 * 1024], ))
server_thread.raise_exception(coro.Shutdown)
def current_thread():
# XXX: Probably doesn't work in "main" thread.
try:
return _active_threads[coro.current().id]
except KeyError:
# Thread was probably not started by threading but by coro instead.
# Try creating a wrapper.
# XXX: This does not install into _active_threads because there isn't
# a safe way to remove it when the thread dies. Access a global from
# __del__ isn't safe.
return Thread(_co=coro.current())
def current_thread():
# XXX: Probably doesn't work in "main" thread.
try:
return _active_threads[coro.current().id]
except KeyError:
# Thread was probably not started by threading but by coro instead.
# Try creating a wrapper.
# XXX: This does not install into _active_threads because there isn't
# a safe way to remove it when the thread dies. Access a global from
# __del__ isn't safe.
return Thread(_co=coro.current())
def test_writev(self):
"""Test writev."""
global send_buffer_size, recv_buffer_size
big_block = '0123456789' * 1024 * 100
def testit(family, address, block_sends, expected_buffer_result,
expected_return):
global finished
finished = coro.condition_variable()
s = coro.make_socket(family, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, send_buffer_size)
coro.with_timeout(5, s.connect, (address, server.port))
blocks = [big_block[:size] for size in block_sends]
rc = coro.with_timeout(5, s.writev, blocks)
s.close()
if finished is not None:
coro.with_timeout(5, finished.wait)
self.assertEqual(expected_buffer_result, current_buffer)
self.assertEqual(expected_return, rc)
# Setting the send/recv buffer size to 1 causes writev to indicate it
# was only able to send 1 byte before blocking. This allows us to test
# the "partial" buffer sent code path.
to_test = [(socket.AF_INET, '127.0.0.1')]
if coro.has_ipv6():
to_test.append((socket.AF_INET6, '::1'))
else:
sys.stderr.write('Warning: No IPv6 support; skipping tests\n')
for family, address in to_test:
for bufsize in (32768, 1):
send_buffer_size = bufsize
recv_buffer_size = bufsize
server = TestServer()
server_thread = coro.spawn(server.serve, family, address)
# Give the server a chance to start.
coro.yield_slice()
for greediness in (1024, 1):
coro.current().set_max_selfish_acts(greediness)
testit(family, address, (), '', 0)
testit(family, address, (5, 3, 7, 8),
'01234012012345601234567', 23)
# bufsize==1 is too slow and not necessary
if bufsize != 1:
testit(family, address, (512 * 1024, ),
big_block[:512 * 1024], 512 * 1024)
server_thread.raise_exception(coro.Shutdown)
def receive_message(self, wait_till):
"""receive_message(self, wait_till) -> message_type, packet
Read a message off the stream.
<wait_till>: List of message types that you are looking for.
"""
if not self._receive_thread:
raise SSH_Protocol_Error('receive thread not running')
self.tmc.add(coro.current(), wait_till)
try:
return coro._yield()
except:
self.tmc.remove(coro.current())
raise
def receive_message(self, wait_till):
"""receive_message(self, wait_till) -> message_type, packet
Read a message off the stream.
<wait_till>: List of message types that you are looking for.
"""
if not self._receive_thread:
raise SSH_Protocol_Error, 'receive thread not running'
self.tmc.add(coro.current(), wait_till)
try:
return coro._yield()
except:
self.tmc.remove(coro.current())
raise
def test_writev(self):
"""Test writev."""
global send_buffer_size, recv_buffer_size
big_block = '0123456789' * 1024 * 100
def testit(family, address, block_sends, expected_buffer_result, expected_return):
global finished
finished = coro.condition_variable()
s = coro.make_socket(family, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, send_buffer_size)
coro.with_timeout(5, s.connect, (address, server.port))
blocks = [ big_block[:size] for size in block_sends ]
rc = coro.with_timeout(5, s.writev, blocks)
s.close()
if finished is not None:
coro.with_timeout(5, finished.wait)
self.assertEqual(expected_buffer_result, current_buffer)
self.assertEqual(expected_return, rc)
# Setting the send/recv buffer size to 1 causes writev to indicate it
# was only able to send 1 byte before blocking. This allows us to test
# the "partial" buffer sent code path.
to_test = [(socket.AF_INET, '127.0.0.1')]
if coro.has_ipv6():
to_test.append((socket.AF_INET6, '::1'))
else:
sys.stderr.write('Warning: No IPv6 support; skipping tests\n')
for family, address in to_test:
for bufsize in (32768, 1):
send_buffer_size = bufsize
recv_buffer_size = bufsize
server = TestServer()
server_thread = coro.spawn(server.serve, family, address)
# Give the server a chance to start.
coro.yield_slice()
for greediness in (1024, 1):
coro.current().set_max_selfish_acts(greediness)
testit(family, address, (), '', 0)
testit(family, address, (5, 3, 7, 8), '01234012012345601234567', 23)
# bufsize==1 is too slow and not necessary
if bufsize != 1:
testit(family, address, (512 * 1024,), big_block[:512*1024], 512*1024)
server_thread.raise_exception(coro.Shutdown)
def test_concurrent_dbm(tries):
dbm = database_manager(debug=True)
for i in xrange(tries):
my_tid = coro.current().thread_id()
db_name = "test_database_%d_%d" % (my_tid, i)
dbm.create_database(db_name)
dbm.drop_database(db_name)
P("done %r" % db_name)
def test_multiple_sleep_interrupt():
# The first interrupt will work just fine
coro.spawn(_test_multiple_sleep_interrupt_ok, coro.current())
# Since it has already been interrupted, you can't interrupt it ever again.
# The following threads will fail. This is to be expected.
# Someday we need to redesign how interrupts work so that these won't fail.
# But that will be very tricky.
coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current())
coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current())
coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current())
# The other thread will interrupt me
coro.print_stderr(' There should be no pause here:\n')
try:
coro.sleep_relative(10)
except coro.Interrupted, why:
if why != 'foobar':
raise ValueError('Incorrect interrupt value.')
def test_readv(self):
"""Test readv."""
global do_sleeps
def testit(family, address, block_sends, block_receives, expected_results):
s = coro.make_socket(family, socket.SOCK_STREAM)
server.block_sends = block_sends
coro.with_timeout(5, s.connect, (address, server.port))
blocks = coro.with_timeout(5, s.readv, block_receives)
self.assertEqual(len(blocks), len(expected_results))
for block, expected_block in zip(blocks, expected_results):
self.assertEqual(block, expected_block)
to_test = [(socket.AF_INET, '127.0.0.1')]
if coro.has_ipv6():
to_test.append((socket.AF_INET6, '::1'))
else:
sys.stderr.write('Warning: No IPv6 support; skipping tests\n')
for family, address in to_test:
server = TestServer()
server_thread = coro.spawn(server.serve, family, address)
# Give the server a chance to start.
coro.yield_slice()
# Different levels of greediness.
for greediness in (1024, 1):
coro.current().set_max_selfish_acts(greediness)
# Do it once without sleeps, once with sleeps.
for sleep in (False, True):
do_sleeps = sleep
testit(family, address, (5, 19, 3, 8), (5, 19, 3, 8), ('01234', '0123456789012345678', '012', '01234567'))
testit(family, address, (5, 19, 3, 8), (24, 3, 8), ('012340123456789012345678', '012', '01234567'))
testit(family, address, (5, 19, 3, 8), (2, 3, 19, 3, 8), ('01', '234', '0123456789012345678', '012', '01234567'))
testit(family, address, (5, 5), (1, 1, 1, 1, 1, 1, 1, 1, 1, 1), ('0', '1', '2', '3', '4', '0', '1', '2', '3', '4'))
testit(family, address, (1, 1, 1, 1, 1, 1, 1, 1, 1, 1), (5, 5), ('00000', '00000'))
testit(family, address, (10,), (5,), ('01234',))
testit(family, address, (5,), (10,), ('01234',))
testit(family, address, (), (), ())
testit(family, address, (), (5, 2, 8), ())
testit(family, address, (5, 9), (5, 10), ('01234', '012345678'))
testit(family, address, (5, 9), (5, 5, 3, 7), ('01234', '01234', '567', '8'))
testit(family, address, (5, 5), (5, 5, 10), ('01234', '01234'))
testit(family, address, (5,), (6,), ('01234',))
testit(family, address, (512*1024,), (512*1024,), (big_block[:512*1024],))
server_thread.raise_exception(coro.Shutdown)
def test_concurrent_dbm(tries):
dbm = database_manager(debug=True)
for i in xrange(tries):
my_tid = coro.current().thread_id()
db_name = "test_database_%d_%d" % (my_tid, i)
dbm.create_database(db_name)
dbm.drop_database(db_name)
P("done %r" % db_name)
def test_with_timeout_with_interrupt():
coro.spawn(_test_with_timeout_with_interrupt, coro.current())
try:
coro.print_stderr(' Should be no pause:\n')
coro.with_timeout(1, coro.sleep_relative, 5)
except coro.Interrupted, why:
if why != 'foobar':
raise ValueError('Interrupt value is not foobar!')
def test_interrupt():
# Resume myself. This should work without problems.
coro.spawn(_test_interrupt_resume, coro.current())
# Interrupt, this should be latent
coro.spawn(_test_interrupt_latent, coro.current())
# Interrupt, this should fail
coro.spawn(_test_interrupt_fail, coro.current())
result = coro.yield()
coro.print_stderr('resuming with result %r\n' % (result,))
if result != 'yoyo':
raise ValueError('Resume with wrong value!')
# Go back to sleep to catch the latent interrupt
try:
result = coro.yield()
except coro.Interrupted, why:
if why != 'foo':
raise ValueError('Wrong why %s' % why)
def stop_receive_thread(self):
"""stop_receive_thread(self) -> None
Stops the receive thread.
"""
# If the receive loop calls a handler that calls disconnect
# then we don't want to raise an exception on ourself.
if self._receive_thread and self._receive_thread is not coro.current():
self._receive_thread.raise_exception(Stop_Receiving_Exception)
self._receive_thread = None
def stop_receive_thread(self):
"""stop_receive_thread(self) -> None
Stops the receive thread.
"""
# If the receive loop calls a handler that calls disconnect
# then we don't want to raise an exception on ourself.
if self._receive_thread and self._receive_thread is not coro.current():
self._receive_thread.raise_exception(Stop_Receiving_Exception)
self._receive_thread = None
def test_raise():
# Resume myself. This should work without problems.
coro.spawn(_test_raise, coro.current())
try:
result = coro.yield()
except ZeroDivisionError, why:
if why[0] != 12345:
raise ValueError('Why is wrong %s' % (why,))
pass
def test_sleep_interrupt():
coro.spawn(_test_sleep_interrupt, coro.current())
# The other thread will interrupt me
coro.print_stderr('There should be no pause here:\n')
try:
coro.sleep_relative(10)
except coro.Interrupted, why:
if why != 'foobar':
raise ValueError('Incorrect interrupt value.')
def send_message(self, msg):
msgid = self.msgid
self.msgid += 1
self.sock.send(SEQUENCE(INTEGER(msgid), msg))
try:
self.pending[msgid] = me = coro.current()
reply = coro.with_timeout(self.default_timeout, me._yield)
return reply
finally:
del self.pending[msgid]
def send_message (self, msg):
msgid = self.msgid
self.msgid += 1
self.sock.send (SEQUENCE (INTEGER (msgid), msg))
try:
self.pending[msgid] = me = coro.current()
reply = coro.with_timeout (self.default_timeout, me._yield)
return reply
finally:
del self.pending[msgid]
def run (self):
self.thread_id = coro.current().thread_id()
while not self.shutdown_flag:
try:
conn, addr = self.accept()
client = self.create_connection()
c = coro.spawn (client.run, conn, addr)
c.set_name ('%s connection on %r' % (self.__class__.__name__, addr,))
except coro.Shutdown:
break
except:
self.log ('error: %r' % (coro.compact_traceback(),))
coro.sleep_relative (0.25)
continue
self.sock.close()
def call (self, properties, payload, exchange, routing_key):
"""Make an RPC call.
*properties*: a properties dictionary (e.g. {'content-type':'application/json', ...})
*payload*: the data/argument to the rpc function.
*exchange*, *routing_key*: as with basic_publish.
"""
correlation_id = str (self.uuid_fun())
properties['correlation-id'] = correlation_id
properties['reply-to'] = self.queue
# W ('sending call to rpc server (uuid=%r)\n' % (correlation_id,))
self.channel.basic_publish (payload, exchange, routing_key, properties=properties)
self.pending[correlation_id] = coro.current()
# W ('waiting for reply\n')
return coro._yield()
def _run (self):
"""Listens on the FTP port accepting connections and spawning sessions."""
self.thread_id = coro.current().thread_id()
s = coro.make_socket (socket.AF_INET, socket.SOCK_STREAM)
try:
s.set_reuse_addr()
done = 0
while not done:
for x in xrange (5):
try:
was_eaddrinuse = 0
s.bind ((self.ip, self.port))
except OSError, why:
if why[0] == errno.EACCES:
coro.print_stderr(
'Access denied binding to %s:%i. Are you running as root?\n' % (self.ip, self.port))
return
elif why[0] == errno.EADDRINUSE:
was_eaddrinuse = 1
elif why[0] != errno.EADDRNOTAVAIL:
raise
else:
done = 1
break
coro.sleep_relative (1)
else:
coro.print_stderr ("cannot bind to %s:%d after 5 attempts\n" % (self.ip, self.port))
if was_eaddrinuse:
qlog.write('FTPD.PORT_IN_USE',
self.ip, str(self.port))
coro.sleep_relative (15)
s.listen (1024)
while 1:
conn_list = s.accept_many()
for conn, addr in conn_list:
qlog.write('FTPD.CONNECTION', self.session_id, addr[0], self.ip)
session = self.channel (self, conn, addr, self.session_id)
self.session_id += 1
thread = coro.spawn (session.run)
thread.set_name (
"%s_%d" % (
session.__class__.__name__,
thread.thread_id()
)
)
self.clients.append(session)
def _run (self):
"""Listens on the FTP port accepting connections and spawning sessions."""
self.thread_id = coro.current().thread_id()
s = coro.make_socket (socket.AF_INET, socket.SOCK_STREAM)
try:
s.set_reuse_addr()
done = 0
while not done:
for x in xrange (5):
try:
was_eaddrinuse = 0
s.bind ((self.ip, self.port))
except OSError, why:
if why[0] == errno.EACCES:
coro.print_stderr('Access denied binding to %s:%i. Are you running as root?\n' % (self.ip, self.port))
return
elif why[0] == errno.EADDRINUSE:
was_eaddrinuse = 1
elif why[0] != errno.EADDRNOTAVAIL:
raise
else:
done = 1
break
coro.sleep_relative (1)
else:
coro.print_stderr ("cannot bind to %s:%d after 5 attempts\n" % (self.ip, self.port))
if was_eaddrinuse:
qlog.write('FTPD.PORT_IN_USE',
self.ip, str(self.port))
coro.sleep_relative (15)
s.listen (1024)
while 1:
conn_list = s.accept_many()
for conn, addr in conn_list:
qlog.write('FTPD.CONNECTION', self.session_id, addr[0], self.ip)
session = self.channel (self, conn, addr, self.session_id)
self.session_id += 1
thread = coro.spawn (session.run)
thread.set_name (
"%s_%d" % (
session.__class__.__name__,
thread.thread_id()
)
)
self.clients.append(session)
def run(self):
self.thread_id = coro.current().thread_id()
while not self.shutdown_flag:
try:
conn, addr = self.accept()
client = self.create_connection(conn, addr)
c = coro.spawn(client.run)
c.set_name(('%s connection on %r' % (
self.__class__.__name__,
addr,
)).encode())
except coro.Shutdown:
break
except:
LOG.exc()
coro.sleep_relative(0.25)
continue
self.sock.close()
def request (self, path, params):
me = coro.current()
qid = self.qid_counter
self.qid_counter += 1
if self.debug:
LOG ('=>', self.conn.fd, qid, path, params)
packet = encode ([KIND_REQUEST, qid, path, params])
self.push (packet)
try:
self.pending_requests[qid] = me
result = me._yield()
finally:
if qid in self.pending_requests:
del self.pending_requests[qid]
error, result = result
if error:
raise RPC_Remote_Exception (*error)
else:
return result
def request(self, path, params):
me = coro.current()
qid = self.qid_counter
self.qid_counter += 1
if self.debug:
LOG('=>', self.conn.fd, qid, path, params)
packet = encode([KIND_REQUEST, qid, path, params])
self.push(packet)
try:
self.pending_requests[qid] = me
result = me._yield()
finally:
if qid in self.pending_requests:
del self.pending_requests[qid]
error, result = result
if error:
raise RPC_Remote_Exception(*error)
else:
return result
def _run (self, server_s):
secure = self.is_secure()
self.thread_id = coro.current().thread_id()
while not self.shutdown_flag:
try:
conn, addr = server_s.accept()
client = http_client()
coro.spawn (client.run, conn, addr, self, self._handlers)
except coro.Shutdown:
# server-shutdown
break
except:
qlog.write('COMMON.APP_FAILURE',
('%s accept handler error %s' %
(self.__class__.__name__, coro.compact_traceback())))
coro.sleep_relative(0.25)
continue
server_s.close()
return None
def _run(self, server_s):
secure = self.is_secure()
self.thread_id = coro.current().thread_id()
while not self.shutdown_flag:
try:
conn, addr = server_s.accept()
client = http_client()
coro.spawn(client.run, conn, addr, self, self._handlers)
except coro.Shutdown:
# server-shutdown
break
except:
qlog.write(
'COMMON.APP_FAILURE',
('%s accept handler error %s' %
(self.__class__.__name__, coro.compact_traceback())))
coro.sleep_relative(0.25)
continue
server_s.close()
return None
def exception_notifier():
me = coro.current()
traceback = coro.compact_traceback()
G.log ('exception', me.id, me.name, traceback)
WY ('exception: %r %r %r\n' % (me.id, me.name, traceback))
def get_ident():
"""Returns the "id" of the current thread."""
return coro.current().id
def exception_notifier():
me = coro.current()
traceback = coro.compact_traceback()
G.log ('exception', me.id, me.name, traceback)
WY ('exception: %r %r %r\n' % (me.id, me.name, traceback))
def test_resume():
# Resume myself. This should work without problems.
coro.spawn(_test_resume, coro.current())
result = coro.yield()
if result != 'yoyo':
raise ValueError('Resume with wrong value!')
def exception_notifier():
me = coro.current()
G.log ('exception', me.id, me.name, coro.compact_traceback())
def _run (self):
self.thread_id = coro.current().thread_id()
try:
# send the greeting
self.send_greeting()
while not self.shutdown_flag:
line, eof = self.read_line_timeout()
if eof:
break
line = orig_line = line.lstrip()
parts = line.split()
if len (parts) < 1:
self.command_not_understood ('')
continue
command = parts[0].lower()
if len(parts) > 1:
args = ' '.join (parts[1:])
# If command arguments include null character, python path parsing
# function will complain. Remove the null characters.
line = [command, args.replace('\0', '')]
else:
line = [command]
# watch especially for 'urgent' abort commands.
if command.find ('abor') != -1:
# strip off telnet sync chars and the like...
while command and command[0] not in string.letters:
command = command[1:]
fun_name = 'cmd_%s' % command
if command != 'pass':
qlog.write('FTPD.RECV', self.session_id, repr(orig_line)[1:-1])
else:
qlog.write('FTPD.RECV', self.session_id, line[0]+' <password>')
self.in_buffer = ''
if not hasattr (self, fun_name):
self.command_not_understood (line[0])
continue
fun = getattr (self, fun_name)
if (not self.authorized) and (command not in ('user', 'pass', 'help', 'quit')):
self.respond ('530 Please log in with USER and PASS')
elif (not self.check_command_authorization (self.user, command)):
self.command_not_authorized (command)
else:
if hasattr (self, '_syntax_%s' % command):
r = getattr (self, '_syntax_%s' % command)
m = re.match (r, orig_line, re.IGNORECASE)
if m is None:
self.respond ('501 Syntax error in parameters or arguments')
continue
try:
result = apply (fun, (line,))
except OSError, why:
if why[0] in self.disconnect_errors:
# log it & ignore
qlog.write ('FTPD.DISCONNECT', self.session_id, why.strerror)
break
else:
raise
except coro.TimeoutError:
qlog.write('FTPD.DISCONNECT', self.session_id, 'Remote side timed out')
break
except coro.Interrupted:
raise
except:
self.server.total_exceptions.increment()
qlog.write('COMMON.APP_FAILURE', tb.traceback_string() +
' fun: ' + `fun` + ' line: ' + `line`)
self.respond ('451 Server Error')
self.close_passive_acceptor()
else:
if result == 'quit':
break
def get_ident():
"""Returns the "id" of the current thread."""
return coro.current().id
