def test_readv(self): """Test readv.""" global do_sleeps def testit(family, address, block_sends, block_receives, expected_results): s = coro.make_socket(family, socket.SOCK_STREAM) server.block_sends = block_sends coro.with_timeout(5, s.connect, (address, server.port)) blocks = coro.with_timeout(5, s.readv, block_receives) self.assertEqual(len(blocks), len(expected_results)) for block, expected_block in zip(blocks, expected_results): self.assertEqual(block, expected_block) to_test = [(socket.AF_INET, '127.0.0.1')] if coro.has_ipv6(): to_test.append((socket.AF_INET6, '::1')) else: sys.stderr.write('Warning: No IPv6 support; skipping tests\n') for family, address in to_test: server = TestServer() server_thread = coro.spawn(server.serve, family, address) # Give the server a chance to start. coro.yield_slice() # Different levels of greediness. for greediness in (1024, 1): coro.current().set_max_selfish_acts(greediness) # Do it once without sleeps, once with sleeps. for sleep in (False, True): do_sleeps = sleep testit(family, address, (5, 19, 3, 8), (5, 19, 3, 8), ('01234', '0123456789012345678', '012', '01234567')) testit(family, address, (5, 19, 3, 8), (24, 3, 8), ('012340123456789012345678', '012', '01234567')) testit(family, address, (5, 19, 3, 8), (2, 3, 19, 3, 8), ('01', '234', '0123456789012345678', '012', '01234567')) testit(family, address, (5, 5), (1, 1, 1, 1, 1, 1, 1, 1, 1, 1), ('0', '1', '2', '3', '4', '0', '1', '2', '3', '4')) testit(family, address, (1, 1, 1, 1, 1, 1, 1, 1, 1, 1), (5, 5), ('00000', '00000')) testit(family, address, (10, ), (5, ), ('01234', )) testit(family, address, (5, ), (10, ), ('01234', )) testit(family, address, (), (), ()) testit(family, address, (), (5, 2, 8), ()) testit(family, address, (5, 9), (5, 10), ('01234', '012345678')) testit(family, address, (5, 9), (5, 5, 3, 7), ('01234', '01234', '567', '8')) testit(family, address, (5, 5), (5, 5, 10), ('01234', '01234')) testit(family, address, (5, ), (6, ), ('01234', )) testit(family, address, (512 * 1024, ), (512 * 1024, ), (big_block[:512 * 1024], )) server_thread.raise_exception(coro.Shutdown)
def current_thread(): # XXX: Probably doesn't work in "main" thread. try: return _active_threads[coro.current().id] except KeyError: # Thread was probably not started by threading but by coro instead. # Try creating a wrapper. # XXX: This does not install into _active_threads because there isn't # a safe way to remove it when the thread dies. Access a global from # __del__ isn't safe. return Thread(_co=coro.current())
def test_writev(self): """Test writev.""" global send_buffer_size, recv_buffer_size big_block = '0123456789' * 1024 * 100 def testit(family, address, block_sends, expected_buffer_result, expected_return): global finished finished = coro.condition_variable() s = coro.make_socket(family, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, send_buffer_size) coro.with_timeout(5, s.connect, (address, server.port)) blocks = [big_block[:size] for size in block_sends] rc = coro.with_timeout(5, s.writev, blocks) s.close() if finished is not None: coro.with_timeout(5, finished.wait) self.assertEqual(expected_buffer_result, current_buffer) self.assertEqual(expected_return, rc) # Setting the send/recv buffer size to 1 causes writev to indicate it # was only able to send 1 byte before blocking. This allows us to test # the "partial" buffer sent code path. to_test = [(socket.AF_INET, '127.0.0.1')] if coro.has_ipv6(): to_test.append((socket.AF_INET6, '::1')) else: sys.stderr.write('Warning: No IPv6 support; skipping tests\n') for family, address in to_test: for bufsize in (32768, 1): send_buffer_size = bufsize recv_buffer_size = bufsize server = TestServer() server_thread = coro.spawn(server.serve, family, address) # Give the server a chance to start. coro.yield_slice() for greediness in (1024, 1): coro.current().set_max_selfish_acts(greediness) testit(family, address, (), '', 0) testit(family, address, (5, 3, 7, 8), '01234012012345601234567', 23) # bufsize==1 is too slow and not necessary if bufsize != 1: testit(family, address, (512 * 1024, ), big_block[:512 * 1024], 512 * 1024) server_thread.raise_exception(coro.Shutdown)
def receive_message(self, wait_till): """receive_message(self, wait_till) -> message_type, packet Read a message off the stream. <wait_till>: List of message types that you are looking for. """ if not self._receive_thread: raise SSH_Protocol_Error('receive thread not running') self.tmc.add(coro.current(), wait_till) try: return coro._yield() except: self.tmc.remove(coro.current()) raise
def receive_message(self, wait_till): """receive_message(self, wait_till) -> message_type, packet Read a message off the stream. <wait_till>: List of message types that you are looking for. """ if not self._receive_thread: raise SSH_Protocol_Error, 'receive thread not running' self.tmc.add(coro.current(), wait_till) try: return coro._yield() except: self.tmc.remove(coro.current()) raise
def test_writev(self): """Test writev.""" global send_buffer_size, recv_buffer_size big_block = '0123456789' * 1024 * 100 def testit(family, address, block_sends, expected_buffer_result, expected_return): global finished finished = coro.condition_variable() s = coro.make_socket(family, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, send_buffer_size) coro.with_timeout(5, s.connect, (address, server.port)) blocks = [ big_block[:size] for size in block_sends ] rc = coro.with_timeout(5, s.writev, blocks) s.close() if finished is not None: coro.with_timeout(5, finished.wait) self.assertEqual(expected_buffer_result, current_buffer) self.assertEqual(expected_return, rc) # Setting the send/recv buffer size to 1 causes writev to indicate it # was only able to send 1 byte before blocking. This allows us to test # the "partial" buffer sent code path. to_test = [(socket.AF_INET, '127.0.0.1')] if coro.has_ipv6(): to_test.append((socket.AF_INET6, '::1')) else: sys.stderr.write('Warning: No IPv6 support; skipping tests\n') for family, address in to_test: for bufsize in (32768, 1): send_buffer_size = bufsize recv_buffer_size = bufsize server = TestServer() server_thread = coro.spawn(server.serve, family, address) # Give the server a chance to start. coro.yield_slice() for greediness in (1024, 1): coro.current().set_max_selfish_acts(greediness) testit(family, address, (), '', 0) testit(family, address, (5, 3, 7, 8), '01234012012345601234567', 23) # bufsize==1 is too slow and not necessary if bufsize != 1: testit(family, address, (512 * 1024,), big_block[:512*1024], 512*1024) server_thread.raise_exception(coro.Shutdown)
def test_concurrent_dbm(tries): dbm = database_manager(debug=True) for i in xrange(tries): my_tid = coro.current().thread_id() db_name = "test_database_%d_%d" % (my_tid, i) dbm.create_database(db_name) dbm.drop_database(db_name) P("done %r" % db_name)
def test_multiple_sleep_interrupt(): # The first interrupt will work just fine coro.spawn(_test_multiple_sleep_interrupt_ok, coro.current()) # Since it has already been interrupted, you can't interrupt it ever again. # The following threads will fail. This is to be expected. # Someday we need to redesign how interrupts work so that these won't fail. # But that will be very tricky. coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current()) coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current()) coro.spawn(_test_multiple_sleep_interrupt_fail, coro.current()) # The other thread will interrupt me coro.print_stderr(' There should be no pause here:\n') try: coro.sleep_relative(10) except coro.Interrupted, why: if why != 'foobar': raise ValueError('Incorrect interrupt value.')
def test_readv(self): """Test readv.""" global do_sleeps def testit(family, address, block_sends, block_receives, expected_results): s = coro.make_socket(family, socket.SOCK_STREAM) server.block_sends = block_sends coro.with_timeout(5, s.connect, (address, server.port)) blocks = coro.with_timeout(5, s.readv, block_receives) self.assertEqual(len(blocks), len(expected_results)) for block, expected_block in zip(blocks, expected_results): self.assertEqual(block, expected_block) to_test = [(socket.AF_INET, '127.0.0.1')] if coro.has_ipv6(): to_test.append((socket.AF_INET6, '::1')) else: sys.stderr.write('Warning: No IPv6 support; skipping tests\n') for family, address in to_test: server = TestServer() server_thread = coro.spawn(server.serve, family, address) # Give the server a chance to start. coro.yield_slice() # Different levels of greediness. for greediness in (1024, 1): coro.current().set_max_selfish_acts(greediness) # Do it once without sleeps, once with sleeps. for sleep in (False, True): do_sleeps = sleep testit(family, address, (5, 19, 3, 8), (5, 19, 3, 8), ('01234', '0123456789012345678', '012', '01234567')) testit(family, address, (5, 19, 3, 8), (24, 3, 8), ('012340123456789012345678', '012', '01234567')) testit(family, address, (5, 19, 3, 8), (2, 3, 19, 3, 8), ('01', '234', '0123456789012345678', '012', '01234567')) testit(family, address, (5, 5), (1, 1, 1, 1, 1, 1, 1, 1, 1, 1), ('0', '1', '2', '3', '4', '0', '1', '2', '3', '4')) testit(family, address, (1, 1, 1, 1, 1, 1, 1, 1, 1, 1), (5, 5), ('00000', '00000')) testit(family, address, (10,), (5,), ('01234',)) testit(family, address, (5,), (10,), ('01234',)) testit(family, address, (), (), ()) testit(family, address, (), (5, 2, 8), ()) testit(family, address, (5, 9), (5, 10), ('01234', '012345678')) testit(family, address, (5, 9), (5, 5, 3, 7), ('01234', '01234', '567', '8')) testit(family, address, (5, 5), (5, 5, 10), ('01234', '01234')) testit(family, address, (5,), (6,), ('01234',)) testit(family, address, (512*1024,), (512*1024,), (big_block[:512*1024],)) server_thread.raise_exception(coro.Shutdown)
def test_with_timeout_with_interrupt(): coro.spawn(_test_with_timeout_with_interrupt, coro.current()) try: coro.print_stderr(' Should be no pause:\n') coro.with_timeout(1, coro.sleep_relative, 5) except coro.Interrupted, why: if why != 'foobar': raise ValueError('Interrupt value is not foobar!')
def test_interrupt(): # Resume myself. This should work without problems. coro.spawn(_test_interrupt_resume, coro.current()) # Interrupt, this should be latent coro.spawn(_test_interrupt_latent, coro.current()) # Interrupt, this should fail coro.spawn(_test_interrupt_fail, coro.current()) result = coro.yield() coro.print_stderr('resuming with result %r\n' % (result,)) if result != 'yoyo': raise ValueError('Resume with wrong value!') # Go back to sleep to catch the latent interrupt try: result = coro.yield() except coro.Interrupted, why: if why != 'foo': raise ValueError('Wrong why %s' % why)
def stop_receive_thread(self): """stop_receive_thread(self) -> None Stops the receive thread. """ # If the receive loop calls a handler that calls disconnect # then we don't want to raise an exception on ourself. if self._receive_thread and self._receive_thread is not coro.current(): self._receive_thread.raise_exception(Stop_Receiving_Exception) self._receive_thread = None
def test_raise(): # Resume myself. This should work without problems. coro.spawn(_test_raise, coro.current()) try: result = coro.yield() except ZeroDivisionError, why: if why[0] != 12345: raise ValueError('Why is wrong %s' % (why,)) pass
def test_sleep_interrupt(): coro.spawn(_test_sleep_interrupt, coro.current()) # The other thread will interrupt me coro.print_stderr('There should be no pause here:\n') try: coro.sleep_relative(10) except coro.Interrupted, why: if why != 'foobar': raise ValueError('Incorrect interrupt value.')
def send_message(self, msg): msgid = self.msgid self.msgid += 1 self.sock.send(SEQUENCE(INTEGER(msgid), msg)) try: self.pending[msgid] = me = coro.current() reply = coro.with_timeout(self.default_timeout, me._yield) return reply finally: del self.pending[msgid]
def send_message (self, msg): msgid = self.msgid self.msgid += 1 self.sock.send (SEQUENCE (INTEGER (msgid), msg)) try: self.pending[msgid] = me = coro.current() reply = coro.with_timeout (self.default_timeout, me._yield) return reply finally: del self.pending[msgid]
def run (self): self.thread_id = coro.current().thread_id() while not self.shutdown_flag: try: conn, addr = self.accept() client = self.create_connection() c = coro.spawn (client.run, conn, addr) c.set_name ('%s connection on %r' % (self.__class__.__name__, addr,)) except coro.Shutdown: break except: self.log ('error: %r' % (coro.compact_traceback(),)) coro.sleep_relative (0.25) continue self.sock.close()
def call (self, properties, payload, exchange, routing_key): """Make an RPC call. *properties*: a properties dictionary (e.g. {'content-type':'application/json', ...}) *payload*: the data/argument to the rpc function. *exchange*, *routing_key*: as with basic_publish. """ correlation_id = str (self.uuid_fun()) properties['correlation-id'] = correlation_id properties['reply-to'] = self.queue # W ('sending call to rpc server (uuid=%r)\n' % (correlation_id,)) self.channel.basic_publish (payload, exchange, routing_key, properties=properties) self.pending[correlation_id] = coro.current() # W ('waiting for reply\n') return coro._yield()
def _run (self): """Listens on the FTP port accepting connections and spawning sessions.""" self.thread_id = coro.current().thread_id() s = coro.make_socket (socket.AF_INET, socket.SOCK_STREAM) try: s.set_reuse_addr() done = 0 while not done: for x in xrange (5): try: was_eaddrinuse = 0 s.bind ((self.ip, self.port)) except OSError, why: if why[0] == errno.EACCES: coro.print_stderr( 'Access denied binding to %s:%i. Are you running as root?\n' % (self.ip, self.port)) return elif why[0] == errno.EADDRINUSE: was_eaddrinuse = 1 elif why[0] != errno.EADDRNOTAVAIL: raise else: done = 1 break coro.sleep_relative (1) else: coro.print_stderr ("cannot bind to %s:%d after 5 attempts\n" % (self.ip, self.port)) if was_eaddrinuse: qlog.write('FTPD.PORT_IN_USE', self.ip, str(self.port)) coro.sleep_relative (15) s.listen (1024) while 1: conn_list = s.accept_many() for conn, addr in conn_list: qlog.write('FTPD.CONNECTION', self.session_id, addr[0], self.ip) session = self.channel (self, conn, addr, self.session_id) self.session_id += 1 thread = coro.spawn (session.run) thread.set_name ( "%s_%d" % ( session.__class__.__name__, thread.thread_id() ) ) self.clients.append(session)
def _run (self): """Listens on the FTP port accepting connections and spawning sessions.""" self.thread_id = coro.current().thread_id() s = coro.make_socket (socket.AF_INET, socket.SOCK_STREAM) try: s.set_reuse_addr() done = 0 while not done: for x in xrange (5): try: was_eaddrinuse = 0 s.bind ((self.ip, self.port)) except OSError, why: if why[0] == errno.EACCES: coro.print_stderr('Access denied binding to %s:%i. Are you running as root?\n' % (self.ip, self.port)) return elif why[0] == errno.EADDRINUSE: was_eaddrinuse = 1 elif why[0] != errno.EADDRNOTAVAIL: raise else: done = 1 break coro.sleep_relative (1) else: coro.print_stderr ("cannot bind to %s:%d after 5 attempts\n" % (self.ip, self.port)) if was_eaddrinuse: qlog.write('FTPD.PORT_IN_USE', self.ip, str(self.port)) coro.sleep_relative (15) s.listen (1024) while 1: conn_list = s.accept_many() for conn, addr in conn_list: qlog.write('FTPD.CONNECTION', self.session_id, addr[0], self.ip) session = self.channel (self, conn, addr, self.session_id) self.session_id += 1 thread = coro.spawn (session.run) thread.set_name ( "%s_%d" % ( session.__class__.__name__, thread.thread_id() ) ) self.clients.append(session)
def run(self): self.thread_id = coro.current().thread_id() while not self.shutdown_flag: try: conn, addr = self.accept() client = self.create_connection(conn, addr) c = coro.spawn(client.run) c.set_name(('%s connection on %r' % ( self.__class__.__name__, addr, )).encode()) except coro.Shutdown: break except: LOG.exc() coro.sleep_relative(0.25) continue self.sock.close()
def request (self, path, params): me = coro.current() qid = self.qid_counter self.qid_counter += 1 if self.debug: LOG ('=>', self.conn.fd, qid, path, params) packet = encode ([KIND_REQUEST, qid, path, params]) self.push (packet) try: self.pending_requests[qid] = me result = me._yield() finally: if qid in self.pending_requests: del self.pending_requests[qid] error, result = result if error: raise RPC_Remote_Exception (*error) else: return result
def request(self, path, params): me = coro.current() qid = self.qid_counter self.qid_counter += 1 if self.debug: LOG('=>', self.conn.fd, qid, path, params) packet = encode([KIND_REQUEST, qid, path, params]) self.push(packet) try: self.pending_requests[qid] = me result = me._yield() finally: if qid in self.pending_requests: del self.pending_requests[qid] error, result = result if error: raise RPC_Remote_Exception(*error) else: return result
def _run (self, server_s): secure = self.is_secure() self.thread_id = coro.current().thread_id() while not self.shutdown_flag: try: conn, addr = server_s.accept() client = http_client() coro.spawn (client.run, conn, addr, self, self._handlers) except coro.Shutdown: # server-shutdown break except: qlog.write('COMMON.APP_FAILURE', ('%s accept handler error %s' % (self.__class__.__name__, coro.compact_traceback()))) coro.sleep_relative(0.25) continue server_s.close() return None
def _run(self, server_s): secure = self.is_secure() self.thread_id = coro.current().thread_id() while not self.shutdown_flag: try: conn, addr = server_s.accept() client = http_client() coro.spawn(client.run, conn, addr, self, self._handlers) except coro.Shutdown: # server-shutdown break except: qlog.write( 'COMMON.APP_FAILURE', ('%s accept handler error %s' % (self.__class__.__name__, coro.compact_traceback()))) coro.sleep_relative(0.25) continue server_s.close() return None
def exception_notifier(): me = coro.current() traceback = coro.compact_traceback() G.log ('exception', me.id, me.name, traceback) WY ('exception: %r %r %r\n' % (me.id, me.name, traceback))
def get_ident(): """Returns the "id" of the current thread.""" return coro.current().id
def test_resume(): # Resume myself. This should work without problems. coro.spawn(_test_resume, coro.current()) result = coro.yield() if result != 'yoyo': raise ValueError('Resume with wrong value!')
def exception_notifier(): me = coro.current() G.log ('exception', me.id, me.name, coro.compact_traceback())
def _run (self): self.thread_id = coro.current().thread_id() try: # send the greeting self.send_greeting() while not self.shutdown_flag: line, eof = self.read_line_timeout() if eof: break line = orig_line = line.lstrip() parts = line.split() if len (parts) < 1: self.command_not_understood ('') continue command = parts[0].lower() if len(parts) > 1: args = ' '.join (parts[1:]) # If command arguments include null character, python path parsing # function will complain. Remove the null characters. line = [command, args.replace('\0', '')] else: line = [command] # watch especially for 'urgent' abort commands. if command.find ('abor') != -1: # strip off telnet sync chars and the like... while command and command[0] not in string.letters: command = command[1:] fun_name = 'cmd_%s' % command if command != 'pass': qlog.write('FTPD.RECV', self.session_id, repr(orig_line)[1:-1]) else: qlog.write('FTPD.RECV', self.session_id, line[0]+' <password>') self.in_buffer = '' if not hasattr (self, fun_name): self.command_not_understood (line[0]) continue fun = getattr (self, fun_name) if (not self.authorized) and (command not in ('user', 'pass', 'help', 'quit')): self.respond ('530 Please log in with USER and PASS') elif (not self.check_command_authorization (self.user, command)): self.command_not_authorized (command) else: if hasattr (self, '_syntax_%s' % command): r = getattr (self, '_syntax_%s' % command) m = re.match (r, orig_line, re.IGNORECASE) if m is None: self.respond ('501 Syntax error in parameters or arguments') continue try: result = apply (fun, (line,)) except OSError, why: if why[0] in self.disconnect_errors: # log it & ignore qlog.write ('FTPD.DISCONNECT', self.session_id, why.strerror) break else: raise except coro.TimeoutError: qlog.write('FTPD.DISCONNECT', self.session_id, 'Remote side timed out') break except coro.Interrupted: raise except: self.server.total_exceptions.increment() qlog.write('COMMON.APP_FAILURE', tb.traceback_string() + ' fun: ' + `fun` + ' line: ' + `line`) self.respond ('451 Server Error') self.close_passive_acceptor() else: if result == 'quit': break