def test_blocking(self): p = Pipe(timeout=1, bsize=10) self.assertTrue(p.readblocking()) self.assertTrue(p.writeblocking()) try: # should block for 1 second and then timeout rresult = p.read(1) self.fail("blocked read returned %s" % repr(rresult)) except IOError as e: self.assertTrue(io_timedout(e)) p.write(b"123") # should not block! rresult = p.read(5) self.assertTrue(rresult == b"123") # should not block, just! p.write(bytearray(b"1234567890")) try: # should block for 1 second wresult = p.write(b"extra") self.fail("blocked write returned %s" % repr(wresult)) except IOError as e: self.assertTrue(io_timedout(e)) try: # should block for 1 second logging.debug("flush waiting for 1s timeout...") p.flush() self.fail("blocked flush returned") except IOError as e: logging.debug("flush caught 1s timeout; %s", str(e))
def test_wblocking(self): p = Pipe(timeout=15, bsize=10) t = threading.Thread(target=self.rrunner, args=(p,)) p.write(b"1234567890") data = b"extra" t.start() try: # should block until the other thread reads wresult = p.write(bytearray(data)) # and should then write at most 1 byte self.assertTrue(wresult == 1, repr(wresult)) except IOError as e: self.fail("Timeout on mutlithreaded pipe; %s" % str(e)) t = threading.Thread(target=self.rallrunner, args=(p,)) t.start() try: # should block until all data has been read logging.debug("flush waiting...") p.flush() logging.debug("flush complete") self.assertTrue(p.canwrite() == 10, "empty after flush") except IOError as e: self.fail("flush timeout on mutlithreaded pipe; %s" % str(e)) # put the other thread out of its misery p.write_eof() logging.debug("eof written, joining rallrunner") t.join()
def test_wblocking(self): p = Pipe(timeout=15, bsize=10) t = threading.Thread(target=self.rrunner, args=(p, )) p.write(b"1234567890") data = b"extra" t.start() try: # should block until the other thread reads wresult = p.write(bytearray(data)) # and should then write at most 1 byte self.assertTrue(wresult == 1, repr(wresult)) except IOError as e: self.fail("Timeout on mutlithreaded pipe; %s" % str(e)) t = threading.Thread(target=self.rallrunner, args=(p, )) t.start() try: # should block until all data has been read logging.debug("flush waiting...") p.flush() logging.debug("flush complete") self.assertTrue(p.canwrite() == 10, "empty after flush") except IOError as e: self.fail("flush timeout on mutlithreaded pipe; %s" % str(e)) # put the other thread out of its misery p.write_eof() logging.debug("eof written, joining rallrunner") t.join()
def test_wnblocking(self): p = Pipe(timeout=1, bsize=10, wblocking=False) self.assertTrue(p.readblocking()) self.assertFalse(p.writeblocking()) p.write(b"1234567890") data = b"extra" try: # should not block wresult = p.write(data) self.assertTrue(wresult is None, repr(wresult)) except IOError as e: self.fail("Timeout on non-blocking write; %s" % str(e)) # read all the data to empty the buffer self.assertTrue(len(p.read(10)) == 10, "in our case, True!") try: # should block for 1 second and then timeout rresult = p.read(1) self.fail("blocked read returned %s" % repr(rresult)) except IOError as e: self.assertTrue(io_timedout(e)) p.write(b"1234567890") try: # should not block! p.flush() self.fail("non-blocking flush returned with data") except io.BlockingIOError: pass except IOError as e: self.fail("non-blocking flush timed out; %s" % str(e))
def test_nochunked(self): """RFC2616: For compatibility with HTTP/1.0 applications, HTTP/1.1 requests containing a message-body MUST include a valid Content-Length header field unless the server is known to be HTTP/1.1 compliant""" t = threading.Thread(target=self.run_legacy, args=(2,)) t.start() request = http.ClientRequest("http://localhost:%i/nochunked" % self.port) self.assertTrue(request.protocol == params.HTTP_1p1) # start off optimistic about keep_alive self.assertTrue(request.keep_alive) self.client.process_request(request) self.assertTrue(request.response.status == 200) self.assertTrue(request.response.protocol == params.HTTP_1p0) # legacy server closes the connection self.assertFalse(request.response.keep_alive) # now try and make a call which would normally default to chunked data = b"How long is a piece of string?" bodytext = Pipe(rblocking=False, timeout=10) bodytext.write(data) request = http.ClientRequest("http://localhost:%i/nochunked" % self.port, "PUT", entity_body=bodytext) # we should now know that the server is 1.0, so we expect an # error when trying to send an unbounded entity without # content-length self.client.process_request(request) self.assertTrue(request.response.status is None) request.set_content_length(len(data)) self.client.process_request(request) self.assertTrue(request.response.status == 204) self.assertFalse(request.response.keep_alive)
def test_readall(self): p = Pipe(timeout=1, bsize=10, rblocking=False) p.write(b"123") # readall should block and timeout try: data = p.readall() self.fail("blocked readall returned: %s" % data) except IOError as e: self.assertTrue(io_timedout(e))
def test_eof(self): p = Pipe(timeout=1, bsize=10) p.write(b"123") self.assertTrue(p.canread()) p.read(3) self.assertFalse(p.canread()) p.write_eof() self.assertTrue(p.canread()) self.assertTrue(p.read(3) == b'')
def test_simple(self): p = Pipe() # default constructor self.assertTrue(isinstance(p, io.RawIOBase)) self.assertFalse(p.closed) try: p.fileno() self.fail("fileno should raise IOError") except IOError: pass self.assertFalse(p.isatty()) self.assertTrue(p.readable()) self.assertFalse(p.seekable()) self.assertTrue(p.writable()) # now for our custom attributes self.assertTrue(p.readblocking()) self.assertTrue(p.writeblocking()) self.assertTrue(p.canwrite() == io.DEFAULT_BUFFER_SIZE) self.assertFalse(p.canread()) # now try a quick read and write test data = b"The quick brown fox jumped over the lazy dog" wlen = p.write(data) self.assertTrue(wlen == len(data)) self.assertTrue(p.canwrite() == io.DEFAULT_BUFFER_SIZE - len(data)) self.assertTrue(p.canread()) self.assertTrue(p.read(3) == data[:3]) self.assertTrue(p.canwrite() == io.DEFAULT_BUFFER_SIZE - len(data) + 3) self.assertTrue(p.canread()) # now deal with EOF conditions p.write_eof() try: p.write(b"extra") self.fail("write past EOF") except IOError: pass try: p.canwrite() self.fail("canwrite called past EOF") except IOError: pass self.assertTrue(p.canread(), "But can still read") self.assertFalse(p.closed) self.assertTrue(p.readall() == data[3:]) self.assertTrue(p.canread(), "Can still read") self.assertTrue(p.read(3) == b'') self.assertTrue(len(p.read()) == 0) self.assertTrue(len(p.readall()) == 0) p.close() self.assertTrue(p.closed) try: p.canread() self.fail("canread called on closed pipe") except IOError: pass
def test_blocked(self): data = b"How long is a piece of string?" src = Pipe(rblocking=False, name="test_blocked") src.write(data) # no eof, default buffer size will trigger block b = BufferedStreamWrapper(src) self.assertTrue(b.readable()) self.assertFalse(b.writable()) self.assertFalse(b.seekable()) self.assertTrue(b.length is None) # but note that we can still peek on the data # that we have buffered self.assertTrue(b.peek(3) == data[0:3])
def test_unblocked(self): data = b"How long is a piece of string?" src = Pipe(rblocking=False, name="test_unblocked") # write one byte at a time for c in data: src.write(byte_to_bstr(c)) src.write_eof() # default buffer size outgrows pipe b = BufferedStreamWrapper(src) self.assertTrue(b.readable()) self.assertFalse(b.writable()) self.assertTrue(b.seekable()) self.assertTrue(b.length == len(data)) # the buffer also smooths access to the data self.assertTrue(b.read(10) == data[0:10])
def test_rnblocking(self): p = Pipe(timeout=1, bsize=10, rblocking=False) self.assertFalse(p.readblocking()) self.assertTrue(p.writeblocking()) try: # should not block rresult = p.read(1) self.assertTrue(rresult is None) except IOError as e: self.fail("Timeout on non-blocking read; %s" % str(e)) # write should still block p.write(b"1234567890") try: # should block for 1 second wresult = p.write(b"extra") self.fail("blocked write returned %s" % repr(wresult)) except IOError as e: self.assertTrue(io_timedout(e))
def test_wwait(self): p = Pipe(timeout=1, bsize=10, wblocking=False) p.write(b"1234567890") data = b"extra" wresult = p.write(data) self.assertTrue(wresult is None, "write blocked") try: p.write_wait(timeout=1) self.fail("wait should time out") except IOError as e: self.assertTrue(io_timedout(e)) t = threading.Thread(target=self.wwait_runner, args=(p, )) t.start() time.sleep(0) try: p.write_wait(timeout=5) pass except IOError: self.fail("write_wait error on multi-threaded read; %s" % str(e))
def test_wwait(self): p = Pipe(timeout=1, bsize=10, wblocking=False) p.write(b"1234567890") data = b"extra" wresult = p.write(data) self.assertTrue(wresult is None, "write blocked") try: p.write_wait(timeout=1) self.fail("wait should time out") except IOError as e: self.assertTrue(io_timedout(e)) t = threading.Thread(target=self.wwait_runner, args=(p,)) t.start() time.sleep(0) try: p.write_wait(timeout=5) pass except IOError: self.fail("write_wait error on multi-threaded read; %s" % str(e))
def test_rdetect(self): p = Pipe(timeout=15, bsize=10) rflag = threading.Event() # set a read event p.set_rflag(rflag) self.assertFalse(rflag.is_set()) t = threading.Thread(target=self.rrunner, args=(p, )) t.start() # the runner will issue a read call, should trigger the event rflag.wait(5.0) self.assertTrue(rflag.is_set()) # write 10 bytes, thread should terminate p.write(b"1234567890") t.join() # one byte read, write another byte p.write(b"A") # buffer should now be full at this point... self.assertFalse(p.canwrite()) self.assertFalse(rflag.is_set()) # the next call to read should set the flag again p.read(1) self.assertTrue(rflag.is_set())
def test_rdetect(self): p = Pipe(timeout=15, bsize=10) rflag = threading.Event() # set a read event p.set_rflag(rflag) self.assertFalse(rflag.is_set()) t = threading.Thread(target=self.rrunner, args=(p,)) t.start() # the runner will issue a read call, should trigger the event rflag.wait(5.0) self.assertTrue(rflag.is_set()) # write 10 bytes, thread should terminate p.write(b"1234567890") t.join() # one byte read, write another byte p.write(b"A") # buffer should now be full at this point... self.assertFalse(p.canwrite()) self.assertFalse(rflag.is_set()) # the next call to read should set the flag again p.read(1) self.assertTrue(rflag.is_set())
def test_nochunked(self): """RFC2616: For compatibility with HTTP/1.0 applications, HTTP/1.1 requests containing a message-body MUST include a valid Content-Length header field unless the server is known to be HTTP/1.1 compliant""" t = threading.Thread(target=self.run_legacy, args=(2, )) t.start() request = http.ClientRequest("http://localhost:%i/nochunked" % self.port) self.assertTrue(request.protocol == params.HTTP_1p1) # start off optimistic about keep_alive self.assertTrue(request.keep_alive) self.client.process_request(request) self.assertTrue(request.response.status == 200) self.assertTrue(request.response.protocol == params.HTTP_1p0) # legacy server closes the connection self.assertFalse(request.response.keep_alive) # now try and make a call which would normally default to chunked data = b"How long is a piece of string?" bodytext = Pipe(rblocking=False, timeout=10) bodytext.write(data) request = http.ClientRequest("http://localhost:%i/nochunked" % self.port, "PUT", entity_body=bodytext) # we should now know that the server is 1.0, so we expect an # error when trying to send an unbounded entity without # content-length self.client.process_request(request) self.assertTrue(request.response.status is None) request.set_content_length(len(data)) self.client.process_request(request) self.assertTrue(request.response.status == 204) self.assertFalse(request.response.keep_alive)
def test_breadmatch(self): p = Pipe(timeout=1, bsize=10, rblocking=True) p.write(b"12") p.write(b"3\r\n") self.assertTrue(p.readmatch() == b"123\r\n") # now check behaviour when no line is present p.write(b"1") p.write(b"2") p.write(b"3") try: p.readmatch() self.fail("blocking readmatch") except IOError as e: self.assertTrue(io_timedout(e)) p.write(b"\r") p.write(b"\nabc") self.assertTrue(p.readmatch() == b"123\r\n") # now check for buffer size exceeded p.write(b"\r3\n4\r56\n") try: p.readmatch() self.fail("blocking full buffer") except IOError as e: self.assertTrue(e.errno == errno.ENOBUFS) # now add an EOF and it should change the result p.write_eof() self.assertTrue(p.readmatch() == b'')
class MockSocketBase(object): id_next = 1 id_lock = threading.RLock() def __init__(self): with self.id_lock: self.id = MockSocketBase.id_next MockSocketBase.id_next += 1 self.blocking = True self.send_pipe = Pipe(timeout=10, name="MockSocket.send[%i]" % self.id) self.recv_pipe = Pipe(timeout=10, name="MockSocket.recv[%i]" % self.id) self.send_rbuffer = io.BufferedReader(self.send_pipe) self.recv_wbuffer = io.BufferedWriter(self.recv_pipe) self.io_error = None pass_select = select.select @classmethod def wrap_select(cls, rlist, wlist, xlist, timeout=5): mrlist = [] mwlist = [] mxlist = [] srlist = [] swlist = [] sxlist = [] for r in rlist: if isinstance(r, cls): mrlist.append(r) else: srlist.append(r) for w in wlist: if isinstance(w, cls): mwlist.append(w) else: swlist.append(w) for x in xlist: if isinstance(x, cls): mxlist.append(x) else: sxlist.append(x) if srlist or swlist or sxlist: return cls.pass_select(srlist, swlist, sxlist, timeout) else: return cls.select(mrlist, mwlist, mxlist, timeout) @classmethod def select(cls, rlist, wlist, xlist, timeout=5): the_time = time.time() if timeout is not None: tstop = the_time + timeout else: tstop = 0 while timeout is None or the_time <= tstop: # we must always go around at least once rs = [] ws = [] for r in rlist: try: if r.recv_pipe.canread(): rs.append(r) except IOError: # raise a socket error raise select.error(errno.EPIPE, os.strerror(errno.EPIPE)) for w in wlist: try: if w.send_pipe.canwrite(): ws.append(w) except IOError: # raise a socket error raise select.error(errno.EPIPE, os.strerror(errno.EPIPE)) if rs or ws: return rs, ws, [] else: time.sleep(1) the_time = time.time() return [], [], [] def setblocking(self, blocking): self.blocking = blocking # turn off blocking for recv and send self.recv_pipe.set_readblocking(blocking) self.send_pipe.set_writeblocking(blocking) def recv(self, nbytes): if self.io_error: raise self.io_error result = self.recv_pipe.read(nbytes) if self.io_error: raise self.io_error if result is None: # non-blocking read, nothing to read raise IOError(errno.EAGAIN, os.strerror(errno.EAGAIN), "MockSocket.recv") return result def send(self, data): if self.io_error: raise self.io_error result = self.send_pipe.write(data) if self.io_error: raise self.io_error if result is None: # non-blocking read, nothing to read raise IOError(errno.EAGAIN, os.strerror(errno.EAGAIN), "MockSocket.send") return result def shutdown(self, how): if self.io_error: raise self.io_error if how in (socket.SHUT_RD, socket.SHUT_RDWR): # don't want any more data self.recv_pipe.write_eof() # don't wait for recv buffer, we stopped reading # self.recv_wbuffer.flush() if how in (socket.SHUT_WR, socket.SHUT_RDWR): self.send_pipe.write_eof() # but wait for the client to finish reading self.send_pipe.set_writeblocking(True) self.send_rbuffer.flush() if self.io_error: raise self.io_error def mock_shutdown(self, how): if how in (socket.SHUT_WR, socket.SHUT_RDWR): self.recv_pipe.write_eof() # wait for the other end to finish reading self.recv_pipe.set_writeblocking(True) self.recv_wbuffer.flush() if how in (socket.SHUT_RD, socket.SHUT_RDWR): # don't want any more data self.send_pipe.write_eof() # but don't wait for send buffer, we stopped reading # self.send_rbuffer.flush() self.send_pipe.close() def close(self): self.send_pipe.close() self.recv_pipe.close()