def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" if self.is_client: self._request_start_line = start_line # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = (start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 304 responses have no body (not even a zero-length body), and so # should not have either Content-Length or Transfer-Encoding. # headers. start_line.code != 304 and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines = [utf8("%s %s %s" % start_line)] lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future
def test_handle_layer_error(self): context = LayerContext(mode="socks", src_stream=self.src_stream, port=443, scheme="h2") layer_manager._handle_layer_error(gen.TimeoutError("timeout"), context) context.src_stream.close.assert_called_once_with() context.src_stream.reset_mock() layer_manager._handle_layer_error( DestNotConnectedError("stream closed"), context) context.src_stream.close.assert_not_called() context.src_stream.reset_mock() layer_manager._handle_layer_error( DestStreamClosedError("stream closed"), context) context.src_stream.close.assert_called_once_with() context.src_stream.reset_mock() layer_manager._handle_layer_error( SrcStreamClosedError("stream closed"), context) context.src_stream.close.assert_not_called() context.src_stream.reset_mock() layer_manager._handle_layer_error( iostream.StreamClosedError("stream closed"), context) context.src_stream.close.assert_called_once_with()
def readFrame(self): # IOStream processes reads one at a time with (yield self._read_lock.acquire()): with self.io_exception_context(): frame_header = yield self.stream.read_bytes(4) if len(frame_header) == 0: raise iostream.StreamClosedError('Read zero bytes from stream') frame_length, = struct.unpack('!i', frame_header) frame = yield self.stream.read_bytes(frame_length) raise gen.Return(frame)
def write_headers(self, start_line, headers, chunk=None, callback=None): lines = [] if self.is_client: self._request_start_line = start_line lines.append( utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) self._chunking_output = (start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line lines.append( utf8('HTTP/1.1 %s %s' % (start_line[1], start_line[2]))) self._chunking_output = (self._request_start_line.version == 'HTTP/1.1' and start_line.code != 304 and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future
def read_frame(self): # IOStream processes reads one at a time with (yield self._read_lock.acquire()): with self.io_exception_context(): frame_header = yield self._read_bytes(4) if len(frame_header) == 0: raise iostream.StreamClosedError( 'Read zero bytes from stream') frame_length, = struct.unpack('!i', frame_header) logging.debug('received frame header, frame length = %d', frame_length) frame = yield self._read_bytes(frame_length) logging.debug('received frame payload: %r', frame) raise gen.Return(frame)
def write(self, chunk, callback=None): future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) self._pending_write.add_done_callback(self._on_write_complete) return future
def write(self, chunk: bytes) -> "Future[None]": """Implements `.HTTPConnection.write`. For backwards compatibility it is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) future_add_done_callback(self._pending_write, self._on_write_complete) return future
def test_start_exceptions_in_all_connections(self): self.add_broker( "kafka01", 9092, connect_error=Exception("oh no!") ) self.add_broker( "kafka02", 9000, connect_error=iostream.StreamClosedError(), ) self.add_broker( "kafka03", 9092, connect_error=exc.BrokerConnectionError("kafka03", 9092), ) c = cluster.Cluster(["kafka01", "kafka02:9000", "kafka03"]) with self.assertRaises(exc.NoBrokersError): yield c.start()
def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility is is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ if self.stream.closed(): self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: self._write_future = Future() self.stream.write(self._format_chunk(chunk), self._on_write_complete) return self._write_future
def test_send_stream_closed(self): self.set_responses( broker_id=1, api="metadata", responses=[ iostream.StreamClosedError(), ] ) c = client.Client(["kafka01", "kafka02"]) request = Mock(api="metadata") results = yield c.send({1: request}) self.assert_sent(1, request) self.assertEqual(c.cluster.heal.called, False) self.assertEqual(results, {})
def write(self, chunk, callback=None): """Implements `.HTTPConnection.write`. For backwards compatibility it is allowed but deprecated to skip `write_headers` and instead call `write()` with a pre-encoded header block. """ future = None if self.stream.closed(): future = self._write_future = Future() self._write_future.set_exception(iostream.StreamClosedError()) self._write_future.exception() else: if callback is not None: warnings.warn("callback argument is deprecated, use returned Future instead", DeprecationWarning) self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() self._pending_write = self.stream.write(self._format_chunk(chunk)) self._pending_write.add_done_callback(self._on_write_complete) return future
def write_headers( self, start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine], headers: httputil.HTTPHeaders, chunk: bytes = None, ) -> "Future[None]": """Implements `.HTTPConnection.write_headers`.""" lines = [] if self.is_client: assert isinstance(start_line, httputil.RequestStartLine) self._request_start_line = start_line lines.append( utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1]))) # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = ( start_line.method in ("POST", "PUT", "PATCH") and "Content-Length" not in headers and ("Transfer-Encoding" not in headers or headers["Transfer-Encoding"] == "chunked")) else: assert isinstance(start_line, httputil.ResponseStartLine) assert self._request_start_line is not None assert self._request_headers is not None self._response_start_line = start_line lines.append( utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2]))) self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == "HTTP/1.1" # 1xx, 204 and 304 responses have no body (not even a zero-length # body), and so should not have either Content-Length or # Transfer-Encoding headers. and start_line.code not in (204, 304) and (start_line.code < 100 or start_line.code >= 200) # No need to chunk the output if a Content-Length is specified. and "Content-Length" not in headers # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. and "Transfer-Encoding" not in headers) # If connection to a 1.1 client will be closed, inform client if (self._request_start_line.version == "HTTP/1.1" and self._disconnect_on_finish): headers["Connection"] = "close" # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == "HTTP/1.0" and self._request_headers.get("Connection", "").lower() == "keep-alive"): headers["Connection"] = "Keep-Alive" if self._chunking_output: headers["Transfer-Encoding"] = "chunked" if not self.is_client and (self._request_start_line.method == "HEAD" or cast(httputil.ResponseStartLine, start_line).code == 304): self._expected_content_remaining = 0 elif "Content-Length" in headers: self._expected_content_remaining = int(headers["Content-Length"]) else: self._expected_content_remaining = None # TODO: headers are supposed to be of type str, but we still have some # cases that let bytes slip through. Remove these native_str calls when those # are fixed. header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) lines.extend(l.encode("latin1") for l in header_lines) for line in lines: if b"\n" in line: raise ValueError("Newline in header: " + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) future_add_done_callback(self._pending_write, self._on_write_complete) return future
def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" lines = [] if self.is_client: self._request_start_line = start_line lines.append( utf8('%s %s HTTP/1.1' % (start_line[0], start_line[1]))) # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. self._chunking_output = (start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line lines.append( utf8('HTTP/1.1 %d %s' % (start_line[1], start_line[2]))) self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 1xx, 204 and 304 responses have no body (not even a zero-length # body), and so should not have either Content-Length or # Transfer-Encoding headers. start_line.code not in (204, 304) and (start_line.code < 100 or start_line.code >= 200) and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If connection to a 1.1 client will be closed, inform client if (self._request_start_line.version == 'HTTP/1.1' and self._disconnect_on_finish): headers['Connection'] = 'close' # If a 1.0 client asked for keep-alive, add the header. if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None # TODO: headers are supposed to be of type str, but we still have some # cases that let bytes slip through. Remove these native_str calls when those # are fixed. header_lines = (native_str(n) + ": " + native_str(v) for n, v in headers.get_all()) if PY3: lines.extend(l.encode('latin1') for l in header_lines) else: lines.extend(header_lines) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) future.exception() else: if callback is not None: self._write_callback = stack_context.wrap(callback) else: future = self._write_future = Future() data = b"\r\n".join(lines) + b"\r\n\r\n" if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future
def test_heal_with_initial_broker_errors(self): initial = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), metadata.Broker(broker_id=7, host="kafka02", port=9000), metadata.Broker(broker_id=8, host="kafka03", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=0, leader=2, replicas=[], isrs=[], ), ] ), ], ) fixed = metadata.MetadataResponse( brokers=[ metadata.Broker(broker_id=2, host="kafka01", port=9092), ], topics=[ metadata.TopicMetadata( error_code=errors.no_error, name="test.topic", partitions=[ metadata.PartitionMetadata( error_code=errors.no_error, partition_id=0, leader=2, replicas=[], isrs=[], ), ] ), ], ) self.add_broker("kafka01", 9092, responses=[initial, fixed]) self.add_broker( "kafka02", 9000, connect_error=iostream.StreamClosedError(), ) self.add_broker( "kafka03", 9092, connect_error=exc.BrokerConnectionError("kafka03", 9092), ) conn1 = cluster.Connection("kafka01", 9092) c = cluster.Cluster(["kafka01", "kafka02:900"]) c.conns = {2: conn1} yield c.heal() self.assertEqual([2], list(c.conns.keys()))
def on_connection_close(self): if _has_stream_request_body(self.__class__): if not self.request.body.done(): self.request.body.set_exception(iostream.StreamClosedError()) self.request.body.exception()
def write_headers(self, start_line, headers, chunk=None, callback=None): """Implements `.HTTPConnection.write_headers`.""" if self.is_client: self._request_start_line = start_line # Client requests with a non-empty body must have either a # Content-Length or a Transfer-Encoding. # 不检查是否 Http/1.0 是不完备的。 self._chunking_output = ( start_line.method in ('POST', 'PUT', 'PATCH') and 'Content-Length' not in headers and 'Transfer-Encoding' not in headers) else: self._response_start_line = start_line # 对于 HTTP/1.0 ``self._chunking_output=False``,不支持分块传输编码。 self._chunking_output = ( # TODO: should this use # self._request_start_line.version or # start_line.version? self._request_start_line.version == 'HTTP/1.1' and # 304 responses have no body (not even a zero-length body), and so # should not have either Content-Length or Transfer-Encoding. # headers. start_line.code != 304 and # No need to chunk the output if a Content-Length is specified. 'Content-Length' not in headers and # Applications are discouraged from touching Transfer-Encoding, # but if they do, leave it alone. 'Transfer-Encoding' not in headers) # If a 1.0 client asked for keep-alive, add the header. # HTTP/1.1 默认就是持久化连接,不需要单独指定。 # 假设客户端请求使用 HTTP/1.0 和 `Connection:Keep-Alive`,服务端响应时没有指定 # `Content-Length` (比如在 handler 中多次调用 flush 方法),那么响应数据就无法 # 判断边界,代码中应该对这个条件做特别处理。 if (self._request_start_line.version == 'HTTP/1.0' and (self._request_headers.get('Connection', '').lower() == 'keep-alive')): headers['Connection'] = 'Keep-Alive' if self._chunking_output: headers['Transfer-Encoding'] = 'chunked' # 服务端响应 `HEAD` 或者 304 时不需要 body 数据。 if (not self.is_client and (self._request_start_line.method == 'HEAD' or start_line.code == 304)): self._expected_content_remaining = 0 elif 'Content-Length' in headers: self._expected_content_remaining = int(headers['Content-Length']) else: self._expected_content_remaining = None lines = [utf8("%s %s %s" % start_line)] # 通过 add 添加的响应头会输出多个,比如:“Set-Cookie” 响应头。 lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()]) for line in lines: if b'\n' in line: raise ValueError('Newline in header: ' + repr(line)) future = None if self.stream.closed(): future = self._write_future = Future() future.set_exception(iostream.StreamClosedError()) else: # "写回调" 是一个实例字段 `_write_callback`,当上一次写操作还没有回调时就再次执行 # 写操作,那么上一次写操作的回调将被放弃(callback is not None) if callback is not None: self._write_callback = stack_context.wrap(callback) else: # 没有 callback 时,返回 Future(self._write_future) future = self._write_future = Future() # Headers data = b"\r\n".join(lines) + b"\r\n\r\n" # message-body if chunk: data += self._format_chunk(chunk) self._pending_write = self.stream.write(data) self._pending_write.add_done_callback(self._on_write_complete) return future