def test_read_headers_out_of_order(self): # If header blocks aren't decoded in the same order they're received, # regardless of the stream they belong to, the decoder state will # become corrupted. e = Encoder() h1 = HeadersFrame(1) h1.data = e.encode([(':status', 200), ('content-type', 'foo/bar')]) h1.flags |= set(['END_HEADERS', 'END_STREAM']) h3 = HeadersFrame(3) h3.data = e.encode([(':status', 200), ('content-type', 'baz/qux')]) h3.flags |= set(['END_HEADERS', 'END_STREAM']) sock = DummySocket() sock.buffer = BytesIO(h1.serialize() + h3.serialize()) c = HTTP20Connection('www.google.com') c._sock = sock r1 = c.request('GET', '/a') r3 = c.request('GET', '/b') assert c.get_response(r3).headers == HTTPHeaderMap( [('content-type', 'baz/qux')] ) assert c.get_response(r1).headers == HTTPHeaderMap( [('content-type', 'foo/bar')] )
def test_response_ignores_unknown_headers(self): headers = HTTPHeaderMap([(':status', '200'), (':reserved', 'yes'), ('no', 'no')]) stream = DummyStream(b'') resp = HTTP20Response(headers, stream) assert resp.headers == HTTPHeaderMap([('no', 'no')])
def test_actual_get(self): h = HTTPHeaderMap() h['k1'] = 'v1, v2' h['k2'] = 'v3' h['k1'] = 'v4' assert h.get('k1') == [b'v1', b'v2', b'v4']
def test_can_receive_trailers(self): headers = [('a', 'b'), ('c', 'd'), (':status', '200')] trailers = [('e', 'f'), ('g', 'h')] s = Stream(1, None, None, None, None, FixedDecoder(headers), None) s.state = STATE_HALF_CLOSED_LOCAL # Provide the first HEADERS frame. f = HeadersFrame(1) f.data = b'hi there!' f.flags.add('END_HEADERS') s.receive_frame(f) assert s.response_headers == HTTPHeaderMap(headers) # Now, replace the dummy decoder to ensure we get a new header block. s._decoder = FixedDecoder(trailers) # Provide the trailers. f = HeadersFrame(1) f.data = b'hi there again!' f.flags.add('END_STREAM') f.flags.add('END_HEADERS') s.receive_frame(f) # Now, check the trailers. assert s.response_trailers == HTTPHeaderMap(trailers) # Confirm we closed the stream. assert s.state == STATE_CLOSED
def test_keys(self): h = HTTPHeaderMap() h['k1'] = 'v1, v2' h['k2'] = 'v3' h['k1'] = 'v4' assert len(list(h.keys())) == 4 assert list(h.keys()) == [b'k1', b'k1', b'k2', b'k1']
def test_values(self): h = HTTPHeaderMap() h['k1'] = 'v1, v2' h['k2'] = 'v3' h['k1'] = 'v4' assert len(list(h.values())) == 4 assert list(h.values()) == [b'v1', b'v2', b'v3', b'v4']
def test_raw_iteration(self): items = [ (b'k1', b'v2'), (b'k2', b'v2, v3, v4'), (b'k2', b'v3'), ] h = HTTPHeaderMap(items) assert list(h.iter_raw()) == items
def test_trailers_are_read(self): headers = HTTPHeaderMap([(':status', '200')]) trailers = HTTPHeaderMap([('a', 'b'), ('c', 'd')]) stream = DummyStream(b'', trailers=trailers) resp = HTTP20Response(headers, stream) assert resp.trailers == trailers assert resp.trailers['a'] == [b'b'] assert resp.trailers['c'] == [b'd']
def test_headers_must_be_strings(self): with pytest.raises(ValueError): HTTPHeaderMap(key=1) h = HTTPHeaderMap() with pytest.raises(ValueError): h['k'] = 1 with pytest.raises(ValueError): h[1] = 'v'
def test_merge_header_map_dict(self): h = HTTPHeaderMap([(b'hi', b'there')]) d = {'cat': 'dog'} h.merge(d) assert list(h.items()) == [ (b'hi', b'there'), (b'cat', b'dog'), ]
def __init__(self, connection, ip, stream_id, host, task, send_cb, close_cb, header_encoder, header_decoder, receive_window_manager, remote_window_size, max_frame_size): self.connection = connection self.ip = ip self.stream_id = stream_id self.host = host self.task = task self.state = STATE_IDLE self.get_head_time = None # There are two flow control windows: one for data we're sending, # one for data being sent to us. self.receive_window_manager = receive_window_manager self.remote_window_size = remote_window_size self.max_frame_size = max_frame_size # This is the callback handed to the stream by its parent connection. # It is called when the stream wants to send data. It expects to # receive a list of frames that will be automatically serialized. self._send_cb = send_cb # This is the callback to be called when the stream is closed. self._close_cb = close_cb # A reference to the header encoder and decoder objects belonging to # the parent connection. self._encoder = header_encoder self._decoder = header_decoder self.request_headers = HTTPHeaderMap() # Convert the body to bytes if needed. self.request_body = to_bytestring(self.task.body) # request body not send blocked by send window # the left body will send when send window opened. self.request_body_left = len(self.request_body) self.request_body_sended = False # data list before decode self.response_header_datas = [] # Set to a key-value set of the response headers once their # HEADERS..CONTINUATION frame sequence finishes. self.response_headers = None # Unconsumed response data chunks self.response_body = [] self.response_body_len = 0 threading.Thread(target=self.timeout_response).start() self.start_request()
def test_pushed_requests_ignore_unexpected_headers(self): headers = HTTPHeaderMap([ (':scheme', 'http'), (':method', 'get'), (':authority', 'google.com'), (':path', '/'), (':reserved', 'no'), ('no', 'no'), ]) p = HTTP20Push(headers, DummyStream(b'')) assert p.request_headers == HTTPHeaderMap([('no', 'no')])
def test_equality(self): h1 = HTTPHeaderMap() h1['k1'] = 'v1, v2' h1['k2'] = 'v3' h1['k1'] = 'v4' h2 = HTTPHeaderMap() h2['k1'] = 'v1, v2' h2['k2'] = 'v3' h2['k1'] = 'v4' assert h1 == h2
def test_items(self): h = HTTPHeaderMap() items = [ (b'k1', b'v2'), (b'k2', b'v2'), (b'k2', b'v3'), ] for k, v in items: h[k] = v for i, pair in enumerate(h.items()): assert items[i] == pair
def test_inequality_of_raw_ordering(self): h1 = HTTPHeaderMap() h1['k1'] = 'v1, v2' h1['k2'] = 'v3' h1['k1'] = 'v4' h2 = HTTPHeaderMap() h2['k1'] = 'v1' h2['k1'] = 'v2' h2['k2'] = 'v3' h2['k1'] = 'v4' assert h1 != h2
def h2_headers(self, node=None, secret=None, multiplexed=None, af=None): headers = HTTPHeaderMap() if node: if isinstance(node, (tuple, list, set)): for n in node: headers.update({Headers.node: n}) else: headers.update({Headers.node: node}) if secret and af != socket.AF_UNIX: headers.update({Headers.secret: secret}) if multiplexed: headers.update({Headers.multiplexed: "true"}) return headers
def test_merge_headermaps_preserves_raw(self): h1 = HTTPHeaderMap([ (b'hi', b'there') ]) h2 = HTTPHeaderMap([ (b'Hi', b'there, sir, maam') ]) h1.merge(h2) assert list(h1.iter_raw()) == [ (b'hi', b'there'), (b'Hi', b'there, sir, maam'), ]
def action(self, nodename, thr=None, stream_id=None, **kwargs): logfile = os.path.join(rcEnv.paths.pathlog, "node.log") ofile = thr._action_logs_open(logfile, 0, "node") request_headers = HTTPHeaderMap( thr.streams[stream_id]["request"].headers) try: content_type = bdecode(request_headers.get("accept").pop()) except: content_type = "application/json" thr.streams[stream_id]["content_type"] = content_type thr.streams[stream_id]["pushers"].append({ "o": self, "fn": "h2_push_logs", "args": [ofile, True], })
def test_getheader(self): headers = HTTPHeaderMap([(':status', '200'), ('content-type', 'application/json')]) stream = DummyStream(b'') resp = HTTP20Response(headers, stream) assert resp.headers[b'content-type'] == [b'application/json']
def test_response_calls_stream_close(self): headers = HTTPHeaderMap([(':status', '200')]) stream = DummyStream('') resp = HTTP20Response(headers, stream) resp.close() assert stream.closed
def _handle_header_block(self, headers): """ Handles the logic for receiving a completed headers block. A headers block is an uninterrupted sequence of one HEADERS frame followed by zero or more CONTINUATION frames, and is terminated by a frame bearing the END_HEADERS flag. HTTP/2 allows receipt of up to three such blocks on a stream. The first is optional, and contains a 1XX response. The second is mandatory, and must contain a final response (200 or higher). The third is optional, and may contain 'trailers', headers that are sent after a chunk-encoded body is sent. Here we only process the simple state: no push, one header frame. """ if self.response_headers is None: self.response_headers = HTTPHeaderMap(headers) else: # Received too many headers blocks. raise ProtocolError("Too many header blocks.") status = int(self.response_headers[b':status'][0]) if status in [400, 403]: # xlog.warn("status:%d host:%s", status, self.host) self.connection.close("get 40x") return
class DummyResponse(object): def __init__(self, headers): self.headers = HTTPHeaderMap(headers.items()) def read(self): ctype = self.headers.get('content-type') if ctype is not None: if 'json' in ctype[0].decode('utf-8'): return b'{"data": "dummy"}' return b'<html>dummy</html>' def getheader(self, name): return self.headers.get(name) def getheaders(self): return self.headers
def test_containment(self): h = HTTPHeaderMap() h['key'] = 'val' assert 'key' in h assert b'key' in h assert 'nonkey' not in h
def test_header_map_can_delete_value(self): h = HTTPHeaderMap() h['key'] = b'v1' del h[b'key'] with pytest.raises(KeyError): h[b'key']
def test_length_counts_lines_separately(self): h = HTTPHeaderMap() h['k1'] = 'v1, v2' h['k2'] = 'v3' h['k1'] = 'v4' assert len(h) == 4
def __init__(self, ip, stream_id, host, task, send_cb, close_cb, header_encoder, header_decoder, receive_window_manager, remote_window_size, max_frame_size): self.ip = ip self.stream_id = stream_id self.host = host self.task = task self.state = STATE_IDLE # There are two flow control windows: one for data we're sending, # one for data being sent to us. self.receive_window_manager = receive_window_manager self.remote_window_size = remote_window_size self.max_frame_size = max_frame_size # This is the callback handed to the stream by its parent connection. # It is called when the stream wants to send data. It expects to # receive a list of frames that will be automatically serialized. self._send_cb = send_cb # This is the callback to be called when the stream is closed. self._close_cb = close_cb # A reference to the header encoder and decoder objects belonging to # the parent connection. self._encoder = header_encoder self._decoder = header_decoder self.request_headers = HTTPHeaderMap() # Convert the body to bytes if needed. self.request_body = to_bytestring(self.task.body) # request body not send blocked by send window # the left body will send when send window opened. self.request_body_left = len(self.request_body) # data list before decode self.response_header_datas = [] # Set to a key-value set of the response headers once their # HEADERS..CONTINUATION frame sequence finishes. self.response_headers = None # Unconsumed response data chunks self.response_body = [] self.response_body_len = 0 self.start_request()
def test_header_map_deletes_all_values(self): h = HTTPHeaderMap() h['key'] = 'v1' h['key'] = 'v2' del h['key'] with pytest.raises(KeyError): h['key']
def test_responses_are_context_managers(self): headers = HTTPHeaderMap([(':status', '200')]) stream = DummyStream('') with HTTP20Response(headers, stream): pass assert stream.closed
def action(self, nodename, thr=None, stream_id=None, **kwargs): options = self.parse_options(kwargs) thr.selector = options.selector if not thr.event_queue: thr.event_queue = queue.Queue() if options.full: data = thr.daemon_status() namespaces = thr.get_namespaces() fevent = { "nodename": rcEnv.nodename, "ts": time.time(), "kind": "full", "data": thr.filter_daemon_status(data, namespaces=namespaces, selector=options.selector), } if thr.h2conn: _msg = fevent elif thr.encrypted: _msg = thr.encrypt(fevent) else: _msg = thr.msg_encode(fevent) thr.event_queue.put(_msg) if not thr in thr.parent.events_clients: thr.parent.events_clients.append(thr) if not stream_id in thr.events_stream_ids: thr.events_stream_ids.append(stream_id) if thr.h2conn: request_headers = HTTPHeaderMap( thr.streams[stream_id]["request"].headers) try: content_type = bdecode(request_headers.get("accept").pop()) except: content_type = "application/json" thr.streams[stream_id]["content_type"] = content_type thr.streams[stream_id]["pushers"].append({ "fn": "h2_push_action_events", }) else: thr.raw_push_action_events()
def test_request_with_unicodestring_body(self): c = HTTP11Connection('httpbin.org') c._sock = DummySocket() with pytest.raises(ValueError): c.request('POST', '/post', headers=HTTPHeaderMap([('User-Agent', 'hyper')]), body=u'hi')
def test_can_create_from_multiple_iterables(self): items = [ (b'k1', b'v2'), (b'k2', b'v2'), (b'k2', b'v3'), ] h = HTTPHeaderMap(items, items, items) assert list(h) == items + items + items
def __init__(self, attrs): self.body = {} self.headers = HTTPHeaderMap() self.items = [] self.method = None self._url = '' self.url = DummyUrlInfo() for key, value in attrs.items(): setattr(self, key, value)
def test_replacing(self): h = HTTPHeaderMap([ (b'name', b'value'), (b'name2', b'value2'), (b'name2', b'value2'), (b'name3', b'value3'), ]) h.replace('name2', '42') h.replace('name4', 'other_value') assert list(h.items()) == [ (b'name', b'value'), (b'name2', b'42'), (b'name3', b'value3'), (b'name4', b'other_value'), ]
def __init__(self, headers): self.headers = HTTPHeaderMap(headers.items())
def test_doesnt_split_set_cookie(self): h = HTTPHeaderMap() h['Set-Cookie'] = 'v1, v2' assert h['set-cookie'] == [b'v1, v2'] assert h.get(b'set-cookie') == [b'v1, v2']
class Stream(object): """ A single HTTP/2 stream. A stream is an independent, bi-directional sequence of HTTP headers and data. Each stream is identified by a single integer. From a HTTP perspective, a stream _approximately_ matches a single request-response pair. """ def __init__(self, logger, config, connection, ip, stream_id, task, send_cb, close_cb, encoder, decoder, receive_window_manager, remote_window_size, max_frame_size): self.logger = logger self.config = config self.connection = connection self.ip = ip self.stream_id = stream_id self.task = task self.state = STATE_IDLE self.get_head_time = None # There are two flow control windows: one for data we're sending, # one for data being sent to us. self.receive_window_manager = receive_window_manager self.remote_window_size = remote_window_size self.max_frame_size = max_frame_size # This is the callback handed to the stream by its parent connection. # It is called when the stream wants to send data. It expects to # receive a list of frames that will be automatically serialized. self._send_cb = send_cb # This is the callback to be called when the stream is closed. self._close_cb = close_cb # A reference to the header encoder and decoder objects belonging to # the parent connection. self._encoder = encoder self._decoder = decoder self.request_headers = HTTPHeaderMap() # Convert the body to bytes if needed. self.request_body = to_bytestring(self.task.body) # request body not send blocked by send window # the left body will send when send window opened. self.request_body_left = len(self.request_body) self.request_body_sended = False # data list before decode self.response_header_datas = [] # Set to a key-value set of the response headers once their # HEADERS..CONTINUATION frame sequence finishes. self.response_headers = None # Unconsumed response data chunks self.response_body = [] self.response_body_len = 0 def start_request(self): """ Open the stream. Does this by encoding and sending the headers: no more calls to ``add_header`` are allowed after this method is called. The `end` flag controls whether this will be the end of the stream, or whether data will follow. """ # Strip any headers invalid in H2. #headers = h2_safe_headers(self.request_headers) host = self.connection.get_host(self.task.host) self.add_header(":method", self.task.method) self.add_header(":scheme", "https") self.add_header(":authority", host) self.add_header(":path", self.task.path) default_headers = (':method', ':scheme', ':authority', ':path') #headers = h2_safe_headers(self.task.headers) for name, value in self.task.headers.items(): is_default = to_native_string(name) in default_headers self.add_header(name, value, replace=is_default) # Encode the headers. encoded_headers = self._encoder(self.request_headers) # It's possible that there is a substantial amount of data here. The # data needs to go into one HEADERS frame, followed by a number of # CONTINUATION frames. For now, for ease of implementation, let's just # assume that's never going to happen (16kB of headers is lots!). # Additionally, since this is so unlikely, there's no point writing a # test for this: it's just so simple. if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover raise ValueError("Header block too large.") header_frame = HeadersFrame(self.stream_id) header_frame.data = encoded_headers # If no data has been provided, this is the end of the stream. Either # way, due to the restriction above it's definitely the end of the # headers. header_frame.flags.add('END_HEADERS') if self.request_body_left == 0: header_frame.flags.add('END_STREAM') # Send the header frame. self.task.set_state("start send header") self._send_cb(header_frame) # Transition the stream state appropriately. self.state = STATE_OPEN self.task.set_state("start send left body") threading.Thread(target=self.left_work).start() def left_work(self): if self.request_body_left > 0: self.send_left_body() self.timeout_response() def add_header(self, name, value, replace=False): """ Adds a single HTTP header to the headers to be sent on the request. """ if not replace: self.request_headers[name] = value else: self.request_headers.replace(name, value) def send_left_body(self): while self.remote_window_size and not self.request_body_sended: send_size = min(self.remote_window_size, self.request_body_left, self.max_frame_size) f = DataFrame(self.stream_id) data_start = len(self.request_body) - self.request_body_left f.data = self.request_body[data_start:data_start+send_size] self.remote_window_size -= send_size self.request_body_left -= send_size # If the length of the data is less than MAX_CHUNK, we're probably # at the end of the file. If this is the end of the data, mark it # as END_STREAM. if self.request_body_left == 0: f.flags.add('END_STREAM') # Send the frame and decrement the flow control window. self._send_cb(f) # If no more data is to be sent on this stream, transition our state. if self.request_body_left == 0: self.request_body_sended = True self._close_local() self.task.set_state("end send left body") def receive_frame(self, frame): """ Handle a frame received on this stream. called by connection. """ # self.logger.debug("stream %d recved frame %r", self.stream_id, frame) if frame.type == WindowUpdateFrame.type: self.remote_window_size += frame.window_increment self.send_left_body() elif frame.type == HeadersFrame.type: # Begin the header block for the response headers. #self.response_header_datas = [frame.data] self.response_header_datas.append(frame.data) elif frame.type == PushPromiseFrame.type: self.logger.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id) elif frame.type == ContinuationFrame.type: # Continue a header block begun with either HEADERS or PUSH_PROMISE. self.response_header_datas.append(frame.data) elif frame.type == DataFrame.type: # Append the data to the buffer. if not self.task.finished: self.task.put_data(frame.data) if 'END_STREAM' not in frame.flags: # Increase the window size. Only do this if the data frame contains # actual data. # don't do it if stream is closed. size = frame.flow_controlled_length increment = self.receive_window_manager._handle_frame(size) #if increment: # self.logger.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment) #content_len = int(self.request_headers.get("Content-Length")[0]) #self.logger.debug("%s get:%d s:%d", self.ip, self.response_body_len, size) if increment and not self._remote_closed: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == BlockedFrame.type: # If we've been blocked we may want to fixup the window. increment = self.receive_window_manager._blocked() if increment: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == RstStreamFrame.type: # Rest Frame send from server is not define in RFC inactive_time = time.time() - self.connection.last_active_time self.logger.debug("%s Stream %d Rest by server, inactive:%d. error code:%d", self.ip, self.stream_id, inactive_time, frame.error_code) self.connection.close("RESET") elif frame.type in FRAMES: # This frame isn't valid at this point. #raise ValueError("Unexpected frame %s." % frame) self.logger.error("%s Unexpected frame %s.", self.ip, frame) else: # pragma: no cover # Unknown frames belong to extensions. Just drop it on the # floor, but log so that users know that something happened. self.logger.error("%s Received unknown frame, type %d", self.ip, frame.type) pass if 'END_HEADERS' in frame.flags: if self.response_headers is not None: raise ProtocolError("Too many header blocks.") # Begin by decoding the header block. If this fails, we need to # tear down the entire connection. if len(self.response_header_datas) == 1: header_data = self.response_header_datas[0] else: header_data = b''.join(self.response_header_datas) try: headers = self._decoder.decode(header_data) except Exception as e: self.logger.exception("decode h2 header %s fail:%r", header_data, e) raise e self.response_headers = HTTPHeaderMap(headers) # We've handled the headers, zero them out. self.response_header_datas = None self.get_head_time = time.time() length = self.response_headers.get("Content-Length", None) if isinstance(length, list): length = int(length[0]) if not self.task.finished: self.task.content_length = length self.task.set_state("h2_get_head") self.send_response() if 'END_STREAM' in frame.flags: #self.logger.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id) time_now = time.time() time_cost = time_now - self.get_head_time if time_cost > 0 and \ isinstance(self.task.content_length, int) and \ not self.task.finished: speed = self.task.content_length / time_cost self.task.set_state("h2_finish[SP:%d]" % speed) self._close_remote() self.close("end stream") if not self.task.finished: self.connection.continue_timeout = 0 def send_response(self): if self.task.responsed: self.logger.error("http2_stream send_response but responsed.%s", self.task.url) self.close("h2 stream send_response but sended.") return self.task.responsed = True status = int(self.response_headers[b':status'][0]) strip_headers(self.response_headers) response = simple_http_client.BaseResponse(status=status, headers=self.response_headers) response.ssl_sock = self.connection.ssl_sock response.worker = self.connection response.task = self.task self.task.queue.put(response) if status in self.config.http2_status_to_close: self.connection.close("status %d" % status) def close(self, reason="close"): if not self.task.responsed: self.connection.retry_task_cb(self.task, reason) else: self.task.finish() # empty block means fail or closed. self._close_remote() self._close_cb(self.stream_id, reason) @property def _local_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL) @property def _remote_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE) @property def _local_open(self): return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE) def _close_local(self): self.state = ( STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN else STATE_CLOSED ) def _close_remote(self): self.state = ( STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN else STATE_CLOSED ) def timeout_response(self): start_time = time.time() while time.time() - start_time < self.task.timeout: time.sleep(1) if self._remote_closed: return self.logger.warn("h2 timeout %s task_trace:%s worker_trace:%s", self.connection.ssl_sock.ip, self.task.get_trace(), self.connection.get_trace()) self.task.set_state("timeout") if self.task.responsed: self.task.finish() else: self.task.response_fail("timeout") self.connection.continue_timeout += 1 if self.connection.continue_timeout >= self.connection.config.http2_max_timeout_tasks and \ time.time() - self.connection.last_active_time > self.connection.config.http2_timeout_active: self.connection.close("down fail")
def receive_frame(self, frame): """ Handle a frame received on this stream. called by connection. """ # self.logger.debug("stream %d recved frame %r", self.stream_id, frame) if frame.type == WindowUpdateFrame.type: self.remote_window_size += frame.window_increment self.send_left_body() elif frame.type == HeadersFrame.type: # Begin the header block for the response headers. #self.response_header_datas = [frame.data] self.response_header_datas.append(frame.data) elif frame.type == PushPromiseFrame.type: self.logger.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id) elif frame.type == ContinuationFrame.type: # Continue a header block begun with either HEADERS or PUSH_PROMISE. self.response_header_datas.append(frame.data) elif frame.type == DataFrame.type: # Append the data to the buffer. if not self.task.finished: self.task.put_data(frame.data) if 'END_STREAM' not in frame.flags: # Increase the window size. Only do this if the data frame contains # actual data. # don't do it if stream is closed. size = frame.flow_controlled_length increment = self.receive_window_manager._handle_frame(size) #if increment: # self.logger.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment) #content_len = int(self.request_headers.get("Content-Length")[0]) #self.logger.debug("%s get:%d s:%d", self.ip, self.response_body_len, size) if increment and not self._remote_closed: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == BlockedFrame.type: # If we've been blocked we may want to fixup the window. increment = self.receive_window_manager._blocked() if increment: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == RstStreamFrame.type: # Rest Frame send from server is not define in RFC inactive_time = time.time() - self.connection.last_active_time self.logger.debug("%s Stream %d Rest by server, inactive:%d. error code:%d", self.ip, self.stream_id, inactive_time, frame.error_code) self.connection.close("RESET") elif frame.type in FRAMES: # This frame isn't valid at this point. #raise ValueError("Unexpected frame %s." % frame) self.logger.error("%s Unexpected frame %s.", self.ip, frame) else: # pragma: no cover # Unknown frames belong to extensions. Just drop it on the # floor, but log so that users know that something happened. self.logger.error("%s Received unknown frame, type %d", self.ip, frame.type) pass if 'END_HEADERS' in frame.flags: if self.response_headers is not None: raise ProtocolError("Too many header blocks.") # Begin by decoding the header block. If this fails, we need to # tear down the entire connection. if len(self.response_header_datas) == 1: header_data = self.response_header_datas[0] else: header_data = b''.join(self.response_header_datas) try: headers = self._decoder.decode(header_data) except Exception as e: self.logger.exception("decode h2 header %s fail:%r", header_data, e) raise e self.response_headers = HTTPHeaderMap(headers) # We've handled the headers, zero them out. self.response_header_datas = None self.get_head_time = time.time() length = self.response_headers.get("Content-Length", None) if isinstance(length, list): length = int(length[0]) if not self.task.finished: self.task.content_length = length self.task.set_state("h2_get_head") self.send_response() if 'END_STREAM' in frame.flags: #self.logger.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id) time_now = time.time() time_cost = time_now - self.get_head_time if time_cost > 0 and \ isinstance(self.task.content_length, int) and \ not self.task.finished: speed = self.task.content_length / time_cost self.task.set_state("h2_finish[SP:%d]" % speed) self._close_remote() self.close("end stream") if not self.task.finished: self.connection.continue_timeout = 0
def test_merge_self_is_no_op(self): h = HTTPHeaderMap([(b'hi', b'there')]) h.merge(h) assert h == HTTPHeaderMap([(b'hi', b'there')])
def test_empty_get(self): h = HTTPHeaderMap() assert h.get('nonexistent', 'hi there') == 'hi there'
class Stream(object): """ A single HTTP/2 stream. A stream is an independent, bi-directional sequence of HTTP headers and data. Each stream is identified by a single integer. From a HTTP perspective, a stream _approximately_ matches a single request-response pair. """ def __init__(self, connection, ip, stream_id, host, task, send_cb, close_cb, header_encoder, header_decoder, receive_window_manager, remote_window_size, max_frame_size): self.connection = connection self.ip = ip self.stream_id = stream_id self.host = host self.task = task self.state = STATE_IDLE # There are two flow control windows: one for data we're sending, # one for data being sent to us. self.receive_window_manager = receive_window_manager self.remote_window_size = remote_window_size self.max_frame_size = max_frame_size # This is the callback handed to the stream by its parent connection. # It is called when the stream wants to send data. It expects to # receive a list of frames that will be automatically serialized. self._send_cb = send_cb # This is the callback to be called when the stream is closed. self._close_cb = close_cb # A reference to the header encoder and decoder objects belonging to # the parent connection. self._encoder = header_encoder self._decoder = header_decoder self.request_headers = HTTPHeaderMap() # Convert the body to bytes if needed. self.request_body = to_bytestring(self.task.body) # request body not send blocked by send window # the left body will send when send window opened. self.request_body_left = len(self.request_body) # data list before decode self.response_header_datas = [] # Set to a key-value set of the response headers once their # HEADERS..CONTINUATION frame sequence finishes. self.response_headers = None # Unconsumed response data chunks self.response_body = [] self.response_body_len = 0 self.start_request() def start_request(self): """ Open the stream. Does this by encoding and sending the headers: no more calls to ``add_header`` are allowed after this method is called. The `end` flag controls whether this will be the end of the stream, or whether data will follow. """ # Strip any headers invalid in H2. #headers = h2_safe_headers(self.request_headers) self.add_header(":method", "POST") self.add_header(":scheme", "https") self.add_header(":authority", self.host) self.add_header(":path", "/_gh/") default_headers = (':method', ':scheme', ':authority', ':path') for name, value in self.task.headers.items(): is_default = to_native_string(name) in default_headers self.add_header(name, value, replace=is_default) # Encode the headers. encoded_headers = self._encoder.encode(self.request_headers) # It's possible that there is a substantial amount of data here. The # data needs to go into one HEADERS frame, followed by a number of # CONTINUATION frames. For now, for ease of implementation, let's just # assume that's never going to happen (16kB of headers is lots!). # Additionally, since this is so unlikely, there's no point writing a # test for this: it's just so simple. if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover raise ValueError("Header block too large.") header_frame = HeadersFrame(self.stream_id) header_frame.data = encoded_headers # If no data has been provided, this is the end of the stream. Either # way, due to the restriction above it's definitely the end of the # headers. header_frame.flags.add('END_HEADERS') # Send the header frame. self._send_cb(header_frame) # Transition the stream state appropriately. self.state = STATE_OPEN self.send_left_body() def add_header(self, name, value, replace=False): """ Adds a single HTTP header to the headers to be sent on the request. """ if not replace: self.request_headers[name] = value else: self.request_headers.replace(name, value) def send_left_body(self): while self.remote_window_size and self.request_body_left: send_size = min(self.remote_window_size, self.request_body_left, self.max_frame_size) f = DataFrame(self.stream_id) data_start = len(self.request_body) - self.request_body_left f.data = self.request_body[data_start:data_start+send_size] self.remote_window_size -= send_size self.request_body_left -= send_size # If the length of the data is less than MAX_CHUNK, we're probably # at the end of the file. If this is the end of the data, mark it # as END_STREAM. if self.request_body_left == 0: f.flags.add('END_STREAM') # Send the frame and decrement the flow control window. self._send_cb(f) # If no more data is to be sent on this stream, transition our state. if self.request_body_left == 0: self._close_local() def receive_frame(self, frame): """ Handle a frame received on this stream. called by connection. """ if frame.type == WindowUpdateFrame.type: self.remote_window_size += frame.window_increment self.send_left_body() elif frame.type == HeadersFrame.type: # Begin the header block for the response headers. self.response_header_datas = [frame.data] elif frame.type == PushPromiseFrame.type: xlog.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id) elif frame.type == ContinuationFrame.type: # Continue a header block begun with either HEADERS or PUSH_PROMISE. self.response_header_datas.append(frame.data) elif frame.type == DataFrame.type: # Append the data to the buffer. self.response_body.append(frame.data) self.response_body_len += len(frame.data) if 'END_STREAM' not in frame.flags: # Increase the window size. Only do this if the data frame contains # actual data. # don't do it if stream is closed. size = frame.flow_controlled_length increment = self.receive_window_manager._handle_frame(size) #if increment: # xlog.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment) #content_len = int(self.request_headers.get("Content-Length")[0]) #xlog.debug("%s get:%d s:%d", self.ip, self.response_body_len, size) if increment and not self._remote_closed: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == BlockedFrame.type: # If we've been blocked we may want to fixup the window. increment = self.receive_window_manager._blocked() if increment: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == RstStreamFrame.type: xlog.warn("%s Stream %d forcefully closed.", self.ip, self.stream_id) self.close("RESET") elif frame.type in FRAMES: # This frame isn't valid at this point. #raise ValueError("Unexpected frame %s." % frame) xlog.error("%s Unexpected frame %s.", self.ip, frame) else: # pragma: no cover # Unknown frames belong to extensions. Just drop it on the # floor, but log so that users know that something happened. xlog.error("%s Received unknown frame, type %d", self.ip, frame.type) pass if 'END_HEADERS' in frame.flags: # Begin by decoding the header block. If this fails, we need to # tear down the entire connection. TODO: actually do that. headers = self._decoder.decode(b''.join(self.response_header_datas)) self._handle_header_block(headers) # We've handled the headers, zero them out. self.response_header_datas = None if 'END_STREAM' in frame.flags: #xlog.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id) self._close_remote() self.send_response() self.close("end stream") def send_response(self): status = int(self.response_headers[b':status'][0]) strip_headers(self.response_headers) body = b''.join(self.response_body) response = BaseResponse(status=status, headers=self.response_headers, body=body) response.ssl_sock = self.connection.ssl_sock response.worker = self.connection self.task.queue.put(response) def close(self, reason=""): self._close_cb(self.stream_id, reason) def _handle_header_block(self, headers): """ Handles the logic for receiving a completed headers block. A headers block is an uninterrupted sequence of one HEADERS frame followed by zero or more CONTINUATION frames, and is terminated by a frame bearing the END_HEADERS flag. HTTP/2 allows receipt of up to three such blocks on a stream. The first is optional, and contains a 1XX response. The second is mandatory, and must contain a final response (200 or higher). The third is optional, and may contain 'trailers', headers that are sent after a chunk-encoded body is sent. Here we only process the simple state: no push, one header frame. """ if self.response_headers is None: self.response_headers = HTTPHeaderMap(headers) else: # Received too many headers blocks. raise ProtocolError("Too many header blocks.") return @property def _local_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL) @property def _remote_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE) @property def _local_open(self): return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE) def _close_local(self): self.state = ( STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN else STATE_CLOSED ) def _close_remote(self): self.state = ( STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN else STATE_CLOSED )