def test_read_headers_out_of_order(self):
# If header blocks aren't decoded in the same order they're received,
# regardless of the stream they belong to, the decoder state will
# become corrupted.
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h1.flags |= set(['END_HEADERS', 'END_STREAM'])
h3 = HeadersFrame(3)
h3.data = e.encode([(':status', 200), ('content-type', 'baz/qux')])
h3.flags |= set(['END_HEADERS', 'END_STREAM'])
sock = DummySocket()
sock.buffer = BytesIO(h1.serialize() + h3.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
r1 = c.request('GET', '/a')
r3 = c.request('GET', '/b')
assert c.get_response(r3).headers == HTTPHeaderMap(
[('content-type', 'baz/qux')]
)
assert c.get_response(r1).headers == HTTPHeaderMap(
[('content-type', 'foo/bar')]
)
def test_response_ignores_unknown_headers(self):
headers = HTTPHeaderMap([(':status', '200'), (':reserved', 'yes'),
('no', 'no')])
stream = DummyStream(b'')
resp = HTTP20Response(headers, stream)
assert resp.headers == HTTPHeaderMap([('no', 'no')])
def test_actual_get(self):
h = HTTPHeaderMap()
h['k1'] = 'v1, v2'
h['k2'] = 'v3'
h['k1'] = 'v4'
assert h.get('k1') == [b'v1', b'v2', b'v4']
def test_can_receive_trailers(self):
headers = [('a', 'b'), ('c', 'd'), (':status', '200')]
trailers = [('e', 'f'), ('g', 'h')]
s = Stream(1, None, None, None, None, FixedDecoder(headers), None)
s.state = STATE_HALF_CLOSED_LOCAL
# Provide the first HEADERS frame.
f = HeadersFrame(1)
f.data = b'hi there!'
f.flags.add('END_HEADERS')
s.receive_frame(f)
assert s.response_headers == HTTPHeaderMap(headers)
# Now, replace the dummy decoder to ensure we get a new header block.
s._decoder = FixedDecoder(trailers)
# Provide the trailers.
f = HeadersFrame(1)
f.data = b'hi there again!'
f.flags.add('END_STREAM')
f.flags.add('END_HEADERS')
s.receive_frame(f)
# Now, check the trailers.
assert s.response_trailers == HTTPHeaderMap(trailers)
# Confirm we closed the stream.
assert s.state == STATE_CLOSED
def test_keys(self):
h = HTTPHeaderMap()
h['k1'] = 'v1, v2'
h['k2'] = 'v3'
h['k1'] = 'v4'
assert len(list(h.keys())) == 4
assert list(h.keys()) == [b'k1', b'k1', b'k2', b'k1']
def test_values(self):
h = HTTPHeaderMap()
h['k1'] = 'v1, v2'
h['k2'] = 'v3'
h['k1'] = 'v4'
assert len(list(h.values())) == 4
assert list(h.values()) == [b'v1', b'v2', b'v3', b'v4']
def test_raw_iteration(self):
items = [
(b'k1', b'v2'),
(b'k2', b'v2, v3, v4'),
(b'k2', b'v3'),
]
h = HTTPHeaderMap(items)
assert list(h.iter_raw()) == items
def test_trailers_are_read(self):
headers = HTTPHeaderMap([(':status', '200')])
trailers = HTTPHeaderMap([('a', 'b'), ('c', 'd')])
stream = DummyStream(b'', trailers=trailers)
resp = HTTP20Response(headers, stream)
assert resp.trailers == trailers
assert resp.trailers['a'] == [b'b']
assert resp.trailers['c'] == [b'd']
def test_headers_must_be_strings(self):
with pytest.raises(ValueError):
HTTPHeaderMap(key=1)
h = HTTPHeaderMap()
with pytest.raises(ValueError):
h['k'] = 1
with pytest.raises(ValueError):
h[1] = 'v'
def test_merge_header_map_dict(self):
h = HTTPHeaderMap([(b'hi', b'there')])
d = {'cat': 'dog'}
h.merge(d)
assert list(h.items()) == [
(b'hi', b'there'),
(b'cat', b'dog'),
]
def __init__(self, connection, ip, stream_id, host, task, send_cb,
close_cb, header_encoder, header_decoder,
receive_window_manager, remote_window_size, max_frame_size):
self.connection = connection
self.ip = ip
self.stream_id = stream_id
self.host = host
self.task = task
self.state = STATE_IDLE
self.get_head_time = None
# There are two flow control windows: one for data we're sending,
# one for data being sent to us.
self.receive_window_manager = receive_window_manager
self.remote_window_size = remote_window_size
self.max_frame_size = max_frame_size
# This is the callback handed to the stream by its parent connection.
# It is called when the stream wants to send data. It expects to
# receive a list of frames that will be automatically serialized.
self._send_cb = send_cb
# This is the callback to be called when the stream is closed.
self._close_cb = close_cb
# A reference to the header encoder and decoder objects belonging to
# the parent connection.
self._encoder = header_encoder
self._decoder = header_decoder
self.request_headers = HTTPHeaderMap()
# Convert the body to bytes if needed.
self.request_body = to_bytestring(self.task.body)
# request body not send blocked by send window
# the left body will send when send window opened.
self.request_body_left = len(self.request_body)
self.request_body_sended = False
# data list before decode
self.response_header_datas = []
# Set to a key-value set of the response headers once their
# HEADERS..CONTINUATION frame sequence finishes.
self.response_headers = None
# Unconsumed response data chunks
self.response_body = []
self.response_body_len = 0
threading.Thread(target=self.timeout_response).start()
self.start_request()
def test_pushed_requests_ignore_unexpected_headers(self):
headers = HTTPHeaderMap([
(':scheme', 'http'),
(':method', 'get'),
(':authority', 'google.com'),
(':path', '/'),
(':reserved', 'no'),
('no', 'no'),
])
p = HTTP20Push(headers, DummyStream(b''))
assert p.request_headers == HTTPHeaderMap([('no', 'no')])
def test_equality(self):
h1 = HTTPHeaderMap()
h1['k1'] = 'v1, v2'
h1['k2'] = 'v3'
h1['k1'] = 'v4'
h2 = HTTPHeaderMap()
h2['k1'] = 'v1, v2'
h2['k2'] = 'v3'
h2['k1'] = 'v4'
assert h1 == h2
def test_items(self):
h = HTTPHeaderMap()
items = [
(b'k1', b'v2'),
(b'k2', b'v2'),
(b'k2', b'v3'),
]
for k, v in items:
h[k] = v
for i, pair in enumerate(h.items()):
assert items[i] == pair
def test_inequality_of_raw_ordering(self):
h1 = HTTPHeaderMap()
h1['k1'] = 'v1, v2'
h1['k2'] = 'v3'
h1['k1'] = 'v4'
h2 = HTTPHeaderMap()
h2['k1'] = 'v1'
h2['k1'] = 'v2'
h2['k2'] = 'v3'
h2['k1'] = 'v4'
assert h1 != h2
def h2_headers(self, node=None, secret=None, multiplexed=None, af=None):
headers = HTTPHeaderMap()
if node:
if isinstance(node, (tuple, list, set)):
for n in node:
headers.update({Headers.node: n})
else:
headers.update({Headers.node: node})
if secret and af != socket.AF_UNIX:
headers.update({Headers.secret: secret})
if multiplexed:
headers.update({Headers.multiplexed: "true"})
return headers
def test_merge_headermaps_preserves_raw(self):
h1 = HTTPHeaderMap([
(b'hi', b'there')
])
h2 = HTTPHeaderMap([
(b'Hi', b'there, sir, maam')
])
h1.merge(h2)
assert list(h1.iter_raw()) == [
(b'hi', b'there'),
(b'Hi', b'there, sir, maam'),
]
def action(self, nodename, thr=None, stream_id=None, **kwargs):
logfile = os.path.join(rcEnv.paths.pathlog, "node.log")
ofile = thr._action_logs_open(logfile, 0, "node")
request_headers = HTTPHeaderMap(
thr.streams[stream_id]["request"].headers)
try:
content_type = bdecode(request_headers.get("accept").pop())
except:
content_type = "application/json"
thr.streams[stream_id]["content_type"] = content_type
thr.streams[stream_id]["pushers"].append({
"o": self,
"fn": "h2_push_logs",
"args": [ofile, True],
})
def test_getheader(self):
headers = HTTPHeaderMap([(':status', '200'),
('content-type', 'application/json')])
stream = DummyStream(b'')
resp = HTTP20Response(headers, stream)
assert resp.headers[b'content-type'] == [b'application/json']
def test_response_calls_stream_close(self):
headers = HTTPHeaderMap([(':status', '200')])
stream = DummyStream('')
resp = HTTP20Response(headers, stream)
resp.close()
assert stream.closed
def _handle_header_block(self, headers):
"""
Handles the logic for receiving a completed headers block.
A headers block is an uninterrupted sequence of one HEADERS frame
followed by zero or more CONTINUATION frames, and is terminated by a
frame bearing the END_HEADERS flag.
HTTP/2 allows receipt of up to three such blocks on a stream. The first
is optional, and contains a 1XX response. The second is mandatory, and
must contain a final response (200 or higher). The third is optional,
and may contain 'trailers', headers that are sent after a chunk-encoded
body is sent.
Here we only process the simple state: no push, one header frame.
"""
if self.response_headers is None:
self.response_headers = HTTPHeaderMap(headers)
else:
# Received too many headers blocks.
raise ProtocolError("Too many header blocks.")
status = int(self.response_headers[b':status'][0])
if status in [400, 403]:
# xlog.warn("status:%d host:%s", status, self.host)
self.connection.close("get 40x")
return
class DummyResponse(object):
def __init__(self, headers):
self.headers = HTTPHeaderMap(headers.items())
def read(self):
ctype = self.headers.get('content-type')
if ctype is not None:
if 'json' in ctype[0].decode('utf-8'):
return b'{"data": "dummy"}'
return b'<html>dummy</html>'
def getheader(self, name):
return self.headers.get(name)
def getheaders(self):
return self.headers
def test_containment(self):
h = HTTPHeaderMap()
h['key'] = 'val'
assert 'key' in h
assert b'key' in h
assert 'nonkey' not in h
def test_header_map_can_delete_value(self):
h = HTTPHeaderMap()
h['key'] = b'v1'
del h[b'key']
with pytest.raises(KeyError):
h[b'key']
class DummyResponse(object):
def __init__(self, headers):
self.headers = HTTPHeaderMap(headers.items())
def read(self):
ctype = self.headers.get('content-type')
if ctype is not None:
if 'json' in ctype[0].decode('utf-8'):
return b'{"data": "dummy"}'
return b'<html>dummy</html>'
def getheader(self, name):
return self.headers.get(name)
def getheaders(self):
return self.headers
def test_length_counts_lines_separately(self):
h = HTTPHeaderMap()
h['k1'] = 'v1, v2'
h['k2'] = 'v3'
h['k1'] = 'v4'
assert len(h) == 4
def __init__(self,
ip,
stream_id,
host,
task,
send_cb,
close_cb,
header_encoder,
header_decoder,
receive_window_manager,
remote_window_size,
max_frame_size):
self.ip = ip
self.stream_id = stream_id
self.host = host
self.task = task
self.state = STATE_IDLE
# There are two flow control windows: one for data we're sending,
# one for data being sent to us.
self.receive_window_manager = receive_window_manager
self.remote_window_size = remote_window_size
self.max_frame_size = max_frame_size
# This is the callback handed to the stream by its parent connection.
# It is called when the stream wants to send data. It expects to
# receive a list of frames that will be automatically serialized.
self._send_cb = send_cb
# This is the callback to be called when the stream is closed.
self._close_cb = close_cb
# A reference to the header encoder and decoder objects belonging to
# the parent connection.
self._encoder = header_encoder
self._decoder = header_decoder
self.request_headers = HTTPHeaderMap()
# Convert the body to bytes if needed.
self.request_body = to_bytestring(self.task.body)
# request body not send blocked by send window
# the left body will send when send window opened.
self.request_body_left = len(self.request_body)
# data list before decode
self.response_header_datas = []
# Set to a key-value set of the response headers once their
# HEADERS..CONTINUATION frame sequence finishes.
self.response_headers = None
# Unconsumed response data chunks
self.response_body = []
self.response_body_len = 0
self.start_request()
def test_header_map_deletes_all_values(self):
h = HTTPHeaderMap()
h['key'] = 'v1'
h['key'] = 'v2'
del h['key']
with pytest.raises(KeyError):
h['key']
def test_responses_are_context_managers(self):
headers = HTTPHeaderMap([(':status', '200')])
stream = DummyStream('')
with HTTP20Response(headers, stream):
pass
assert stream.closed
def action(self, nodename, thr=None, stream_id=None, **kwargs):
options = self.parse_options(kwargs)
thr.selector = options.selector
if not thr.event_queue:
thr.event_queue = queue.Queue()
if options.full:
data = thr.daemon_status()
namespaces = thr.get_namespaces()
fevent = {
"nodename":
rcEnv.nodename,
"ts":
time.time(),
"kind":
"full",
"data":
thr.filter_daemon_status(data,
namespaces=namespaces,
selector=options.selector),
}
if thr.h2conn:
_msg = fevent
elif thr.encrypted:
_msg = thr.encrypt(fevent)
else:
_msg = thr.msg_encode(fevent)
thr.event_queue.put(_msg)
if not thr in thr.parent.events_clients:
thr.parent.events_clients.append(thr)
if not stream_id in thr.events_stream_ids:
thr.events_stream_ids.append(stream_id)
if thr.h2conn:
request_headers = HTTPHeaderMap(
thr.streams[stream_id]["request"].headers)
try:
content_type = bdecode(request_headers.get("accept").pop())
except:
content_type = "application/json"
thr.streams[stream_id]["content_type"] = content_type
thr.streams[stream_id]["pushers"].append({
"fn":
"h2_push_action_events",
})
else:
thr.raw_push_action_events()
def test_request_with_unicodestring_body(self):
c = HTTP11Connection('httpbin.org')
c._sock = DummySocket()
with pytest.raises(ValueError):
c.request('POST',
'/post',
headers=HTTPHeaderMap([('User-Agent', 'hyper')]),
body=u'hi')
def test_can_create_from_multiple_iterables(self):
items = [
(b'k1', b'v2'),
(b'k2', b'v2'),
(b'k2', b'v3'),
]
h = HTTPHeaderMap(items, items, items)
assert list(h) == items + items + items
def __init__(self, attrs):
self.body = {}
self.headers = HTTPHeaderMap()
self.items = []
self.method = None
self._url = ''
self.url = DummyUrlInfo()
for key, value in attrs.items():
setattr(self, key, value)
def test_replacing(self):
h = HTTPHeaderMap([
(b'name', b'value'),
(b'name2', b'value2'),
(b'name2', b'value2'),
(b'name3', b'value3'),
])
h.replace('name2', '42')
h.replace('name4', 'other_value')
assert list(h.items()) == [
(b'name', b'value'),
(b'name2', b'42'),
(b'name3', b'value3'),
(b'name4', b'other_value'),
]
def __init__(self, headers):
self.headers = HTTPHeaderMap(headers.items())
def test_doesnt_split_set_cookie(self):
h = HTTPHeaderMap()
h['Set-Cookie'] = 'v1, v2'
assert h['set-cookie'] == [b'v1, v2']
assert h.get(b'set-cookie') == [b'v1, v2']
class Stream(object):
"""
A single HTTP/2 stream.
A stream is an independent, bi-directional sequence of HTTP headers and
data. Each stream is identified by a single integer. From a HTTP
perspective, a stream _approximately_ matches a single request-response
pair.
"""
def __init__(self,
logger,
config,
connection,
ip,
stream_id,
task,
send_cb,
close_cb,
encoder,
decoder,
receive_window_manager,
remote_window_size,
max_frame_size):
self.logger = logger
self.config = config
self.connection = connection
self.ip = ip
self.stream_id = stream_id
self.task = task
self.state = STATE_IDLE
self.get_head_time = None
# There are two flow control windows: one for data we're sending,
# one for data being sent to us.
self.receive_window_manager = receive_window_manager
self.remote_window_size = remote_window_size
self.max_frame_size = max_frame_size
# This is the callback handed to the stream by its parent connection.
# It is called when the stream wants to send data. It expects to
# receive a list of frames that will be automatically serialized.
self._send_cb = send_cb
# This is the callback to be called when the stream is closed.
self._close_cb = close_cb
# A reference to the header encoder and decoder objects belonging to
# the parent connection.
self._encoder = encoder
self._decoder = decoder
self.request_headers = HTTPHeaderMap()
# Convert the body to bytes if needed.
self.request_body = to_bytestring(self.task.body)
# request body not send blocked by send window
# the left body will send when send window opened.
self.request_body_left = len(self.request_body)
self.request_body_sended = False
# data list before decode
self.response_header_datas = []
# Set to a key-value set of the response headers once their
# HEADERS..CONTINUATION frame sequence finishes.
self.response_headers = None
# Unconsumed response data chunks
self.response_body = []
self.response_body_len = 0
def start_request(self):
"""
Open the stream. Does this by encoding and sending the headers: no more
calls to ``add_header`` are allowed after this method is called.
The `end` flag controls whether this will be the end of the stream, or
whether data will follow.
"""
# Strip any headers invalid in H2.
#headers = h2_safe_headers(self.request_headers)
host = self.connection.get_host(self.task.host)
self.add_header(":method", self.task.method)
self.add_header(":scheme", "https")
self.add_header(":authority", host)
self.add_header(":path", self.task.path)
default_headers = (':method', ':scheme', ':authority', ':path')
#headers = h2_safe_headers(self.task.headers)
for name, value in self.task.headers.items():
is_default = to_native_string(name) in default_headers
self.add_header(name, value, replace=is_default)
# Encode the headers.
encoded_headers = self._encoder(self.request_headers)
# It's possible that there is a substantial amount of data here. The
# data needs to go into one HEADERS frame, followed by a number of
# CONTINUATION frames. For now, for ease of implementation, let's just
# assume that's never going to happen (16kB of headers is lots!).
# Additionally, since this is so unlikely, there's no point writing a
# test for this: it's just so simple.
if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover
raise ValueError("Header block too large.")
header_frame = HeadersFrame(self.stream_id)
header_frame.data = encoded_headers
# If no data has been provided, this is the end of the stream. Either
# way, due to the restriction above it's definitely the end of the
# headers.
header_frame.flags.add('END_HEADERS')
if self.request_body_left == 0:
header_frame.flags.add('END_STREAM')
# Send the header frame.
self.task.set_state("start send header")
self._send_cb(header_frame)
# Transition the stream state appropriately.
self.state = STATE_OPEN
self.task.set_state("start send left body")
threading.Thread(target=self.left_work).start()
def left_work(self):
if self.request_body_left > 0:
self.send_left_body()
self.timeout_response()
def add_header(self, name, value, replace=False):
"""
Adds a single HTTP header to the headers to be sent on the request.
"""
if not replace:
self.request_headers[name] = value
else:
self.request_headers.replace(name, value)
def send_left_body(self):
while self.remote_window_size and not self.request_body_sended:
send_size = min(self.remote_window_size, self.request_body_left, self.max_frame_size)
f = DataFrame(self.stream_id)
data_start = len(self.request_body) - self.request_body_left
f.data = self.request_body[data_start:data_start+send_size]
self.remote_window_size -= send_size
self.request_body_left -= send_size
# If the length of the data is less than MAX_CHUNK, we're probably
# at the end of the file. If this is the end of the data, mark it
# as END_STREAM.
if self.request_body_left == 0:
f.flags.add('END_STREAM')
# Send the frame and decrement the flow control window.
self._send_cb(f)
# If no more data is to be sent on this stream, transition our state.
if self.request_body_left == 0:
self.request_body_sended = True
self._close_local()
self.task.set_state("end send left body")
def receive_frame(self, frame):
"""
Handle a frame received on this stream.
called by connection.
"""
# self.logger.debug("stream %d recved frame %r", self.stream_id, frame)
if frame.type == WindowUpdateFrame.type:
self.remote_window_size += frame.window_increment
self.send_left_body()
elif frame.type == HeadersFrame.type:
# Begin the header block for the response headers.
#self.response_header_datas = [frame.data]
self.response_header_datas.append(frame.data)
elif frame.type == PushPromiseFrame.type:
self.logger.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id)
elif frame.type == ContinuationFrame.type:
# Continue a header block begun with either HEADERS or PUSH_PROMISE.
self.response_header_datas.append(frame.data)
elif frame.type == DataFrame.type:
# Append the data to the buffer.
if not self.task.finished:
self.task.put_data(frame.data)
if 'END_STREAM' not in frame.flags:
# Increase the window size. Only do this if the data frame contains
# actual data.
# don't do it if stream is closed.
size = frame.flow_controlled_length
increment = self.receive_window_manager._handle_frame(size)
#if increment:
# self.logger.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment)
#content_len = int(self.request_headers.get("Content-Length")[0])
#self.logger.debug("%s get:%d s:%d", self.ip, self.response_body_len, size)
if increment and not self._remote_closed:
w = WindowUpdateFrame(self.stream_id)
w.window_increment = increment
self._send_cb(w)
elif frame.type == BlockedFrame.type:
# If we've been blocked we may want to fixup the window.
increment = self.receive_window_manager._blocked()
if increment:
w = WindowUpdateFrame(self.stream_id)
w.window_increment = increment
self._send_cb(w)
elif frame.type == RstStreamFrame.type:
# Rest Frame send from server is not define in RFC
inactive_time = time.time() - self.connection.last_active_time
self.logger.debug("%s Stream %d Rest by server, inactive:%d. error code:%d",
self.ip, self.stream_id, inactive_time, frame.error_code)
self.connection.close("RESET")
elif frame.type in FRAMES:
# This frame isn't valid at this point.
#raise ValueError("Unexpected frame %s." % frame)
self.logger.error("%s Unexpected frame %s.", self.ip, frame)
else: # pragma: no cover
# Unknown frames belong to extensions. Just drop it on the
# floor, but log so that users know that something happened.
self.logger.error("%s Received unknown frame, type %d", self.ip, frame.type)
pass
if 'END_HEADERS' in frame.flags:
if self.response_headers is not None:
raise ProtocolError("Too many header blocks.")
# Begin by decoding the header block. If this fails, we need to
# tear down the entire connection.
if len(self.response_header_datas) == 1:
header_data = self.response_header_datas[0]
else:
header_data = b''.join(self.response_header_datas)
try:
headers = self._decoder.decode(header_data)
except Exception as e:
self.logger.exception("decode h2 header %s fail:%r", header_data, e)
raise e
self.response_headers = HTTPHeaderMap(headers)
# We've handled the headers, zero them out.
self.response_header_datas = None
self.get_head_time = time.time()
length = self.response_headers.get("Content-Length", None)
if isinstance(length, list):
length = int(length[0])
if not self.task.finished:
self.task.content_length = length
self.task.set_state("h2_get_head")
self.send_response()
if 'END_STREAM' in frame.flags:
#self.logger.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id)
time_now = time.time()
time_cost = time_now - self.get_head_time
if time_cost > 0 and \
isinstance(self.task.content_length, int) and \
not self.task.finished:
speed = self.task.content_length / time_cost
self.task.set_state("h2_finish[SP:%d]" % speed)
self._close_remote()
self.close("end stream")
if not self.task.finished:
self.connection.continue_timeout = 0
def send_response(self):
if self.task.responsed:
self.logger.error("http2_stream send_response but responsed.%s", self.task.url)
self.close("h2 stream send_response but sended.")
return
self.task.responsed = True
status = int(self.response_headers[b':status'][0])
strip_headers(self.response_headers)
response = simple_http_client.BaseResponse(status=status, headers=self.response_headers)
response.ssl_sock = self.connection.ssl_sock
response.worker = self.connection
response.task = self.task
self.task.queue.put(response)
if status in self.config.http2_status_to_close:
self.connection.close("status %d" % status)
def close(self, reason="close"):
if not self.task.responsed:
self.connection.retry_task_cb(self.task, reason)
else:
self.task.finish()
# empty block means fail or closed.
self._close_remote()
self._close_cb(self.stream_id, reason)
@property
def _local_closed(self):
return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL)
@property
def _remote_closed(self):
return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE)
@property
def _local_open(self):
return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE)
def _close_local(self):
self.state = (
STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN
else STATE_CLOSED
)
def _close_remote(self):
self.state = (
STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN
else STATE_CLOSED
)
def timeout_response(self):
start_time = time.time()
while time.time() - start_time < self.task.timeout:
time.sleep(1)
if self._remote_closed:
return
self.logger.warn("h2 timeout %s task_trace:%s worker_trace:%s",
self.connection.ssl_sock.ip,
self.task.get_trace(),
self.connection.get_trace())
self.task.set_state("timeout")
if self.task.responsed:
self.task.finish()
else:
self.task.response_fail("timeout")
self.connection.continue_timeout += 1
if self.connection.continue_timeout >= self.connection.config.http2_max_timeout_tasks and \
time.time() - self.connection.last_active_time > self.connection.config.http2_timeout_active:
self.connection.close("down fail")
def receive_frame(self, frame):
"""
Handle a frame received on this stream.
called by connection.
"""
# self.logger.debug("stream %d recved frame %r", self.stream_id, frame)
if frame.type == WindowUpdateFrame.type:
self.remote_window_size += frame.window_increment
self.send_left_body()
elif frame.type == HeadersFrame.type:
# Begin the header block for the response headers.
#self.response_header_datas = [frame.data]
self.response_header_datas.append(frame.data)
elif frame.type == PushPromiseFrame.type:
self.logger.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id)
elif frame.type == ContinuationFrame.type:
# Continue a header block begun with either HEADERS or PUSH_PROMISE.
self.response_header_datas.append(frame.data)
elif frame.type == DataFrame.type:
# Append the data to the buffer.
if not self.task.finished:
self.task.put_data(frame.data)
if 'END_STREAM' not in frame.flags:
# Increase the window size. Only do this if the data frame contains
# actual data.
# don't do it if stream is closed.
size = frame.flow_controlled_length
increment = self.receive_window_manager._handle_frame(size)
#if increment:
# self.logger.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment)
#content_len = int(self.request_headers.get("Content-Length")[0])
#self.logger.debug("%s get:%d s:%d", self.ip, self.response_body_len, size)
if increment and not self._remote_closed:
w = WindowUpdateFrame(self.stream_id)
w.window_increment = increment
self._send_cb(w)
elif frame.type == BlockedFrame.type:
# If we've been blocked we may want to fixup the window.
increment = self.receive_window_manager._blocked()
if increment:
w = WindowUpdateFrame(self.stream_id)
w.window_increment = increment
self._send_cb(w)
elif frame.type == RstStreamFrame.type:
# Rest Frame send from server is not define in RFC
inactive_time = time.time() - self.connection.last_active_time
self.logger.debug("%s Stream %d Rest by server, inactive:%d. error code:%d",
self.ip, self.stream_id, inactive_time, frame.error_code)
self.connection.close("RESET")
elif frame.type in FRAMES:
# This frame isn't valid at this point.
#raise ValueError("Unexpected frame %s." % frame)
self.logger.error("%s Unexpected frame %s.", self.ip, frame)
else: # pragma: no cover
# Unknown frames belong to extensions. Just drop it on the
# floor, but log so that users know that something happened.
self.logger.error("%s Received unknown frame, type %d", self.ip, frame.type)
pass
if 'END_HEADERS' in frame.flags:
if self.response_headers is not None:
raise ProtocolError("Too many header blocks.")
# Begin by decoding the header block. If this fails, we need to
# tear down the entire connection.
if len(self.response_header_datas) == 1:
header_data = self.response_header_datas[0]
else:
header_data = b''.join(self.response_header_datas)
try:
headers = self._decoder.decode(header_data)
except Exception as e:
self.logger.exception("decode h2 header %s fail:%r", header_data, e)
raise e
self.response_headers = HTTPHeaderMap(headers)
# We've handled the headers, zero them out.
self.response_header_datas = None
self.get_head_time = time.time()
length = self.response_headers.get("Content-Length", None)
if isinstance(length, list):
length = int(length[0])
if not self.task.finished:
self.task.content_length = length
self.task.set_state("h2_get_head")
self.send_response()
if 'END_STREAM' in frame.flags:
#self.logger.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id)
time_now = time.time()
time_cost = time_now - self.get_head_time
if time_cost > 0 and \
isinstance(self.task.content_length, int) and \
not self.task.finished:
speed = self.task.content_length / time_cost
self.task.set_state("h2_finish[SP:%d]" % speed)
self._close_remote()
self.close("end stream")
if not self.task.finished:
self.connection.continue_timeout = 0
def test_merge_self_is_no_op(self):
h = HTTPHeaderMap([(b'hi', b'there')])
h.merge(h)
assert h == HTTPHeaderMap([(b'hi', b'there')])
def test_empty_get(self):
h = HTTPHeaderMap()
assert h.get('nonexistent', 'hi there') == 'hi there'
class Stream(object):
"""
A single HTTP/2 stream.
A stream is an independent, bi-directional sequence of HTTP headers and
data. Each stream is identified by a single integer. From a HTTP
perspective, a stream _approximately_ matches a single request-response
pair.
"""
def __init__(self,
connection,
ip,
stream_id,
host,
task,
send_cb,
close_cb,
header_encoder,
header_decoder,
receive_window_manager,
remote_window_size,
max_frame_size):
self.connection = connection
self.ip = ip
self.stream_id = stream_id
self.host = host
self.task = task
self.state = STATE_IDLE
# There are two flow control windows: one for data we're sending,
# one for data being sent to us.
self.receive_window_manager = receive_window_manager
self.remote_window_size = remote_window_size
self.max_frame_size = max_frame_size
# This is the callback handed to the stream by its parent connection.
# It is called when the stream wants to send data. It expects to
# receive a list of frames that will be automatically serialized.
self._send_cb = send_cb
# This is the callback to be called when the stream is closed.
self._close_cb = close_cb
# A reference to the header encoder and decoder objects belonging to
# the parent connection.
self._encoder = header_encoder
self._decoder = header_decoder
self.request_headers = HTTPHeaderMap()
# Convert the body to bytes if needed.
self.request_body = to_bytestring(self.task.body)
# request body not send blocked by send window
# the left body will send when send window opened.
self.request_body_left = len(self.request_body)
# data list before decode
self.response_header_datas = []
# Set to a key-value set of the response headers once their
# HEADERS..CONTINUATION frame sequence finishes.
self.response_headers = None
# Unconsumed response data chunks
self.response_body = []
self.response_body_len = 0
self.start_request()
def start_request(self):
"""
Open the stream. Does this by encoding and sending the headers: no more
calls to ``add_header`` are allowed after this method is called.
The `end` flag controls whether this will be the end of the stream, or
whether data will follow.
"""
# Strip any headers invalid in H2.
#headers = h2_safe_headers(self.request_headers)
self.add_header(":method", "POST")
self.add_header(":scheme", "https")
self.add_header(":authority", self.host)
self.add_header(":path", "/_gh/")
default_headers = (':method', ':scheme', ':authority', ':path')
for name, value in self.task.headers.items():
is_default = to_native_string(name) in default_headers
self.add_header(name, value, replace=is_default)
# Encode the headers.
encoded_headers = self._encoder.encode(self.request_headers)
# It's possible that there is a substantial amount of data here. The
# data needs to go into one HEADERS frame, followed by a number of
# CONTINUATION frames. For now, for ease of implementation, let's just
# assume that's never going to happen (16kB of headers is lots!).
# Additionally, since this is so unlikely, there's no point writing a
# test for this: it's just so simple.
if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover
raise ValueError("Header block too large.")
header_frame = HeadersFrame(self.stream_id)
header_frame.data = encoded_headers
# If no data has been provided, this is the end of the stream. Either
# way, due to the restriction above it's definitely the end of the
# headers.
header_frame.flags.add('END_HEADERS')
# Send the header frame.
self._send_cb(header_frame)
# Transition the stream state appropriately.
self.state = STATE_OPEN
self.send_left_body()
def add_header(self, name, value, replace=False):
"""
Adds a single HTTP header to the headers to be sent on the request.
"""
if not replace:
self.request_headers[name] = value
else:
self.request_headers.replace(name, value)
def send_left_body(self):
while self.remote_window_size and self.request_body_left:
send_size = min(self.remote_window_size, self.request_body_left, self.max_frame_size)
f = DataFrame(self.stream_id)
data_start = len(self.request_body) - self.request_body_left
f.data = self.request_body[data_start:data_start+send_size]
self.remote_window_size -= send_size
self.request_body_left -= send_size
# If the length of the data is less than MAX_CHUNK, we're probably
# at the end of the file. If this is the end of the data, mark it
# as END_STREAM.
if self.request_body_left == 0:
f.flags.add('END_STREAM')
# Send the frame and decrement the flow control window.
self._send_cb(f)
# If no more data is to be sent on this stream, transition our state.
if self.request_body_left == 0:
self._close_local()
def receive_frame(self, frame):
"""
Handle a frame received on this stream.
called by connection.
"""
if frame.type == WindowUpdateFrame.type:
self.remote_window_size += frame.window_increment
self.send_left_body()
elif frame.type == HeadersFrame.type:
# Begin the header block for the response headers.
self.response_header_datas = [frame.data]
elif frame.type == PushPromiseFrame.type:
xlog.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id)
elif frame.type == ContinuationFrame.type:
# Continue a header block begun with either HEADERS or PUSH_PROMISE.
self.response_header_datas.append(frame.data)
elif frame.type == DataFrame.type:
# Append the data to the buffer.
self.response_body.append(frame.data)
self.response_body_len += len(frame.data)
if 'END_STREAM' not in frame.flags:
# Increase the window size. Only do this if the data frame contains
# actual data.
# don't do it if stream is closed.
size = frame.flow_controlled_length
increment = self.receive_window_manager._handle_frame(size)
#if increment:
# xlog.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment)
#content_len = int(self.request_headers.get("Content-Length")[0])
#xlog.debug("%s get:%d s:%d", self.ip, self.response_body_len, size)
if increment and not self._remote_closed:
w = WindowUpdateFrame(self.stream_id)
w.window_increment = increment
self._send_cb(w)
elif frame.type == BlockedFrame.type:
# If we've been blocked we may want to fixup the window.
increment = self.receive_window_manager._blocked()
if increment:
w = WindowUpdateFrame(self.stream_id)
w.window_increment = increment
self._send_cb(w)
elif frame.type == RstStreamFrame.type:
xlog.warn("%s Stream %d forcefully closed.", self.ip, self.stream_id)
self.close("RESET")
elif frame.type in FRAMES:
# This frame isn't valid at this point.
#raise ValueError("Unexpected frame %s." % frame)
xlog.error("%s Unexpected frame %s.", self.ip, frame)
else: # pragma: no cover
# Unknown frames belong to extensions. Just drop it on the
# floor, but log so that users know that something happened.
xlog.error("%s Received unknown frame, type %d", self.ip, frame.type)
pass
if 'END_HEADERS' in frame.flags:
# Begin by decoding the header block. If this fails, we need to
# tear down the entire connection. TODO: actually do that.
headers = self._decoder.decode(b''.join(self.response_header_datas))
self._handle_header_block(headers)
# We've handled the headers, zero them out.
self.response_header_datas = None
if 'END_STREAM' in frame.flags:
#xlog.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id)
self._close_remote()
self.send_response()
self.close("end stream")
def send_response(self):
status = int(self.response_headers[b':status'][0])
strip_headers(self.response_headers)
body = b''.join(self.response_body)
response = BaseResponse(status=status, headers=self.response_headers, body=body)
response.ssl_sock = self.connection.ssl_sock
response.worker = self.connection
self.task.queue.put(response)
def close(self, reason=""):
self._close_cb(self.stream_id, reason)
def _handle_header_block(self, headers):
"""
Handles the logic for receiving a completed headers block.
A headers block is an uninterrupted sequence of one HEADERS frame
followed by zero or more CONTINUATION frames, and is terminated by a
frame bearing the END_HEADERS flag.
HTTP/2 allows receipt of up to three such blocks on a stream. The first
is optional, and contains a 1XX response. The second is mandatory, and
must contain a final response (200 or higher). The third is optional,
and may contain 'trailers', headers that are sent after a chunk-encoded
body is sent.
Here we only process the simple state: no push, one header frame.
"""
if self.response_headers is None:
self.response_headers = HTTPHeaderMap(headers)
else:
# Received too many headers blocks.
raise ProtocolError("Too many header blocks.")
return
@property
def _local_closed(self):
return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL)
@property
def _remote_closed(self):
return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE)
@property
def _local_open(self):
return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE)
def _close_local(self):
self.state = (
STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN
else STATE_CLOSED
)
def _close_remote(self):
self.state = (
STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN
else STATE_CLOSED
)
