class DummyResponse(object): def __init__(self, headers): self.headers = HTTPHeaderMap(headers.items()) def read(self): ctype = self.headers.get('content-type') if ctype is not None: if 'json' in ctype[0].decode('utf-8'): return b'{"data": "dummy"}' return b'<html>dummy</html>' def getheader(self, name): return self.headers.get(name) def getheaders(self): return self.headers
def test_actual_get(self): h = HTTPHeaderMap() h['k1'] = 'v1, v2' h['k2'] = 'v3' h['k1'] = 'v4' assert h.get('k1') == [b'v1', b'v2', b'v4']
def action(self, nodename, thr=None, stream_id=None, **kwargs): logfile = os.path.join(rcEnv.paths.pathlog, "node.log") ofile = thr._action_logs_open(logfile, 0, "node") request_headers = HTTPHeaderMap( thr.streams[stream_id]["request"].headers) try: content_type = bdecode(request_headers.get("accept").pop()) except: content_type = "application/json" thr.streams[stream_id]["content_type"] = content_type thr.streams[stream_id]["pushers"].append({ "o": self, "fn": "h2_push_logs", "args": [ofile, True], })
def action(self, nodename, thr=None, stream_id=None, **kwargs): options = self.parse_options(kwargs) thr.selector = options.selector if not thr.event_queue: thr.event_queue = queue.Queue() if options.full: data = thr.daemon_status() namespaces = thr.get_namespaces() fevent = { "nodename": rcEnv.nodename, "ts": time.time(), "kind": "full", "data": thr.filter_daemon_status(data, namespaces=namespaces, selector=options.selector), } if thr.h2conn: _msg = fevent elif thr.encrypted: _msg = thr.encrypt(fevent) else: _msg = thr.msg_encode(fevent) thr.event_queue.put(_msg) if not thr in thr.parent.events_clients: thr.parent.events_clients.append(thr) if not stream_id in thr.events_stream_ids: thr.events_stream_ids.append(stream_id) if thr.h2conn: request_headers = HTTPHeaderMap( thr.streams[stream_id]["request"].headers) try: content_type = bdecode(request_headers.get("accept").pop()) except: content_type = "application/json" thr.streams[stream_id]["content_type"] = content_type thr.streams[stream_id]["pushers"].append({ "fn": "h2_push_action_events", }) else: thr.raw_push_action_events()
def action(self, nodename, thr=None, stream_id=None, **kwargs): options = self.parse_options(kwargs) svc = thr.get_service(options.path) if svc is None: raise HTTP(404, "%s not found" % options.path) request_headers = HTTPHeaderMap( thr.streams[stream_id]["request"].headers) try: content_type = bdecode(request_headers.get("accept").pop()) except: content_type = "application/json" thr.streams[stream_id]["content_type"] = content_type logfile = os.path.join(svc.log_d, svc.name + ".log") ofile = thr._action_logs_open(logfile, 0, svc.path) thr.streams[stream_id]["pushers"].append({ "o": self, "fn": "h2_push_logs", "args": [ofile, True], })
class Stream(object): """ A single HTTP/2 stream. A stream is an independent, bi-directional sequence of HTTP headers and data. Each stream is identified by a single integer. From a HTTP perspective, a stream _approximately_ matches a single request-response pair. """ def __init__(self, logger, config, connection, ip, stream_id, task, send_cb, close_cb, encoder, decoder, receive_window_manager, remote_window_size, max_frame_size): self.logger = logger self.config = config self.connection = connection self.ip = ip self.stream_id = stream_id self.task = task self.state = STATE_IDLE self.get_head_time = None # There are two flow control windows: one for data we're sending, # one for data being sent to us. self.receive_window_manager = receive_window_manager self.remote_window_size = remote_window_size self.max_frame_size = max_frame_size # This is the callback handed to the stream by its parent connection. # It is called when the stream wants to send data. It expects to # receive a list of frames that will be automatically serialized. self._send_cb = send_cb # This is the callback to be called when the stream is closed. self._close_cb = close_cb # A reference to the header encoder and decoder objects belonging to # the parent connection. self._encoder = encoder self._decoder = decoder self.request_headers = HTTPHeaderMap() # Convert the body to bytes if needed. self.request_body = to_bytestring(self.task.body) # request body not send blocked by send window # the left body will send when send window opened. self.request_body_left = len(self.request_body) self.request_body_sended = False # data list before decode self.response_header_datas = [] # Set to a key-value set of the response headers once their # HEADERS..CONTINUATION frame sequence finishes. self.response_headers = None # Unconsumed response data chunks self.response_body = [] self.response_body_len = 0 def start_request(self): """ Open the stream. Does this by encoding and sending the headers: no more calls to ``add_header`` are allowed after this method is called. The `end` flag controls whether this will be the end of the stream, or whether data will follow. """ # Strip any headers invalid in H2. #headers = h2_safe_headers(self.request_headers) host = self.connection.get_host(self.task.host) self.add_header(":method", self.task.method) self.add_header(":scheme", "https") self.add_header(":authority", host) self.add_header(":path", self.task.path) default_headers = (':method', ':scheme', ':authority', ':path') #headers = h2_safe_headers(self.task.headers) for name, value in self.task.headers.items(): is_default = to_native_string(name) in default_headers self.add_header(name, value, replace=is_default) # Encode the headers. encoded_headers = self._encoder(self.request_headers) # It's possible that there is a substantial amount of data here. The # data needs to go into one HEADERS frame, followed by a number of # CONTINUATION frames. For now, for ease of implementation, let's just # assume that's never going to happen (16kB of headers is lots!). # Additionally, since this is so unlikely, there's no point writing a # test for this: it's just so simple. if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover raise ValueError("Header block too large.") header_frame = HeadersFrame(self.stream_id) header_frame.data = encoded_headers # If no data has been provided, this is the end of the stream. Either # way, due to the restriction above it's definitely the end of the # headers. header_frame.flags.add('END_HEADERS') if self.request_body_left == 0: header_frame.flags.add('END_STREAM') # Send the header frame. self.task.set_state("start send header") self._send_cb(header_frame) # Transition the stream state appropriately. self.state = STATE_OPEN self.task.set_state("start send left body") threading.Thread(target=self.left_work).start() def left_work(self): if self.request_body_left > 0: self.send_left_body() self.timeout_response() def add_header(self, name, value, replace=False): """ Adds a single HTTP header to the headers to be sent on the request. """ if not replace: self.request_headers[name] = value else: self.request_headers.replace(name, value) def send_left_body(self): while self.remote_window_size and not self.request_body_sended: send_size = min(self.remote_window_size, self.request_body_left, self.max_frame_size) f = DataFrame(self.stream_id) data_start = len(self.request_body) - self.request_body_left f.data = self.request_body[data_start:data_start+send_size] self.remote_window_size -= send_size self.request_body_left -= send_size # If the length of the data is less than MAX_CHUNK, we're probably # at the end of the file. If this is the end of the data, mark it # as END_STREAM. if self.request_body_left == 0: f.flags.add('END_STREAM') # Send the frame and decrement the flow control window. self._send_cb(f) # If no more data is to be sent on this stream, transition our state. if self.request_body_left == 0: self.request_body_sended = True self._close_local() self.task.set_state("end send left body") def receive_frame(self, frame): """ Handle a frame received on this stream. called by connection. """ # self.logger.debug("stream %d recved frame %r", self.stream_id, frame) if frame.type == WindowUpdateFrame.type: self.remote_window_size += frame.window_increment self.send_left_body() elif frame.type == HeadersFrame.type: # Begin the header block for the response headers. #self.response_header_datas = [frame.data] self.response_header_datas.append(frame.data) elif frame.type == PushPromiseFrame.type: self.logger.error("%s receive PushPromiseFrame:%d", self.ip, frame.stream_id) elif frame.type == ContinuationFrame.type: # Continue a header block begun with either HEADERS or PUSH_PROMISE. self.response_header_datas.append(frame.data) elif frame.type == DataFrame.type: # Append the data to the buffer. if not self.task.finished: self.task.put_data(frame.data) if 'END_STREAM' not in frame.flags: # Increase the window size. Only do this if the data frame contains # actual data. # don't do it if stream is closed. size = frame.flow_controlled_length increment = self.receive_window_manager._handle_frame(size) #if increment: # self.logger.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment) #content_len = int(self.request_headers.get("Content-Length")[0]) #self.logger.debug("%s get:%d s:%d", self.ip, self.response_body_len, size) if increment and not self._remote_closed: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == BlockedFrame.type: # If we've been blocked we may want to fixup the window. increment = self.receive_window_manager._blocked() if increment: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == RstStreamFrame.type: # Rest Frame send from server is not define in RFC inactive_time = time.time() - self.connection.last_active_time self.logger.debug("%s Stream %d Rest by server, inactive:%d. error code:%d", self.ip, self.stream_id, inactive_time, frame.error_code) self.connection.close("RESET") elif frame.type in FRAMES: # This frame isn't valid at this point. #raise ValueError("Unexpected frame %s." % frame) self.logger.error("%s Unexpected frame %s.", self.ip, frame) else: # pragma: no cover # Unknown frames belong to extensions. Just drop it on the # floor, but log so that users know that something happened. self.logger.error("%s Received unknown frame, type %d", self.ip, frame.type) pass if 'END_HEADERS' in frame.flags: if self.response_headers is not None: raise ProtocolError("Too many header blocks.") # Begin by decoding the header block. If this fails, we need to # tear down the entire connection. if len(self.response_header_datas) == 1: header_data = self.response_header_datas[0] else: header_data = b''.join(self.response_header_datas) try: headers = self._decoder.decode(header_data) except Exception as e: self.logger.exception("decode h2 header %s fail:%r", header_data, e) raise e self.response_headers = HTTPHeaderMap(headers) # We've handled the headers, zero them out. self.response_header_datas = None self.get_head_time = time.time() length = self.response_headers.get("Content-Length", None) if isinstance(length, list): length = int(length[0]) if not self.task.finished: self.task.content_length = length self.task.set_state("h2_get_head") self.send_response() if 'END_STREAM' in frame.flags: #self.logger.debug("%s Closing remote side of stream:%d", self.ip, self.stream_id) time_now = time.time() time_cost = time_now - self.get_head_time if time_cost > 0 and \ isinstance(self.task.content_length, int) and \ not self.task.finished: speed = self.task.content_length / time_cost self.task.set_state("h2_finish[SP:%d]" % speed) self._close_remote() self.close("end stream") if not self.task.finished: self.connection.continue_timeout = 0 def send_response(self): if self.task.responsed: self.logger.error("http2_stream send_response but responsed.%s", self.task.url) self.close("h2 stream send_response but sended.") return self.task.responsed = True status = int(self.response_headers[b':status'][0]) strip_headers(self.response_headers) response = simple_http_client.BaseResponse(status=status, headers=self.response_headers) response.ssl_sock = self.connection.ssl_sock response.worker = self.connection response.task = self.task self.task.queue.put(response) if status in self.config.http2_status_to_close: self.connection.close("status %d" % status) def close(self, reason="close"): if not self.task.responsed: self.connection.retry_task_cb(self.task, reason) else: self.task.finish() # empty block means fail or closed. self._close_remote() self._close_cb(self.stream_id, reason) @property def _local_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL) @property def _remote_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE) @property def _local_open(self): return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE) def _close_local(self): self.state = ( STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN else STATE_CLOSED ) def _close_remote(self): self.state = ( STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN else STATE_CLOSED ) def timeout_response(self): start_time = time.time() while time.time() - start_time < self.task.timeout: time.sleep(1) if self._remote_closed: return self.logger.warn("h2 timeout %s task_trace:%s worker_trace:%s", self.connection.ssl_sock.ip, self.task.get_trace(), self.connection.get_trace()) self.task.set_state("timeout") if self.task.responsed: self.task.finish() else: self.task.response_fail("timeout") self.connection.continue_timeout += 1 if self.connection.continue_timeout >= self.connection.config.http2_max_timeout_tasks and \ time.time() - self.connection.last_active_time > self.connection.config.http2_timeout_active: self.connection.close("down fail")
def test_doesnt_split_set_cookie(self): h = HTTPHeaderMap() h['Set-Cookie'] = 'v1, v2' assert h['set-cookie'] == [b'v1, v2'] assert h.get(b'set-cookie') == [b'v1, v2']
def test_empty_get(self): h = HTTPHeaderMap() assert h.get('nonexistent', 'hi there') == 'hi there'
class Stream(object): """ A single HTTP/2 stream. A stream is an independent, bi-directional sequence of HTTP headers and data. Each stream is identified by a single integer. From a HTTP perspective, a stream _approximately_ matches a single request-response pair. """ def __init__(self, logger, config, connection, ip_str, stream_id, task, send_cb, close_cb, encoder, decoder, receive_window_manager, remote_window_size, max_frame_size): self.logger = logger self.config = config self.connection = connection self.ip_str = ip_str self.stream_id = stream_id self.task = task self.state = STATE_IDLE self.get_head_time = None # There are two flow control windows: one for data we're sending, # one for data being sent to us. self.receive_window_manager = receive_window_manager self.remote_window_size = remote_window_size self.max_frame_size = max_frame_size # This is the callback handed to the stream by its parent connection. # It is called when the stream wants to send data. It expects to # receive a list of frames that will be automatically serialized. self._send_cb = send_cb # This is the callback to be called when the stream is closed. self._close_cb = close_cb # A reference to the header encoder and decoder objects belonging to # the parent connection. self._encoder = encoder self._decoder = decoder self.request_headers = HTTPHeaderMap() # Convert the body to bytes if needed. self.request_body = to_bytestring(self.task.body) # request body not send blocked by send window # the left body will send when send window opened. self.request_body_left = len(self.request_body) self.request_body_sended = False # data list before decode self.response_header_datas = [] # Set to a key-value set of the response headers once their # HEADERS..CONTINUATION frame sequence finishes. self.response_headers = None # Unconsumed response data chunks self.response_body = [] self.response_body_len = 0 self.start_time = time.time() def start_request(self): """ Open the stream. Does this by encoding and sending the headers: no more calls to ``add_header`` are allowed after this method is called. The `end` flag controls whether this will be the end of the stream, or whether data will follow. """ # Strip any headers invalid in H2. #headers = h2_safe_headers(self.request_headers) host = self.connection.get_host(self.task.host) self.add_header(":method", self.task.method) self.add_header(":scheme", "https") self.add_header(":authority", host) self.add_header(":path", self.task.path) default_headers = (':method', ':scheme', ':authority', ':path') #headers = h2_safe_headers(self.task.headers) for name, value in list(self.task.headers.items()): is_default = to_native_string(name) in default_headers self.add_header(name, value, replace=is_default) # Encode the headers. encoded_headers = self._encoder(self.request_headers) # It's possible that there is a substantial amount of data here. The # data needs to go into one HEADERS frame, followed by a number of # CONTINUATION frames. For now, for ease of implementation, let's just # assume that's never going to happen (16kB of headers is lots!). # Additionally, since this is so unlikely, there's no point writing a # test for this: it's just so simple. if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover raise ValueError("Header block too large.") header_frame = HeadersFrame(self.stream_id) header_frame.data = encoded_headers # If no data has been provided, this is the end of the stream. Either # way, due to the restriction above it's definitely the end of the # headers. header_frame.flags.add('END_HEADERS') if self.request_body_left == 0: header_frame.flags.add('END_STREAM') # Send the header frame. self.task.set_state("start send header") self._send_cb(header_frame) # Transition the stream state appropriately. self.state = STATE_OPEN self.task.set_state("start send left body") if self.request_body_left > 0: self.send_left_body() def add_header(self, name, value, replace=False): """ Adds a single HTTP header to the headers to be sent on the request. """ if not replace: self.request_headers[name] = value else: self.request_headers.replace(name, value) def send_left_body(self): while self.remote_window_size and not self.request_body_sended: send_size = min(self.remote_window_size, self.request_body_left, self.max_frame_size) f = DataFrame(self.stream_id) data_start = len(self.request_body) - self.request_body_left f.data = self.request_body[data_start:data_start + send_size] self.remote_window_size -= send_size self.request_body_left -= send_size # If the length of the data is less than MAX_CHUNK, we're probably # at the end of the file. If this is the end of the data, mark it # as END_STREAM. if self.request_body_left == 0: f.flags.add('END_STREAM') # Send the frame and decrement the flow control window. self._send_cb(f) # If no more data is to be sent on this stream, transition our state. if self.request_body_left == 0: self.request_body_sended = True self._close_local() self.task.set_state("end send left body") def receive_frame(self, frame): """ Handle a frame received on this stream. called by connection. """ # self.logger.debug("stream %d recved frame %r", self.stream_id, frame) if frame.type == WindowUpdateFrame.type: self.remote_window_size += frame.window_increment self.send_left_body() elif frame.type == HeadersFrame.type: # Begin the header block for the response headers. #self.response_header_datas = [frame.data] self.response_header_datas.append(frame.data) elif frame.type == PushPromiseFrame.type: self.logger.error("%s receive PushPromiseFrame:%d", self.ip_str, frame.stream_id) elif frame.type == ContinuationFrame.type: # Continue a header block begun with either HEADERS or PUSH_PROMISE. self.response_header_datas.append(frame.data) elif frame.type == DataFrame.type: # Append the data to the buffer. if not self.task.finished: self.task.put_data(frame.data) if 'END_STREAM' not in frame.flags: # Increase the window size. Only do this if the data frame contains # actual data. # don't do it if stream is closed. size = frame.flow_controlled_length increment = self.receive_window_manager._handle_frame(size) #if increment: # self.logger.debug("stream:%d frame size:%d increase win:%d", self.stream_id, size, increment) #content_len = int(self.request_headers.get("Content-Length")[0]) #self.logger.debug("%s get:%d s:%d", self.ip, self.response_body_len, size) if increment and not self._remote_closed: w = WindowUpdateFrame(self.stream_id) w.window_increment = increment self._send_cb(w) elif frame.type == RstStreamFrame.type: # Rest Frame send from server is not define in RFC inactive_time = time.time() - self.connection.last_recv_time self.logger.debug( "%s Stream %d Rest by server, inactive:%d. error code:%d", self.ip_str, self.stream_id, inactive_time, frame.error_code) self.connection.close("RESET") elif frame.type in FRAMES: # This frame isn't valid at this point. #raise ValueError("Unexpected frame %s." % frame) self.logger.error("%s Unexpected frame %s.", self.ip_str, frame) else: # pragma: no cover # Unknown frames belong to extensions. Just drop it on the # floor, but log so that users know that something happened. self.logger.error("%s Received unknown frame, type %d", self.ip_str, frame.type) pass if 'END_HEADERS' in frame.flags: if self.config.http2_show_debug: self.logger.debug("END_HEADERS") if self.response_headers is not None: raise ProtocolError("Too many header blocks.") # Begin by decoding the header block. If this fails, we need to # tear down the entire connection. if len(self.response_header_datas) == 1: header_data = self.response_header_datas[0] else: header_data = b''.join(self.response_header_datas) try: headers = self._decoder.decode(header_data) except Exception as e: self.logger.exception("decode h2 header %s fail:%r", header_data, e) raise e self.response_headers = HTTPHeaderMap(headers) # We've handled the headers, zero them out. self.response_header_datas = None self.get_head_time = time.time() length = self.response_headers.get("Content-Length", None) if isinstance(length, list): length = int(length[0]) if not self.task.finished: self.task.content_length = length self.task.set_state("h2_get_head") self.send_response() if 'END_STREAM' in frame.flags: if self.config.http2_show_debug: self.logger.debug("%s Closing remote side of stream:%d", self.connection.ssl_sock.ip_str, self.stream_id) time_now = time.time() time_cost = time_now - self.get_head_time if time_cost > 0 and \ isinstance(self.task.content_length, int) and \ not self.task.finished: speed = self.task.content_length / time_cost self.task.set_state("h2_finish[SP:%d]" % speed) self._close_remote() self.close("end stream") if not self.task.finished: self.connection.continue_timeout = 0 def send_response(self): if self.task.responsed: self.logger.error("http2_stream send_response but responsed.%s", self.task.url) self.close("h2 stream send_response but sended.") return self.task.responsed = True status = int(self.response_headers[b':status'][0]) strip_headers(self.response_headers) response = simple_http_client.BaseResponse( status=status, headers=self.response_headers) response.ssl_sock = self.connection.ssl_sock response.worker = self.connection response.task = self.task if self.config.http2_show_debug: self.logger.debug("self.task.queue.put(response)") self.task.queue.put(response) if status in self.config.http2_status_to_close: self.connection.close("status %d" % status) def close(self, reason="close"): if not self.task.responsed: self.connection.retry_task_cb(self.task, reason) else: self.task.finish() # empty block means fail or closed. self._close_remote() self._close_cb(self.stream_id, reason) @property def _local_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_LOCAL) @property def _remote_closed(self): return self.state in (STATE_CLOSED, STATE_HALF_CLOSED_REMOTE) @property def _local_open(self): return self.state in (STATE_OPEN, STATE_HALF_CLOSED_REMOTE) def _close_local(self): self.state = (STATE_HALF_CLOSED_LOCAL if self.state == STATE_OPEN else STATE_CLOSED) def _close_remote(self): self.state = (STATE_HALF_CLOSED_REMOTE if self.state == STATE_OPEN else STATE_CLOSED) def check_timeout(self, now): if time.time() - self.start_time < self.task.timeout: return if self._remote_closed: return self.logger.warn("h2 timeout %s task_trace:%s worker_trace:%s", self.connection.ssl_sock.ip_str, self.task.get_trace(), self.connection.get_trace()) self.task.set_state("timeout") if self.task.responsed: self.task.finish() else: self.task.response_fail("timeout") self.connection.continue_timeout += 1