def read_http_req_header(self, reader):
"""read & parse http headers"""
line = reader.read_line()
if line is None:
return None
line = line.strip()
if not content_utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b"transfer-encoding" in header_dict and b'chunked' in header_dict[
b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.compress = content_utils.get_compress_type(
header_dict[b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
# we only deal with 100-continue now...
if b'100-continue' in header_dict[b'expect']:
req_header.expect = True
req_header.raw_data = b'\n'.join(lines)
return req_header
def on_packet(self, packet):
"""
:type packet: httpcap.packet_parser.TcpPack
"""
self.last_timestamp = packet.timestamp
if self.is_http is None and packet.body:
self.is_http = is_request(packet.body)
if not self.is_http:
return
if packet.source_key() == self.client_key:
send_stream = self.up_stream
confirm_stream = self.down_stream
pac_type = HttpType.RESPONSE
else:
send_stream = self.down_stream
confirm_stream = self.up_stream
pac_type = HttpType.REQUEST
if len(packet.body) > 0:
send_stream.append_packet(packet)
if packet.syn:
pass
if packet.ack:
packets = confirm_stream.retrieve_packet(packet.ack_seq)
if packets:
for packet in packets:
self.http_parser.data_received(pac_type, packet.body)
if packet.fin or packet.rst:
send_stream.status = 1
def read_http_req_header(self, reader):
"""read & parse http headers"""
line = reader.read_line()
if line is None:
return None
line = line.strip()
if not content_utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b"transfer-encoding" in header_dict and b'chunked' in header_dict[b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.compress = content_utils.get_compress_type(header_dict[b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
# we only deal with 100-continue now...
if b'100-continue' in header_dict[b'expect']:
req_header.expect = True
req_header.raw_data = b'\n'.join(lines)
return req_header
def run_parser(produce_packet):
conn_dict = OrderedDict()
def clear_connection():
# finish connection which not close yet
for conn in conn_dict.values():
conn.finish()
cleanups.register(clear_connection)
_filter = config.get_filter()
count = 0
for tcp_pac in packet_parser.read_tcp_packet(produce_packet):
# filter
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].on_packet(tcp_pac)
# conn closed.
if conn_dict[key].closed():
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.syn and not tcp_pac.ack:
conn_dict[key] = TcpConnection(tcp_pac)
elif content_utils.is_request(tcp_pac.body):
# tcp init before capture, we start from a possible http request header.
conn_dict[key] = TcpConnection(tcp_pac)
count += 1
if count % 100 == 0:
# check timeout connection
keys = []
for k, conn in conn_dict.items():
if tcp_pac.timestamp - conn.last_timestamp > 100 * 1000 * 100:
conn.finish()
keys.append(k)
for k in keys:
del conn_dict[k]
clear_connection()
def _init(self, http_type, data):
"""
Called when receive first packet. do init jobs
"""
if not content_utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
# start parser worker
self.request_reader = DataReader() # request data
self.response_reader = DataReader() # response data
worker = HttpParserWorker(self.request_reader, self.response_reader, self.processor)
worker.setName("Http parser worker")
# worker.setDaemon(True)
worker.start()
self.worker = worker
def _init(self, http_type, data):
"""
Called when receive first packet. do init jobs
"""
if not content_utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
# start parser worker
self.request_reader = DataReader() # request data
self.response_reader = DataReader() # response data
worker = HttpParserWorker(self.request_reader, self.response_reader,
self.processor)
worker.setName("Http parser worker")
# worker.setDaemon(True)
worker.start()
self.worker = worker
def test_is_request(self):
self.assertTrue(content_utils.is_request(b'GET /test'))
self.assertTrue(content_utils.is_request(b'POST /test'))
