def read_http_req_header(self, reader):
"""read & parse http headers"""
line = reader.readline()
if line is None:
return None
line = line.strip()
if not utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b'chunked' in header_dict[b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.gzip = (b'gzip' in header_dict[b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
req_header.expect = header_dict[b'expect']
req_header.raw_data = b'\n'.join(lines)
return req_header
def append(self, tcp_pac):
if len(tcp_pac.body) == 0:
return
if self.status == HttpConn.STATUS_ERROR or self.status == HttpConn.STATUS_CLOSED:
# not http conn or conn already closed.
return
if self.status == HttpConn.STATUS_BEGIN:
if tcp_pac.body:
if utils.is_request(tcp_pac.body):
self.status = HttpConn.STATUS_RUNNING
if tcp_pac.source == self.source_ip:
http_type = HttpType.REQUEST
else:
http_type = HttpType.RESPONSE
if self.status == HttpConn.STATUS_RUNNING and tcp_pac.body:
self.http_parser.send(http_type, tcp_pac.body)
if tcp_pac.pac_type == -1:
# end of connection
if self.status == HttpConn.STATUS_RUNNING:
self.status = HttpConn.STATUS_CLOSED
else:
self.status = HttpConn.STATUS_ERROR
def append(self, tcp_pac):
if len(tcp_pac.body) == 0:
return
if self.status == HttpConn.STATUS_ERROR or self.status == HttpConn.STATUS_CLOSED:
# not http conn or conn already closed.
return
if self.status == HttpConn.STATUS_BEGIN:
if tcp_pac.body:
if utils.is_request(tcp_pac.body):
self.status = HttpConn.STATUS_RUNNING
if tcp_pac.source == self.source_ip:
http_type = HttpType.REQUEST
else:
http_type = HttpType.RESPONSE
if self.status == HttpConn.STATUS_RUNNING and tcp_pac.body:
self.http_parser.send(http_type, tcp_pac.body)
if tcp_pac.pac_type == -1:
# end of connection
if self.status == HttpConn.STATUS_RUNNING:
self.status = HttpConn.STATUS_CLOSED
else:
self.status = HttpConn.STATUS_ERROR
def append(self, tcp_pac):
print("xiaozhen, HttpConn")
print([tcp_pac,self.source_ip, self.source_port, self.dest_ip, self.dest_port])
print("xiaozhen, HttpConn end ")
if len(tcp_pac.body) == 0:
return
#Store the tcp infos, will be used to display
self.tcp_pac_list.append([tcp_pac,self.source_ip, self.source_port, self.dest_ip, self.dest_port])
if self.status == HttpConn.STATUS_ERROR or self.status == HttpConn.STATUS_CLOSED:
# not http conn or conn already closed.
return
if self.status == HttpConn.STATUS_BEGIN:
if tcp_pac.body:
if utils.is_request(tcp_pac.body):
self.status = HttpConn.STATUS_RUNNING
if tcp_pac.source == self.source_ip:
http_type = HttpType.REQUEST
else:
http_type = HttpType.RESPONSE
if self.status == HttpConn.STATUS_RUNNING and tcp_pac.body:
self.http_parser.send(http_type, tcp_pac.body)
if tcp_pac.pac_type == -1:
# end of connection
if self.status == HttpConn.STATUS_RUNNING:
self.status = HttpConn.STATUS_CLOSED
else:
self.status = HttpConn.STATUS_ERROR
def read_request(self, reader, message):
""" read and output one http request. """
if message.expect_header and not utils.is_request(reader.fetchline()):
req_header = message.expect_header
message.expect_header = None
else:
req_header = self.read_http_req_header(reader)
if req_header is None:
# read header error, we skip all data.
reader.skipall()
return
if req_header.expect:
# it is expect:continue-100 post request
message.expect_header = req_header
# deal with body
if not req_header.chunked:
content = reader.read(req_header.content_len)
else:
content = self.read_chunked_body(reader)
_filter = config.get_filter()
show = _filter.by_domain(req_header.host) and _filter.by_uri(req_header.uri)
message.filtered = not show
if show:
self.processor.on_http_req(req_header, content)
def get_tcpconn(infile):
pcap_file = parse_pcap_file(infile)
conn_dict = OrderedDict()
conn_sorted = []
for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].on_packet(tcp_pac)
# conn closed.
if conn_dict[key].closed():
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.syn and not tcp_pac.ack:
conn_dict[key] = TcpConnection(tcp_pac)
conn_sorted.append(conn_dict[key])
elif utils.is_request(tcp_pac.body):
# tcp init before capture, we start from a possible http request header.
conn_dict[key] = TcpConnection(tcp_pac)
conn_sorted.append(conn_dict[key])
return conn_sorted
def on_packet(self, packet):
"""
:type packet: TcpPack
"""
if self.is_http is None and packet.body:
self.is_http = is_request(packet.body)
if self.is_http == False:
return
if packet.source_key() == self.client_key:
send_stream = self.up_stream
confirm_stream = self.down_stream
pac_type = HttpType.RESPONSE
else:
send_stream = self.down_stream
confirm_stream = self.up_stream
pac_type = HttpType.REQUEST
if len(packet.body) > 0:
send_stream.append_packet(packet)
if packet.syn:
pass
if packet.ack:
packets = confirm_stream.retrieve_packet(packet.ack_seq)
if packets:
for packet in packets:
self.http_parser.send(pac_type, packet.body)
if packet.fin:
send_stream.status = 1
def read_request(self, reader, message):
"""read and output one http request.
Args:
reader (DataReader): type of reader that is required
message (RequestMessage): response packet
Returns:
None if error happened , content otherwise
"""
if message.expect_header and not utils.is_request(reader.fetch_line()):
req_header = message.expect_header
message.expect_header = None
else:
req_header = self.read_http_req_header(reader)
if req_header is None:
# read header error, we skip all data.
reader.skip_all()
return None
if req_header.expect:
# it is expect:continue-100 post request
message.expect_header = req_header
# deal with body
# if not req_header.chunked:
# content = reader.read(req_header.content_len)
# else:
# content = self.read_chunked_body(reader)
fullUrl = "http://" + str(req_header.host.decode("utf-8")) + str(
req_header.uri.decode("utf-8"))
return fullUrl
def read_http_req_header(self, reader):
"""read & parse http headers"""
line = reader.read_line()
if line is None:
return None
line = line.strip()
if not utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b"transfer-encoding" in header_dict and b'chunked' in header_dict[
b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.compress = utils.get_compress_type(
header_dict[b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
req_header.expect = header_dict[b'expect']
req_header.raw_data = b'\n'.join(lines)
return req_header
def read_request(self, reader, message):
""" read and output one http request. """
if message.expect_header and not utils.is_request(reader.fetchline()):
req_header = message.expect_header
message.expect_header = None
else:
req_header = self.read_http_req_header(reader)
if req_header is None:
# read header error, we skip all data.
reader.skipall()
return
if req_header.expect:
# it is expect:continue-100 post request
message.expect_header = req_header
# deal with body
if not req_header.chunked:
content = reader.read(req_header.content_len)
else:
content = self.read_chunked_body(reader)
_filter = config.get_filter()
show = _filter.by_domain(req_header.host) and _filter.by_uri(
req_header.uri)
message.filtered = not show
if show:
self.processor.on_http_req(req_header, content)
def _init(self, http_type, data):
if not utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
self.task_queue = Queue() # one task is an http request or http response stream
self.worker = threading.Thread(target=self.process_tasks, args=(self.task_queue,))
self.worker.setDaemon(True)
self.worker.start()
def parse_pcap_file(file_path):
"""pcap parser.
parse a pcap file to get a list :class:`TcpPacket` objects
Args:
file_path (str): address of the Pcap file that is ready to be parsed
Returns:
list of :class:TcpPacket of found conversations in the Pcap file
Raises:
:class:FileParsingException if either file format were not recognized or file was not found
"""
conn_dict = OrderedDict()
all_packets = []
try:
with io.open(file_path, "rb") as infile:
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
FileParsingException("unknown file format.")
for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
url = conn_dict[key].on_packet(tcp_pac)
if url is not None:
packet = TcpPacket()
packet.request = url
splited = str(key).split('-')
packet.sourceHost = splited[0].split(':')[0]
packet.destinationHost = splited[1].split(':')[0]
packet.sourcePort = splited[0].split(':')[1]
packet.destinationPort = splited[1].split(':')[1]
all_packets.append(packet)
# conn closed.
if conn_dict[key].closed():
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.syn and not tcp_pac.ack:
conn_dict[key] = TcpConnection(tcp_pac)
elif utils.is_request(tcp_pac.body):
# tcp init before capture, we start from a possible http request header.
conn_dict[key] = TcpConnection(tcp_pac)
except (FileNotFoundError, FileParsingException):
raise FileParsingException("parse_pcap failed to parse " + str(
file_path))
# finish connection which not close yet
for conn in conn_dict.values():
conn.finish()
return all_packets
def parse_pcap_file(file_path):
"""pcap parser.
parse a pcap file to get a list :class:`TcpPacket` objects
Args:
file_path (str): address of the Pcap file that is ready to be parsed
Returns:
list of :class:TcpPacket of found conversations in the Pcap file
Raises:
:class:FileParsingException if either file format were not recognized or file was not found
"""
conn_dict = OrderedDict()
all_packets = []
try:
with io.open(file_path, "rb") as infile:
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
FileParsingException("unknown file format.")
for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
url = conn_dict[key].on_packet(tcp_pac)
if url is not None:
packet = TcpPacket()
packet.request = url
splited = str(key).split('-')
packet.sourceHost = splited[0].split(':')[0]
packet.destinationHost = splited[1].split(':')[0]
packet.sourcePort = splited[0].split(':')[1]
packet.destinationPort = splited[1].split(':')[1]
all_packets.append(packet)
# conn closed.
if conn_dict[key].closed():
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.syn and not tcp_pac.ack:
conn_dict[key] = TcpConnection(tcp_pac)
elif utils.is_request(tcp_pac.body):
# tcp init before capture, we start from a possible http request header.
conn_dict[key] = TcpConnection(tcp_pac)
except (FileNotFoundError, FileParsingException):
raise FileParsingException("parse_pcap failed to parse " +
str(file_path))
# finish connection which not close yet
for conn in conn_dict.values():
conn.finish()
return all_packets
def _init(self, http_type, data):
if not utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
self.task_queue = Queue(
) # one task is an http request or http response stream
self.worker = threading.Thread(target=self.process_tasks,
args=(self.task_queue, ))
self.worker.setDaemon(True)
self.worker.start()
def _init(self, http_type, data):
"""check if request is http
Args:
http_type (HttpType): :class:`HttpType.REQUEST` or :class:`HttpType.RESPONSE`
data (str): data of unparsed packet
"""
if not utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
def pcap_file(conn_dict, infile):
"""
:type conn_dict: dict
:type infile:file
"""
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.", file=sys.stderr)
sys.exit(1)
_filter = config.get_filter()
for tcp_pac in packet_parser.read_package_r(pcap_file):
# filter
# get time
if CTCore.activity_date_time == "":
CTCore.activity_date_time = datetime.fromtimestamp(
int(str(tcp_pac.micro_second)[:10]))
if CTCore.client.headers["IP"] == "":
CTCore.client.headers["IP"] = tcp_pac.source
if CTCore.client.headers["MAC"] == "":
CTCore.client.headers["MAC"] = tcp_pac.src_mac
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port)
or _filter.by_port(tcp_pac.dest_port)):
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac)
elif tcp_pac.pac_type == 0:
# tcp init before capture, we found a http request header, begin parse
# if is a http request?
if utils.is_request(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac)
def pcap_file(conn_dict, infile):
"""
:type conn_dict: dict
:type infile:file
"""
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.", file=sys.stderr)
sys.exit(1)
_filter = config.get_filter()
for tcp_pac in packet_parser.read_package_r(pcap_file):
# filter
# get time
if CTCore.activity_date_time == "":
CTCore.activity_date_time = time.strftime('%a, %x %X', time.gmtime(int(str(tcp_pac.micro_second)[:10])))
if CTCore.client.headers["IP"] == "":
CTCore.client.headers["IP"] = tcp_pac.source
if CTCore.client.headers["MAC"] == "":
CTCore.client.headers["MAC"] = tcp_pac.src_mac
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac)
elif tcp_pac.pac_type == 0:
# tcp init before capture, we found a http request header, begin parse
# if is a http request?
if utils.is_request(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac)
def parse_pcap_file(infile):
"""
:type infile:file
"""
conn_dict = OrderedDict()
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.", file=sys.stderr)
sys.exit(1)
_filter = config.get_filter()
for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
# filter
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port)
or _filter.by_port(tcp_pac.dest_port)):
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].on_packet(tcp_pac)
# conn closed.
if conn_dict[key].closed():
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.syn and not tcp_pac.ack:
conn_dict[key] = TcpConnection(tcp_pac)
elif utils.is_request(tcp_pac.body):
# tcp init before capture, we start from a possible http request header.
conn_dict[key] = TcpConnection(tcp_pac)
# finish connection which not close yet
for conn in conn_dict.values():
conn.finish()
def read_http_req_header(self, reader):
"""read & parse http headers"""
line = reader.readline()
if line is None:
return None
line = line.strip()
if not utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
# In case proxy was used
if req_header.uri.find("http://") == 0:
req_header.uri = req_header.uri[len("http://") +
len(header_dict[b"host"]):]
for key in header_dict.iterkeys():
CTCore.client.add_header(key, header_dict[key])
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b'chunked' in header_dict[b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.compress = utils.get_compress_type(
header_dict[b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
req_header.expect = header_dict[b'expect']
req_header.referer = ""
if b"referer" in header_dict:
req_header.referer = header_dict[b'referer']
req_header.raw_data = b'\n'.join(lines)
return req_header
def parse_pcap_file(infile):
"""
:type infile:file
"""
conn_dict = OrderedDict()
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.", file=sys.stderr)
sys.exit(1)
_filter = config.get_filter()
for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
# filter
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].on_packet(tcp_pac)
# conn closed.
if conn_dict[key].closed():
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.syn and not tcp_pac.ack:
conn_dict[key] = TcpConnection(tcp_pac)
elif utils.is_request(tcp_pac.body):
# tcp init before capture, we start from a possible http request header.
conn_dict[key] = TcpConnection(tcp_pac)
# finish connection which not close yet
for conn in conn_dict.values():
conn.finish()
def read_http_req_header(self, reader):
"""read & parse http headers"""
line = reader.readline()
if line is None:
return None
line = line.strip()
if not utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
# In case proxy was used
if req_header.uri.find("http://") == 0:
req_header.uri = req_header.uri[len("http://") + len(header_dict[b"host"]):]
for key in header_dict.iterkeys():
CTCore.client.add_header(key, header_dict[key])
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b'chunked' in header_dict[b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.compress = utils.get_compress_type(header_dict[b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
req_header.expect = header_dict[b'expect']
req_header.referer = ""
if b"referer" in header_dict:
req_header.referer = header_dict[b'referer']
req_header.raw_data = b'\n'.join(lines)
return req_header
def on_packet(self, packet):
"""parse a :class:`TcpPack` object.
change attributes(ack, sin, fin , ...) of a :class:`TcpPack` object
Args:
packet (TcpPack): packet that need to be appended
Returns:
a tcp request header
"""
all_packets = None
if self.is_http is None and packet.body:
self.is_http = is_request(packet.body)
if self.is_http is False:
return
if packet.source_key() == self.client_key:
send_stream = self.up_stream
confirm_stream = self.down_stream
pac_type = HttpType.RESPONSE
else:
send_stream = self.down_stream
confirm_stream = self.up_stream
pac_type = HttpType.REQUEST
if len(packet.body) > 0:
send_stream.append_packet(packet)
if packet.syn:
pass
if packet.ack:
packets = confirm_stream.retrieve_packet(packet.ack_seq)
if packets:
for packet in packets:
tcp = self.http_parser.send(pac_type, packet.body)
if tcp is not None:
all_packets = tcp
if packet.fin:
send_stream.status = 1
return all_packets
def on_packet(self, packet):
"""parse a :class:`TcpPack` object.
change attributes(ack, sin, fin , ...) of a :class:`TcpPack` object
Args:
packet (TcpPack): packet that need to be appended
Returns:
a tcp request header
"""
all_packets = None
if self.is_http is None and packet.body:
self.is_http = is_request(packet.body)
if self.is_http is False:
return
if packet.source_key() == self.client_key:
send_stream = self.up_stream
confirm_stream = self.down_stream
pac_type = HttpType.RESPONSE
else:
send_stream = self.down_stream
confirm_stream = self.up_stream
pac_type = HttpType.REQUEST
if len(packet.body) > 0:
send_stream.append_packet(packet)
if packet.syn:
pass
if packet.ack:
packets = confirm_stream.retrieve_packet(packet.ack_seq)
if packets:
for packet in packets:
tcp = self.http_parser.send(pac_type, packet.body)
if tcp is not None:
all_packets = tcp
if packet.fin:
send_stream.status = 1
return all_packets
def parse_pcap_file(conn_dict, infile):
"""
:type conn_dict: dict
:type infile:file
"""
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.", file=sys.stderr)
sys.exit(1)
_filter = config.get_filter()
for tcp_pac in packet_parser.read_package_r(pcap_file):
# filter
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac)
elif tcp_pac.pac_type == 0:
# tcp init before capture, we found a http request header, begin parse
# if is a http request?
if utils.is_request(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac)
def read_http_req_header(self, reader):
"""read & parse request http headers
Args:
reader (DataReader): type of reader that is required
Returns:
`None` if not a response or any error happend , `dict` of parsed_header otherwise
"""
line = reader.read_line()
if line is None:
return None
line = line.strip()
if not utils.is_request(line):
return None
req_header = HttpRequestHeader()
items = line.split(b' ')
if len(items) == 3:
req_header.method = items[0]
req_header.uri = items[1]
req_header.protocol = items[2]
lines = [line]
header_dict = self.read_headers(reader, lines)
if b"content-length" in header_dict:
req_header.content_len = int(header_dict[b"content-length"])
if b"transfer-encoding" in header_dict and b'chunked' in header_dict[
b"transfer-encoding"]:
req_header.chunked = True
req_header.content_type = header_dict[b'content-type']
req_header.compress = utils.get_compress_type(header_dict[
b"content-encoding"])
req_header.host = header_dict[b"host"]
if b'expect' in header_dict:
req_header.expect = header_dict[b'expect']
req_header.raw_data = b'\n'.join(lines)
return req_header
def process_pcap_file(conn_dict, infile):
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.")
sys.exit(1)
#used for diameter decode
diameterConn = DiameterConn()
_filter = config.get_filter()
for tcp_pac in packet_parser.read_package_r(pcap_file):
# filter
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port)
or _filter.by_port(tcp_pac.dest_port)):
continue
#IP Protocol
if (tcp_pac.source_port == 8083 or tcp_pac.dest_port == 8083):
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
for package in conn_dict[key].tcp_pac_list:
msg = Message()
msg.type = 'http'
msg.sourceIP = package[1]
msg.sourcePort = package[2]
msg.destIP = package[3]
msg.destPort = package[4]
msg.body = package[0].body
msg.seconds = package[0].seconds
msg.suseconds = package[0].suseconds
yield msg
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac)
elif tcp_pac.pac_type == 0:
# tcp init before capture, we found a http request header, begin parse
# if is a http request?
if utils.is_request(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac)
elif (tcp_pac.source_port in (6553, 16553, 3868)
or tcp_pac.dest_port in (3868, 6553, 16553)):
if len(tcp_pac.body) > 20:
version = struct.unpack('!b', tcp_pac.body[0:1])[0]
message_length = struct.unpack('!I',
b'\x00' + tcp_pac.body[1:4])[0]
Command_Flags = struct.unpack('!b', tcp_pac.body[4:5])[0]
command_code = struct.unpack('!I',
b'\x00' + tcp_pac.body[5:8])[0]
application_ID, hop_by_hop_ID, end_to_end_ID = struct.unpack(
'!III', tcp_pac.body[8:20])
if (version == 1):
#print(version, message_length, command_code)
headerinfo, tree = diameterConn.decode(tcp_pac.body)
body = headerinfo + '\n'
for avp in tree:
body += avp.AVPoutput() + '\n'
msg = Message()
msg.type = 'diameter'
msg.sourceIP = tcp_pac.source
msg.sourcePort = tcp_pac.source_port
msg.destIP = tcp_pac.dest
msg.destPort = tcp_pac.dest_port
msg.body = body
msg.seconds = tcp_pac.seconds
msg.suseconds = tcp_pac.suseconds
yield msg
def test_is_request(self):
self.assertEqual('foo'.upper(), 'FOO')
self.assertTrue(utils.is_request(b'GET /test'))
self.assertTrue(utils.is_request(b'POST /test'))
def test_is_request(self):
self.assertEqual('foo'.upper(), 'FOO')
self.assertTrue(utils.is_request(b'GET /test'))
self.assertTrue(utils.is_request(b'POST /test'))
def _init(self, http_type, data):
if not utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
def process_pcap_file(conn_dict, infile):
file_format, head = get_file_format(infile)
if file_format == FileFormat.PCAP:
pcap_file = pcap.PcapFile(infile, head).read_packet
elif file_format == FileFormat.PCAP_NG:
pcap_file = pcapng.PcapngFile(infile, head).read_packet
else:
print("unknown file format.")
sys.exit(1)
#used for diameter decode
diameterConn = DiameterConn()
_filter = config.get_filter()
for tcp_pac in packet_parser.read_package_r(pcap_file):
# filter
if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
continue
if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
continue
#IP Protocol
if(tcp_pac.source_port == 8083 or tcp_pac.dest_port == 8083):
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
for package in conn_dict[key].tcp_pac_list:
msg = Message()
msg.type = 'http'
msg.sourceIP = package[1]
msg.sourcePort = package[2]
msg.destIP = package[3]
msg.destPort = package[4]
msg.body = package[0].body
msg.seconds = package[0].seconds
msg.suseconds = package[0].suseconds
yield msg
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac)
elif tcp_pac.pac_type == 0:
# tcp init before capture, we found a http request header, begin parse
# if is a http request?
if utils.is_request(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac)
elif(tcp_pac.source_port in (6553,16553,3868) or tcp_pac.dest_port in (3868,6553,16553)):
if len(tcp_pac.body) > 20:
version = struct.unpack('!b',tcp_pac.body[0:1])[0]
message_length = struct.unpack('!I',b'\x00' + tcp_pac.body[1:4])[0]
Command_Flags = struct.unpack('!b',tcp_pac.body[4:5])[0]
command_code = struct.unpack('!I',b'\x00' + tcp_pac.body[5:8])[0]
application_ID, hop_by_hop_ID, end_to_end_ID = struct.unpack('!III', tcp_pac.body[8:20])
if(version == 1):
#print(version, message_length, command_code)
headerinfo, tree = diameterConn.decode(tcp_pac.body)
body = headerinfo + '\n'
for avp in tree:
body += avp.AVPoutput() + '\n'
msg = Message()
msg.type = 'diameter'
msg.sourceIP = tcp_pac.source
msg.sourcePort = tcp_pac.source_port
msg.destIP = tcp_pac.dest
msg.destPort = tcp_pac.dest_port
msg.body = body
msg.seconds = tcp_pac.seconds
msg.suseconds = tcp_pac.suseconds
yield msg
def _init(self, http_type, data):
if not utils.is_request(data) or http_type != HttpType.REQUEST:
# not a http request
self.is_http = False
else:
self.is_http = True
