def main():
parser = argparse.ArgumentParser()
parser.add_argument("pcap_file", help="the pcap file to parse")
parser.add_argument("-i", "--ip", help="only parse packages with specified source OR dest ip")
parser.add_argument("-p", "--port", type=int, help="only parse packages with specified source OR dest port")
parser.add_argument("-v", "--verbosity", help="increase output verbosity(-vv is recommended)", action="count")
parser.add_argument("-o", "--output", help="output to file instead of stdout")
parser.add_argument("-e", "--encoding", help="decode the data use specified encodings.")
parser.add_argument("-b", "--beauty", help="output json in a pretty way.", action="store_true")
args = parser.parse_args()
filepath = args.pcap_file
port = args.port
ip = args.ip
if args.verbosity:
parse_config.level = args.verbosity
if args.encoding:
parse_config.encoding = args.encoding
parse_config.pretty = args.beauty
if args.output:
outputfile = open(args.output, "w+")
else:
outputfile = sys.stdout
with open(filepath) as pcap_file:
conn_dict = OrderedDict()
for tcp_pac in pcap.readPcapPackageRegular(pcap_file):
#filter
if port is not None and tcp_pac.source_port != port and tcp_pac.dest_port != port:
continue
if ip is not None and tcp_pac.source != ip and tcp_pac.dest != ip:
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == pcap.TcpPack.TYPE_CLOSE:
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac, outputfile)
elif tcp_pac.pac_type == 0:
# tcp init before capature, we found a http request header, begin parse
# if is a http request?
if textutils.ishttprequest(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac, outputfile)
for conn in conn_dict.values():
conn.finish()
if args.output:
outputfile.close()
def append(self, tcp_pac):
if len(tcp_pac.body) == 0:
return
if self.status == HttpConn.STATUS_ERROR or self.status == HttpConn.STATUS_CLOSED:
# not http conn or conn already closed.
return
if self.status == HttpConn.STATUS_BEGIN:
if tcp_pac.body:
if textutils.ishttprequest(tcp_pac.body):
self.status = HttpConn.STATUS_RUNNING
if tcp_pac.pac_type == -1:
# end of connection
if self.status == HttpConn.STATUS_RUNNING:
self.status = HttpConn.STATUS_CLOSED
else:
self.status = HttpConn.STATUS_ERROR
if tcp_pac.source == self.source_ip:
httptype = HttpType.REQUEST
else:
httptype = HttpType.RESPONSE
if tcp_pac.body:
self.queue.put((httptype, tcp_pac.body))
def append(self, tcp_pac):
if len(tcp_pac.body) == 0:
return
if self.status == HttpConn.STATUS_ERROR or self.status == HttpConn.STATUS_CLOSED:
# not http conn or conn already closed.
return
if self.status == HttpConn.STATUS_BEGIN:
if tcp_pac.body:
if textutils.ishttprequest(tcp_pac.body):
self.status = HttpConn.STATUS_RUNNING
if tcp_pac.pac_type == -1:
# end of connection
if self.status == HttpConn.STATUS_RUNNING:
self.status = HttpConn.STATUS_CLOSED
else:
self.status = HttpConn.STATUS_ERROR
return
if tcp_pac.source == self.source_ip:
httptype = HttpType.REQUEST
else:
httptype = HttpType.RESPONSE
if tcp_pac.body:
self.queue.put((httptype, tcp_pac.body))
def read_request(reader, outputfile, request_status, parse_config):
"""
read and output one http request.
"""
if 'expect' in request_status and not textutils.ishttprequest(
reader.fetchline()):
headers = request_status['expect']
del request_status['expect']
else:
headers = read_http_headers(reader, outputfile, parse_config.level)
if headers.expect:
# assume it is expect:continue-100
request_status['expect'] = headers
if headers is None or not isinstance(headers, HttpRequestHeader):
outputfile.write("{Error, cannot parse http request headers.}")
outputfile.write('\n')
print reader.readall()
return
mime, charset = textutils.parse_content_type(headers.content_type)
# usually charset is not set in http post
output_body = parse_config.level >= OutputLevel.ALL_BODY and not textutils.isbinarybody(mime) \
or parse_config.level >= OutputLevel.TEXT_BODY and textutils.istextbody(mime)
content = ''
# deal with body
if not headers.chunked:
if output_body:
content = reader.read(headers.content_len)
else:
reader.skip(headers.content_len)
else:
content = read_chunked_body(reader)
if not headers.gzip:
# if is gzip by content magic header
# someone missed the content-encoding header
headers.gzip = textutils.isgzip(content)
# if it is form url encode
if 'expect' in request_status and not content:
content = '{Expect-continue-100, see next content for http post body}'
if output_body:
#unescape www-form-encoded data.x-www-form-urlencoded
if parse_config.encoding and not charset:
charset = parse_config.encoding
print_body(content, headers.gzip, charset, outputfile,
'www-form-encoded' in mime, parse_config.pretty)
def read_request(reader, outputfile, request_status, parse_config):
"""
read and output one http request.
"""
if 'expect' in request_status and not textutils.ishttprequest(reader.fetchline()):
headers = request_status['expect']
del request_status['expect']
else:
headers = read_http_headers(reader, outputfile, parse_config.level)
if headers is None or not isinstance(headers, HttpRequestHeader):
outputfile.write("{Error, cannot parse http request headers.}")
outputfile.write('\n')
reader.skipall()
return
if headers.expect:
# assume it is expect:continue-100
request_status['expect'] = headers
mime, charset = textutils.parse_content_type(headers.content_type)
# usually charset is not set in http post
output_body = parse_config.level >= OutputLevel.ALL_BODY and not textutils.isbinarybody(mime) \
or parse_config.level >= OutputLevel.TEXT_BODY and textutils.istextbody(mime)
content = ''
# deal with body
if not headers.chunked:
if output_body:
content = reader.read(headers.content_len)
else:
reader.skip(headers.content_len)
else:
content = read_chunked_body(reader)
if not headers.gzip:
# if is gzip by content magic header
# someone missed the content-encoding header
headers.gzip = textutils.isgzip(content)
# if it is form url encode
if 'expect' in request_status and not content:
content = '{Expect-continue-100, see next content for http post body}'
if output_body:
#unescape www-form-encoded data.x-www-form-urlencoded
if parse_config.encoding and not charset:
charset = parse_config.encoding
print_body(content, headers.gzip, charset, outputfile, mime and 'form-urlencoded' in mime, parse_config.pretty)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("pcap_file", help="the pcap file to parse")
parser.add_argument(
"-i",
"--ip",
help="only parse packages with specified source OR dest ip")
parser.add_argument(
"-p",
"--port",
type=int,
help="only parse packages with specified source OR dest port")
parser.add_argument("-v",
"--verbosity",
help="increase output verbosity(-vv is recommended)",
action="count")
parser.add_argument("-o",
"--output",
help="output to file instead of stdout")
parser.add_argument("-e",
"--encoding",
help="decode the data use specified encodings.")
parser.add_argument("-b",
"--beauty",
help="output json in a pretty way.",
action="store_true")
args = parser.parse_args()
filepath = args.pcap_file
port = args.port
ip = args.ip
if args.verbosity:
parse_config.level = args.verbosity
if args.encoding:
parse_config.encoding = args.encoding
parse_config.pretty = args.beauty
if args.output:
outputfile = open(args.output, "w+")
else:
outputfile = sys.stdout
with open(filepath, "rb") as pcap_file:
conn_dict = OrderedDict()
for tcp_pac in pcap.read_package_r(pcap_file):
#filter
if port is not None and tcp_pac.source_port != port and tcp_pac.dest_port != port:
continue
if ip is not None and tcp_pac.source != ip and tcp_pac.dest != ip:
continue
key = tcp_pac.gen_key()
# we already have this conn
if key in conn_dict:
conn_dict[key].append(tcp_pac)
# conn closed.
if tcp_pac.pac_type == pcap.TcpPack.TYPE_CLOSE:
conn_dict[key].finish()
del conn_dict[key]
# begin tcp connection.
elif tcp_pac.pac_type == 1:
conn_dict[key] = HttpConn(tcp_pac, outputfile)
elif tcp_pac.pac_type == 0:
# tcp init before capature, we found a http request header, begin parse
# if is a http request?
if textutils.ishttprequest(tcp_pac.body):
conn_dict[key] = HttpConn(tcp_pac, outputfile)
for conn in conn_dict.values():
conn.finish()
if args.output:
outputfile.close()
def read_http_headers(reader, outputfile, level):
"""read & parse http headers"""
line = reader.fetchline()
if line is None:
return line
line = line.strip()
if textutils.ishttprequest(line):
headers = HttpRequestHeader()
items = line.split(' ')
if len(items) == 3:
headers.method = items[0]
headers.uri = items[1]
if level == OutputLevel.ONLY_URL:
outputfile.write(line)
outputfile.write('\n')
elif textutils.ishttpresponse(line):
headers = HttpReponseHeader()
items = line.split(' ')
if len(items) == 3:
headers.status_code = items[1]
headers.protocal = items[0]
if level == OutputLevel.ONLY_URL:
outputfile.write(line)
outputfile.write('\n')
else:
# not httprequest or httpresponse
return None
if level >= OutputLevel.HEADER:
outputfile.write(line)
outputfile.write('\n')
reader.readline()
header_dict = defaultdict(str)
while True:
line = reader.readline()
if line is None:
break
line = line.strip()
if not line:
break
if level >= OutputLevel.HEADER:
outputfile.write(line)
outputfile.write('\n')
key, value = textutils.parse_http_header(line)
if key is None:
# incorrect headers.
continue
header_dict[key.lower()] = value
if "content-length" in header_dict:
headers.content_len = int(header_dict["content-length"])
if 'chunked' in header_dict["transfer-encoding"]:
headers.chunked = True
headers.content_type = header_dict['content-type']
headers.gzip = ('gzip' in header_dict["content-encoding"])
headers.host = header_dict["host"]
headers.connectionclose = (header_dict['connection'] == 'close')
if 'expect' in header_dict:
headers.expect = header_dict['expect']
if level >= OutputLevel.HEADER:
outputfile.write('\n')
return headers
def read_http_headers(reader, outputfile, level):
"""read & parse http headers"""
line = reader.fetchline()
if line is None:
return line
line = line.strip()
if textutils.ishttprequest(line):
headers = HttpRequestHeader()
items = line.split(' ')
if len(items) == 3:
headers.method = items[0]
headers.uri = items[1]
if level == OutputLevel.ONLY_URL:
outputfile.write(line)
outputfile.write('\n')
elif textutils.ishttpresponse(line):
headers = HttpReponseHeader()
items = line.split(' ')
if len(items) == 3:
headers.status_code = items[1]
headers.protocal = items[0]
if level == OutputLevel.ONLY_URL:
outputfile.write(line)
outputfile.write('\n')
else:
# not httprequest or httpresponse
return None
if level >= OutputLevel.HEADER:
outputfile.write(line)
outputfile.write('\n')
reader.readline()
header_dict = defaultdict(str)
while True:
line = reader.readline()
if line is None:
break
line = line.strip()
if not line:
break
if level >= OutputLevel.HEADER:
outputfile.write(line)
outputfile.write('\n')
key, value = textutils.parse_http_header(line)
if key is None:
# incorrect headers.
continue
header_dict[key.lower()] = value
if "content-length" in header_dict:
headers.content_len = int(header_dict["content-length"])
if 'chunked' in header_dict["transfer-encoding"]:
headers.chunked = True
headers.content_type = header_dict['content-type']
headers.gzip = ('gzip' in header_dict["content-encoding"])
headers.host = header_dict["host"]
headers.connectionclose = (header_dict['connection'] == 'close')
if 'expect' in header_dict:
headers.expect = header_dict['expect']
if level >= OutputLevel.HEADER:
outputfile.write('\n')
return headers
