forked from yandex/tcplanz
-
Notifications
You must be signed in to change notification settings - Fork 0
/
decode-pcap.py
executable file
·157 lines (111 loc) · 4.49 KB
/
decode-pcap.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
#!/usr/bin/env pypy
# Turns a pcap file with http gzip compressed data into plain text, making it
# easier to follow.
import dpkt
import gzip
import sys
import os
import signal
from multiprocessing import Pool
from litesession import pcap_lite_sessions, packet_parse_helper
from tcpsession import TCPSession, tcp_flags
from httpsession import parse_http_streams, HTTPParsingError, HTTPResponse, HTTPRequest
import printing, parsing
from parsing import handle_lite_tcp_session
from refactoring.parsing_ng import handle_lite_tcp_session_ng
def pcap_stream(filenames, ports):
file_num = 0
for filename in filenames:
packet_num = 0
ignored = 0
# Open the pcap file
if filename[-8:]=='.pcap.gz':
f = gzip.open(filename, 'rb')
elif filename[-5:]=='.pcap':
f = open(filename, 'rb')
else:
assert False, "Don't know how to parse file: "+filename
print >> sys.stderr, file_num, filename
with f:
pcap = dpkt.pcap.Reader(f)
for ts, buf in pcap:
packet_num += 1
result = packet_parse_helper(pcap.datalink(),buf)
if result is not None and (ports is None or result[1].dport in ports or result[1].sport in ports):
yield (ts, file_num*1000000000 + packet_num, buf, result[0], result[1])
else:
ignored += 1
result = None
print >> sys.stderr, "total packets(%s): %s, ignored %s" % (filename, packet_num, ignored)
file_num += 1
def parse_pcap_files(filenames, ng = False):
print >> printing.http_stream, printing.header()
for lite_tcp_session in pcap_lite_sessions(pcap_stream(filenames,ports)):
if ng:
handle_lite_tcp_session_ng(lite_tcp_session)
else:
handle_lite_tcp_session(lite_tcp_session)
def parse_splitted_files(filenames, ng = False):
print >> printing.http_stream, printing.header()
for filename in sorted(filenames):
for lite_tcp_session in pcap_lite_sessions(pcap_stream([filename],ports)):
if ng:
handle_lite_tcp_session_ng(lite_tcp_session)
else:
handle_lite_tcp_session(lite_tcp_session)
def split_pcap_files(filenames):
f = None
pcap = None
fnum = 0
num = 0
for lite_tcp_session in pcap_lite_sessions(pcap_stream(filenames,ports)):
if f is None or num > 1000000:
if pcap:
pcap.close()
if f:
f.close()
fnum += 1
num %= 1000000
f=gzip.open(outdir+"/pcap%03d.pcap.gz" % fnum, 'wb')
pcap = dpkt.pcap.Writer(f)
print >> sys.stderr, outdir+"/pcap%03d.pcap.gz" % fnum
for ts,buf in lite_tcp_session.raw_packets():
pcap.writepkt(buf,ts)
num += 1
if pcap:
pcap.close()
if f:
f.close()
if __name__ == '__main__':
import sys
if len(sys.argv) <= 3:
print "%s outdir (parse|sparse|split) <pcap filename(s)>" % sys.argv[0]
sys.exit(2)
new_generation = 'ng' in sys.argv[0]
outdir = sys.argv[1]
printing.debug_stream = open(outdir+"/debug.tmp","w")
ports = set([80,8080]) # set([80,8080]) #uncomment this if yu don't need all traffic
if sys.argv[2]=="parse":
assert os.path.isdir(outdir), "%s is not a directory" % outdir
printing.http_stream = open(outdir+"/http.tmp","w")
parse_pcap_files(sorted(sys.argv[3:]), ng=new_generation )
printing.http_stream.close()
os.rename(outdir+"/http.tmp",outdir+"/http.txt")
if printing.debug_stream:
printing.debug_stream.close()
printing.debug_stream=None
os.rename(outdir+"/debug.tmp",outdir+"/debug.txt")
elif sys.argv[2]=="sparse":
assert os.path.isdir(outdir), "%s is not a directory" % outdir
printing.http_stream = open(outdir+"/http.tmp","w")
parse_splitted_files(sorted(sys.argv[3:]), ng = new_generation)
printing.http_stream.close()
os.rename(outdir+"/http.tmp",outdir+"/http.txt")
if printing.debug_stream:
printing.debug_stream.close()
printing.debug_stream=None
os.rename(outdir+"/debug.tmp",outdir+"/debug.txt")
elif sys.argv[2]=="split":
split_pcap_files(sorted(sys.argv[3:]))
else:
print "%s outdir (parse|sparse|split) <pcap filename(s)>" % sys.argv[0]