def make_tcp_sessions_ng(session):
connection = None # key == directed_key
reverse_connection = None
for ip,tcp in session:
directed_key = TCPSession.directed_key(ip.src,ip.dst,tcp.sport,tcp.dport)
not_repeat = None
while not not_repeat:
if not connection:
connection=TCPSession(directed_key)
reversed_key = TCPSession.directed_key(ip.dst,ip.src,tcp.dport,tcp.sport)
reverse_connection=TCPSession(reversed_key)
connection.pair = reverse_connection
reverse_connection.pair = connection
tcp.string_flags = tcp_flags(tcp.flags)
#tcp.partof=set()
if directed_key == connection.directed_key:
not_repeat=connection.packet(tcp)
elif directed_key == reverse_connection.directed_key:
not_repeat=reverse_connection.packet(tcp)
else:
assert False
if not not_repeat:
yield (connection,reverse_connection)
connection=None
reverse_connection=None
yield (connection,reverse_connection)
def handle_lite_tcp_session(lite_tcp_session):
unpacked_content = list(lite_tcp_session.packets())
try:
for http_session, c, r, conn, revconn in parse_session(
unpacked_content):
direction_hint = None
for i in range(0, len(http_session) - 1):
http_request = http_session[i]
http_response = http_session[i + 1]
#assert isinstance(http_request,HTTPRequest) or isinstance(http_request,HTTPResponse)
if isinstance(http_request, HTTPRequest) and isinstance(
http_response, HTTPResponse):
print_results(printing.http_stream, http_request,
http_response)
if direction_hint is None:
direction_hint = http_response.directed_key
ssl_parsed = False
#HACK for SB. Change to config file.
if direction_hint is None:
src, sport, dst, dport = TCPSession.split_key(
conn.directed_key)
if (dst[:3] == "10." or dst[:2]
== "fd") and not (src[:3] == "10." or src[:2] == "fd"):
direction_hint = revconn.directed_key
src, sport, dst, dport = TCPSession.split_key(
revconn.directed_key)
if (dst[:3] == "10." or dst[:2]
== "fd") and not (src[:3] == "10." or src[:2] == "fd"):
direction_hint = conn.directed_key
if printing.debug_stream:
print_tcp_session(conn.content + revconn.content, c, r,
direction_hint, lite_tcp_session.num)
http_session = None
#tcp_session=None
conn.cleanup()
revconn.cleanup()
unpacked_content = None
c = None
r = None
except (SeqException) as e:
#except (SeqException,SSLSkipError) as e:
print >> sys.stderr, e
def print_results(http_stream,http_request,http_response):
request_start_time=http_request.start_packet.ts
request_end_time=http_request.finish_packet.ts
response_start_time=http_response.start_packet.ts
response_end_time=http_response.finish_packet.ts
response_start_acked=http_response.start_acked_by
if response_start_acked:
response_start_acked="%lf" % response_start_acked.ts
response_end_acked=http_response.finish_acked_by
if response_end_acked:
response_end_acked="%lf" % response_end_acked.ts
request_start_acked=http_request.start_acked_by
if request_start_acked:
request_start_acked="%lf" % request_start_acked.ts
request_end_acked=http_request.finish_acked_by
if request_end_acked:
request_end_acked="%lf" % request_end_acked.ts
request_metrics = "%s\t%s\t%s" % (http_request.num_packets, http_request.total_len, http_request.real_len)
response_metrics = "%s\t%s\t%s" % (http_response.num_packets, http_response.total_len, http_response.real_len)
server,server_port,client,client_port = TCPSession.split_key(http_response.directed_key)
server_as = None
server_24 = None
client_as = None
client_24 = None
if http_request.user_agent:
http_request.user_agent=http_request.user_agent.replace('\000','')
http_request.uri=http_request.uri.replace('\000','')
http_request.user_agent=http_request.user_agent.replace('"','')
http_request.uri=http_request.uri.replace('"','')
http_request.user_agent=http_request.user_agent.replace('\t',' ')
http_request.uri=http_request.uri.replace('\t',' ')
r = (server,server_port,client,client_port,http_request.method,http_request.uri,http_response.method,http_response.status,http_response.reqid,\
request_start_time,request_end_time,response_start_time,response_end_time,response_start_acked,response_end_acked,\
request_start_acked,request_end_acked,http_request.retransmits,http_request.false_retransmits,http_request.keepalive_retransmits,
http_response.retransmits, http_response.false_retransmits, http_response.keepalive_retransmits, \
http_request.min_rtt, http_request.median_rtt, http_request.max_rtt, http_response.min_rtt, http_response.median_rtt, http_response.max_rtt,\
http_request.avg_retr_time, http_response.avg_retr_time,\
request_metrics,response_metrics,http_request.user_agent,http_request.host,
server_as,server_24,client_as,client_24)
def fmt(t):
if t is float:
return "%lf" % t
else:
return str(t)
r = map(fmt, r)
print >> http_stream, '\t'.join(r)
def append(self, ts, num, buf, ip, tcp, track_end):
#tmp
tcp.num = num
tcp.ts = ts
if self.start_ts is None:
self.start_ts = tcp.ts
self.fastkey = TCPSession.nondirected_key(ip.src, ip.dst,
tcp.sport, tcp.dport)
#else:
# assert tcp.ts >= self.start_ts, "ts=%.6f startts=%.6f" % (tcp.ts,self.start_ts)
direction_less = ip.src < ip.dst
if tcp.flags & dpkt.tcp.TH_SYN:
if self.syn_seq is None:
self.syn_seq = tcp.seq
if track_end:
if tcp.flags & dpkt.tcp.TH_SYN and not tcp.flags & dpkt.tcp.TH_ACK:
if self.syn_seq is not None:
if self.syn_seq != tcp.seq:
return True, None
if self.last_ts and (tcp.ts - self.last_ts > 30):
prev_seq = None
if direction_less:
prevseq = self.last_seq_less
else:
prevseq = self.last_seq_more
if prev_seq is not None:
delta = ctypes.c_uint32(tcp.seq - prev_seq).value
if delta >= 0x80000000:
#retransmit - packet with previous seq
delta = ctypes.c_uint32(prev_seq - tcp.seq).value
if delta > 0x00800000:
#reasonable large window
return True, None
#this means we have started new TCP stream
if direction_less:
self.last_seq_less = tcp.seq
else:
self.last_seq_more = tcp.seq
self.last_ts = ts
ts_num = struct.pack("dQ", ts, num)
assert len(ts_num) == 16
self.packed_content.append(ts_num + buf)
return False, 16 + len(buf)
def append(self,ts,num,buf,ip,tcp,track_end):
#tmp
tcp.num=num
tcp.ts=ts
if self.start_ts is None:
self.start_ts = tcp.ts
self.fastkey = TCPSession.nondirected_key(ip.src,ip.dst,tcp.sport,tcp.dport)
#else:
# assert tcp.ts >= self.start_ts, "ts=%.6f startts=%.6f" % (tcp.ts,self.start_ts)
direction_less = ip.src < ip.dst
if tcp.flags & dpkt.tcp.TH_SYN:
if self.syn_seq is None:
self.syn_seq=tcp.seq
if track_end:
if tcp.flags & dpkt.tcp.TH_SYN and not tcp.flags & dpkt.tcp.TH_ACK:
if self.syn_seq is not None:
if self.syn_seq != tcp.seq:
return True, None
if self.last_ts and (tcp.ts-self.last_ts > 30):
prev_seq=None
if direction_less:
prevseq = self.last_seq_less
else:
prevseq = self.last_seq_more
if prev_seq is not None:
delta = ctypes.c_uint32(tcp.seq-prev_seq).value
if delta >= 0x80000000:
#retransmit - packet with previous seq
delta = ctypes.c_uint32(prev_seq-tcp.seq).value
if delta > 0x00800000:
#reasonable large window
return True, None
#this means we have started new TCP stream
if direction_less:
self.last_seq_less = tcp.seq
else:
self.last_seq_more = tcp.seq
self.last_ts=ts
ts_num=struct.pack("dQ",ts,num)
assert len(ts_num)==16
self.packed_content.append(ts_num+buf)
return False, 16+len(buf)
def __str__(self):
try:
r = str(len(self.content()))
except (RecordClassError) as err:
r = str(err)
src, sport, dst, dport = TCPSession.split_key(self.direction())
return "\t".join([str(s) for s in (src, sport, dst, dport, len(self.packets), r, self.first.ts, self.last.ts)])
def pcap_lite_sessions(stream, track_end=True):
connections = defaultdict(LiteSession)
total_bytes = 0
total_connections = 0
last_print = 0
total_packets = 0
packets_ignored = 0
for ts, num, buf, ip, tcp in stream:
ended = True
total_packets += 1
fastkey = TCPSession.nondirected_key(ip.src, ip.dst, tcp.sport,
tcp.dport)
while ended:
connection = connections[fastkey]
if isinstance(connection, int):
connection = LiteSession(connection)
connections[fastkey] = connection
ended, data_increment = connection.append(ts, num, buf, ip, tcp,
track_end)
if ended and track_end:
bytes = sum([len(s) for s in connection.packed_content])
yield connection
#print >> sys.stderr, "splitting %s [%d] connection total bytes %d" % (TCPSession.display_key(fastkey),connection.num,bytes)
total_bytes -= bytes
connections[fastkey] = connection.num + 1
total_connections += 1
total_bytes += data_increment
if last_print != total_bytes / 1024 / 1024:
last_print = total_bytes / 1024 / 1024
print >> sys.stderr, "packets:", total_packets, ", conn:", total_connections, ", data:", last_print
#print >> sys.stderr, "total packets %s, ignored %s, total connections %s" % (total_packets,packets_ignored,total_connections)
#return unfinished connections as well
for fastkey in connections.iterkeys():
connection = connections[fastkey]
if connection:
yield connections[fastkey]
total_connections += 1
connections[fastkey] = None
total_bytes -= sum([len(s) for s in connection.packed_content])
if last_print != total_bytes / 1024 / 1024:
last_print = total_bytes / 1024 / 1024
print >> sys.stderr, "packets:", total_packets, ", conn:", total_connections, ", data:", last_print
def pcap_lite_sessions(stream, track_end=True):
connections = defaultdict(LiteSession)
total_bytes = 0
total_connections = 0
last_print=0
total_packets = 0
packets_ignored = 0
for ts,num,buf,ip,tcp in stream:
ended = True
total_packets += 1
fastkey = TCPSession.nondirected_key(ip.src,ip.dst,tcp.sport,tcp.dport)
while ended:
connection=connections[fastkey]
if isinstance(connection,int):
connection=LiteSession(connection)
connections[fastkey]=connection
ended,data_increment = connection.append(ts,num,buf,ip,tcp,track_end)
if ended and track_end:
bytes=sum([len(s) for s in connection.packed_content])
yield connection
#print >> sys.stderr, "splitting %s [%d] connection total bytes %d" % (TCPSession.display_key(fastkey),connection.num,bytes)
total_bytes -= bytes
connections[fastkey]=connection.num+1
total_connections += 1
total_bytes += data_increment
if last_print!=total_bytes/1024/1024:
last_print=total_bytes/1024/1024
print >> sys.stderr, "packets:", total_packets, ", conn:", total_connections, ", data:", last_print
#print >> sys.stderr, "total packets %s, ignored %s, total connections %s" % (total_packets,packets_ignored,total_connections)
#return unfinished connections as well
for fastkey in connections.iterkeys():
connection = connections[fastkey]
if connection:
yield connections[fastkey]
total_connections += 1
connections[fastkey]=None
total_bytes -= sum([len(s) for s in connection.packed_content])
if last_print!=total_bytes/1024/1024:
last_print=total_bytes/1024/1024
print >> sys.stderr, "packets:", total_packets, ", conn:", total_connections, ", data:", last_print
def __str__(self): direction = TCPSession.display_key(self.direction) if self.direction else None errstr = "%s: %s" % (self.__class__.__name__, self.what) if direction: errstr += ", at %s" % direction if len(self.packets)>0: errstr += ", at %s" % [p.num for p in self.packets] return errstr
def __str__(self):
try:
r = str(len(self.content()))
except (RecordClassError) as err:
r = str(err)
src, sport, dst, dport = TCPSession.split_key(self.direction())
return '\t'.join([
str(s) for s in (src, sport, dst, dport, len(self.packets), r,
self.first.ts, self.last.ts)
])
def print_tcp_session(packets, connection_stream, reverse_connection_stream,
direction_hint, num):
packets = sorted(packets, key=lambda x: x[1].ts)
prev_packet = None
if direction_hint:
server, server_port, client, client_port = TCPSession.split_key(
direction_hint)
print >> debug_stream, "server: %s:%s <-> client %s:%s" % (
server, server_port, client, client_port)
for seq, tcp in packets:
prefix = ''
if prev_packet is not None:
ts_delta = int((tcp.ts - prev_packet.ts) * 1000)
if ts_delta > 5:
prefix = ".." + str(ts_delta) + ".."
prefix = "%10s" % prefix
if getattr(tcp, 'unknown_start', False):
prefix = '>' + prefix
else:
prefix = ' ' + prefix
print >> debug_stream, prefix,
print_packet(debug_stream, tcp, direction_hint)
prev_packet = tcp
if connection_stream:
for real, seq, data, comment in connection_stream:
l = len(data)
data = [c for c in data[:1000] if c >= ' ' and c < 'z']
data = ''.join(data)
print >> debug_stream, "->", real, seq, comment, l, "[ " + data[:
100] + " ]"
if reverse_connection_stream:
for real, seq, data, comment in reverse_connection_stream:
l = len(data)
data = [c for c in data if c >= ' ' and c < 'z']
data = ''.join(data)
print >> debug_stream, "<-", real, seq, comment, l, "[ " + data[:
100] + " ]"
print >> debug_stream, ""
def print_tcp_session(packets,connection_stream,reverse_connection_stream,direction_hint, num):
packets = sorted(packets, key = lambda x: x[1].ts)
prev_packet = None
if direction_hint:
server,server_port,client,client_port = TCPSession.split_key(direction_hint)
print >> debug_stream, "server: %s:%s <-> client %s:%s" % (server,server_port,client,client_port)
for seq,tcp in packets:
prefix = ''
if prev_packet is not None:
ts_delta = int((tcp.ts - prev_packet.ts)*1000)
if ts_delta > 5:
prefix = ".." + str(ts_delta) + ".."
prefix = "%10s" % prefix
if getattr(tcp,'unknown_start',False):
prefix = '>' + prefix
else:
prefix = ' ' + prefix
print >> debug_stream, prefix,
print_packet(debug_stream,tcp,direction_hint)
prev_packet=tcp
if connection_stream:
for real,seq,data,comment in connection_stream:
l = len(data)
data = [ c for c in data[:1000] if c>=' ' and c < 'z' ]
data = ''.join(data)
print >> debug_stream, "->", real, seq, comment, l, "[ "+data[:100]+" ]"
if reverse_connection_stream:
for real,seq,data,comment in reverse_connection_stream:
l = len(data)
data = [ c for c in data if c>=' ' and c < 'z' ]
data = ''.join(data)
print >> debug_stream, "<-", real, seq, comment, l, "[ "+data[:100]+" ]"
print >> debug_stream, ""
def parse_http_streams(tupl, data, reverse_content, content):
if len(data)==0:
return
all_packets = [tcp for seq,tcp in content]
for real, start_seq, stream, comment in data:
if real == False:
continue
try:
http = None
pp = list()
start_packet = TCPSession.in_packet(start_seq, content)[0]
start_acked_by = start_packet.acked_by
#assert start_packet is not None, "No begin packet. Wtf? Teardrop? seq=%d" % start_seq
if start_packet is None:
raise HTTPParsingError(what="No begin packet. Wtf? Teardrop?", packet_num=0, seq=start_seq, content=content)
assert len(start_packet.data)>0
if stream[:len(start_packet.data)]!=start_packet.data:
#could be double data in different packets - we don't really know whish one is in effect
#print >> sys.stderr, tupl, "Diff packets(not error)", current_seq-1, start_packet.seq, len(start_packet.data), start_packet.data
pass
if stream[:4] == 'HTTP':
http = HTTPResponse(stream,start_packet.num,tupl)
elif stream[:4] == 'POST' or stream[:4] == 'HEAD' or stream[:3] == 'GET':
http = HTTPRequest(stream,start_packet.num,tupl)
else:
raise HTTPParsingError(what="wtf is this data", packet_num=start_packet.num, seq=start_seq, stream=stream)
if len(http.data)>0:
raise HTTPParsingError(what="data after end of parsing", packet_num=start_packet.num, seq=start_seq+len(stream)-len(http.data), stream=http.data)
end_seq = start_seq + len(stream)-1
finish_packet = TCPSession.in_packet(end_seq, content)[0]
finish_acked_by = finish_packet.acked_by
#assert finish_packet is not None, "No end packet. Wtf? Teardrop? seq=%s" % (current_seq-1)
if finish_packet is None:
raise HTTPParsingError(what="No end packet. Wtf? Teardrop?", packet_num=0, seq=end_seq, content=content)
assert len(finish_packet.data)>0
http.start_acked_by=start_acked_by
http.start_packet=start_packet
http.finish_acked_by=finish_acked_by
http.finish_packet=finish_packet
http.start_seq=start_seq
http.end_seq=end_seq
rtt=TCPSession.rtt(content,start_seq,end_seq)
if rtt is not None:
http.min_rtt=rtt[0]
http.median_rtt=rtt[1]
http.max_rtt=rtt[2]
else:
http.min_rtt=None
http.median_rtt=None
http.max_rtt=None
retransmits, false_retransmits, keepalive_retransmits, avg_retr_time, =TCPSession.retransmits(content,start_seq,end_seq,http.min_rtt)
http.retransmits=retransmits
http.false_retransmits=false_retransmits
http.keepalive_retransmits = keepalive_retransmits
http.avg_retr_time = avg_retr_time
http_start_index = all_packets.index(http.start_packet)
http_end_index = all_packets.index(http.finish_packet)
assert http_start_index < len(all_packets) and http_start_index is not None
total_packets = 0
total_len = http.finish_packet.adjusted_seq-http.start_packet.adjusted_seq+len(http.finish_packet.data)
real_len = 0
for tcp in all_packets[http_start_index:http_end_index+1]:
if len(tcp.data)>0:
total_packets += 1
real_len += len(tcp.data)
if not hasattr(http.finish_packet,'partof'):
http.finish_packet.partof=defaultdict(str)
if not hasattr(http.start_packet,'partof'):
http.start_packet.partof=defaultdict(str)
http.start_packet.partof[http] += "<"
http.finish_packet.partof[http] += ">"
http.num_packets=total_packets
http.total_len=total_len
http.real_len=real_len
yield http
except (dpkt.UnpackError) as err:
yield HTTPParsingError(what=str(err), packet_num=start_packet.num, etype=err.__class__.__name__)
except (ValueError,TypeError,IndexError) as err:
#TODO: check all this errors thrown from HTTP Parser
yield HTTPParsingError(what=str(err), packet_num=start_packet.num, etype="ValueError")
except HTTPParsingError as err:
yield err
def print_results(http_stream, http_request, http_response):
request_start_time = http_request.start_packet.ts
request_end_time = http_request.finish_packet.ts
response_start_time = http_response.start_packet.ts
response_end_time = http_response.finish_packet.ts
response_start_acked = http_response.start_acked_by
if response_start_acked:
response_start_acked = "%lf" % response_start_acked.ts
response_end_acked = http_response.finish_acked_by
if response_end_acked:
response_end_acked = "%lf" % response_end_acked.ts
request_start_acked = http_request.start_acked_by
if request_start_acked:
request_start_acked = "%lf" % request_start_acked.ts
request_end_acked = http_request.finish_acked_by
if request_end_acked:
request_end_acked = "%lf" % request_end_acked.ts
request_metrics = "%s\t%s\t%s" % (http_request.num_packets,
http_request.total_len,
http_request.real_len)
response_metrics = "%s\t%s\t%s" % (http_response.num_packets,
http_response.total_len,
http_response.real_len)
server, server_port, client, client_port = TCPSession.split_key(
http_response.directed_key)
server_as = None
server_24 = None
client_as = None
client_24 = None
if http_request.user_agent:
http_request.user_agent = http_request.user_agent.replace('\000', '')
http_request.uri = http_request.uri.replace('\000', '')
http_request.user_agent = http_request.user_agent.replace('"', '')
http_request.uri = http_request.uri.replace('"', '')
http_request.user_agent = http_request.user_agent.replace('\t', ' ')
http_request.uri = http_request.uri.replace('\t', ' ')
r = (server,server_port,client,client_port,http_request.method,http_request.uri,http_response.method,http_response.status,http_response.reqid,\
request_start_time,request_end_time,response_start_time,response_end_time,response_start_acked,response_end_acked,\
request_start_acked,request_end_acked,http_request.retransmits,http_request.false_retransmits,http_request.keepalive_retransmits,
http_response.retransmits, http_response.false_retransmits, http_response.keepalive_retransmits, \
http_request.min_rtt, http_request.median_rtt, http_request.max_rtt, http_response.min_rtt, http_response.median_rtt, http_response.max_rtt,\
http_request.avg_retr_time, http_response.avg_retr_time,\
request_metrics,response_metrics,http_request.user_agent,http_request.host,
server_as,server_24,client_as,client_24)
def fmt(t):
if t is float:
return "%lf" % t
else:
return str(t)
r = map(fmt, r)
print >> http_stream, '\t'.join(r)
def print_packet(file, tcp, direction_hint=None):
if direction_hint is None:
display_key = TCPSession.split_key(tcp.connection.directed_key)
display_key = " %s : %s -> %s : %s " % display_key
else:
if direction_hint == tcp.connection.directed_key:
display_key = " --> "
else:
display_key = " <-- "
string_flags = tcp_flags(tcp.flags)
retr = ' '
if tcp.retransmit_original is not None:
retr = 'R'
if 'A' in string_flags:
ack = tcp.adjusted_ack
else:
ack = ''
acked = getattr(tcp, 'acked_by', None)
if acked is not None: acked = acked.num
sacked = getattr(tcp, 'acked_sacked_by', None)
if sacked is not None: sacked = sacked.num
if sacked == acked:
sacked = ''
else:
sacked = "/" + str(sacked)
acked = str(acked)
sacked_acked = ''
if len(tcp.data) > 0 or 'S' in string_flags:
sacked_acked = acked + sacked
rtt = tcp.rtt
if rtt is not None and abs(rtt) > 1000:
rtt = str(rtt / 1000) + 's'
rtt = "[%4s]" % rtt
else:
rtt = ''
adjusted_sack = getattr(tcp, 'adjusted_sack', '')
print >> file, "%10d %7s %f %ls %4d %4s %7ds %7sa %5s %10s %s" % (
tcp.num, rtt, tcp.ts, display_key, len(tcp.data), string_flags,
getattr(tcp, 'adjusted_seq',
-1), ack, retr, sacked_acked, adjusted_sack),
partof = getattr(tcp, 'partof', None)
if partof is not None:
for http, partkind in partof.iteritems():
print >> file, "%2s" % partkind,
print >> file, http.method,
if hasattr(http, 'status'):
print >> file, http.status,
if hasattr(http, 'uri'):
print >> file, http.uri,
if getattr(http, 'reqid', None):
print >> file, http.reqid,
print >> file, ""
def parse_http_streams(tupl, data, reverse_content, content):
if len(data) == 0:
return
all_packets = [tcp for seq, tcp in content]
for real, start_seq, stream, comment in data:
if real == False:
continue
try:
http = None
pp = list()
start_packet = TCPSession.in_packet(start_seq, content)[0]
start_acked_by = start_packet.acked_by
#assert start_packet is not None, "No begin packet. Wtf? Teardrop? seq=%d" % start_seq
if start_packet is None:
raise HTTPParsingError(what="No begin packet. Wtf? Teardrop?",
packet_num=0,
seq=start_seq,
content=content)
assert len(start_packet.data) > 0
if stream[:len(start_packet.data)] != start_packet.data:
#could be double data in different packets - we don't really know whish one is in effect
#print >> sys.stderr, tupl, "Diff packets(not error)", current_seq-1, start_packet.seq, len(start_packet.data), start_packet.data
pass
if stream[:4] == 'HTTP':
http = HTTPResponse(stream, start_packet.num, tupl)
elif stream[:
4] == 'POST' or stream[:
4] == 'HEAD' or stream[:
3] == 'GET':
http = HTTPRequest(stream, start_packet.num, tupl)
else:
raise HTTPParsingError(what="wtf is this data",
packet_num=start_packet.num,
seq=start_seq,
stream=stream)
if len(http.data) > 0:
raise HTTPParsingError(what="data after end of parsing",
packet_num=start_packet.num,
seq=start_seq + len(stream) -
len(http.data),
stream=http.data)
end_seq = start_seq + len(stream) - 1
finish_packet = TCPSession.in_packet(end_seq, content)[0]
finish_acked_by = finish_packet.acked_by
#assert finish_packet is not None, "No end packet. Wtf? Teardrop? seq=%s" % (current_seq-1)
if finish_packet is None:
raise HTTPParsingError(what="No end packet. Wtf? Teardrop?",
packet_num=0,
seq=end_seq,
content=content)
assert len(finish_packet.data) > 0
http.start_acked_by = start_acked_by
http.start_packet = start_packet
http.finish_acked_by = finish_acked_by
http.finish_packet = finish_packet
http.start_seq = start_seq
http.end_seq = end_seq
rtt = TCPSession.rtt(content, start_seq, end_seq)
if rtt is not None:
http.min_rtt = rtt[0]
http.median_rtt = rtt[1]
http.max_rtt = rtt[2]
else:
http.min_rtt = None
http.median_rtt = None
http.max_rtt = None
retransmits, false_retransmits, keepalive_retransmits, avg_retr_time, = TCPSession.retransmits(
content, start_seq, end_seq, http.min_rtt)
http.retransmits = retransmits
http.false_retransmits = false_retransmits
http.keepalive_retransmits = keepalive_retransmits
http.avg_retr_time = avg_retr_time
http_start_index = all_packets.index(http.start_packet)
http_end_index = all_packets.index(http.finish_packet)
assert http_start_index < len(
all_packets) and http_start_index is not None
total_packets = 0
total_len = http.finish_packet.adjusted_seq - http.start_packet.adjusted_seq + len(
http.finish_packet.data)
real_len = 0
for tcp in all_packets[http_start_index:http_end_index + 1]:
if len(tcp.data) > 0:
total_packets += 1
real_len += len(tcp.data)
if not hasattr(http.finish_packet, 'partof'):
http.finish_packet.partof = defaultdict(str)
if not hasattr(http.start_packet, 'partof'):
http.start_packet.partof = defaultdict(str)
http.start_packet.partof[http] += "<"
http.finish_packet.partof[http] += ">"
http.num_packets = total_packets
http.total_len = total_len
http.real_len = real_len
yield http
except (dpkt.UnpackError) as err:
yield HTTPParsingError(what=str(err),
packet_num=start_packet.num,
etype=err.__class__.__name__)
except (ValueError, TypeError, IndexError) as err:
#TODO: check all this errors thrown from HTTP Parser
yield HTTPParsingError(what=str(err),
packet_num=start_packet.num,
etype="ValueError")
except HTTPParsingError as err:
yield err
def print_packet(file, tcp, direction_hint=None):
if direction_hint is None:
display_key = TCPSession.split_key(tcp.connection.directed_key)
display_key = " %s : %s -> %s : %s " % display_key
else:
if direction_hint==tcp.connection.directed_key:
display_key = " --> "
else:
display_key = " <-- "
string_flags = tcp_flags(tcp.flags)
retr = ' '
if tcp.retransmit_original is not None:
retr = 'R'
if 'A' in string_flags:
ack = tcp.adjusted_ack
else:
ack=''
acked=getattr(tcp,'acked_by',None)
if acked is not None: acked=acked.num
sacked=getattr(tcp,'acked_sacked_by',None)
if sacked is not None: sacked=sacked.num
if sacked == acked:
sacked = ''
else:
sacked = "/"+str(sacked)
acked = str(acked)
sacked_acked=''
if len(tcp.data)>0 or 'S' in string_flags:
sacked_acked=acked+sacked
rtt = tcp.rtt
if rtt is not None and abs(rtt)>1000:
rtt = str(rtt/1000)+'s'
rtt = "[%4s]" % rtt
else:
rtt=''
adjusted_sack=getattr(tcp,'adjusted_sack', '')
print >> file, "%10d %7s %f %ls %4d %4s %7ds %7sa %5s %10s %s" % (tcp.num, rtt, tcp.ts, display_key, len(tcp.data), string_flags, getattr(tcp,'adjusted_seq',-1), ack, retr, sacked_acked, adjusted_sack),
partof = getattr(tcp,'partof',None)
if partof is not None:
for http, partkind in partof.iteritems():
print >> file, "%2s" % partkind,
print >> file, http.method,
if hasattr(http,'status'):
print >> file, http.status,
if hasattr(http,'uri'):
print >> file, http.uri,
if getattr(http,'reqid',None):
print >> file, http.reqid,
print >> file, ""