def parse_http_streams(tupl, data, reverse_content, content): if len(data) == 0: return all_packets = [tcp for seq, tcp in content] for real, start_seq, stream, comment in data: if real == False: continue try: http = None pp = list() start_packet = TCPSession.in_packet(start_seq, content)[0] start_acked_by = start_packet.acked_by #assert start_packet is not None, "No begin packet. Wtf? Teardrop? seq=%d" % start_seq if start_packet is None: raise HTTPParsingError(what="No begin packet. Wtf? Teardrop?", packet_num=0, seq=start_seq, content=content) assert len(start_packet.data) > 0 if stream[:len(start_packet.data)] != start_packet.data: #could be double data in different packets - we don't really know whish one is in effect #print >> sys.stderr, tupl, "Diff packets(not error)", current_seq-1, start_packet.seq, len(start_packet.data), start_packet.data pass if stream[:4] == 'HTTP': http = HTTPResponse(stream, start_packet.num, tupl) elif stream[: 4] == 'POST' or stream[: 4] == 'HEAD' or stream[: 3] == 'GET': http = HTTPRequest(stream, start_packet.num, tupl) else: raise HTTPParsingError(what="wtf is this data", packet_num=start_packet.num, seq=start_seq, stream=stream) if len(http.data) > 0: raise HTTPParsingError(what="data after end of parsing", packet_num=start_packet.num, seq=start_seq + len(stream) - len(http.data), stream=http.data) end_seq = start_seq + len(stream) - 1 finish_packet = TCPSession.in_packet(end_seq, content)[0] finish_acked_by = finish_packet.acked_by #assert finish_packet is not None, "No end packet. Wtf? Teardrop? seq=%s" % (current_seq-1) if finish_packet is None: raise HTTPParsingError(what="No end packet. Wtf? Teardrop?", packet_num=0, seq=end_seq, content=content) assert len(finish_packet.data) > 0 http.start_acked_by = start_acked_by http.start_packet = start_packet http.finish_acked_by = finish_acked_by http.finish_packet = finish_packet http.start_seq = start_seq http.end_seq = end_seq rtt = TCPSession.rtt(content, start_seq, end_seq) if rtt is not None: http.min_rtt = rtt[0] http.median_rtt = rtt[1] http.max_rtt = rtt[2] else: http.min_rtt = None http.median_rtt = None http.max_rtt = None retransmits, false_retransmits, keepalive_retransmits, avg_retr_time, = TCPSession.retransmits( content, start_seq, end_seq, http.min_rtt) http.retransmits = retransmits http.false_retransmits = false_retransmits http.keepalive_retransmits = keepalive_retransmits http.avg_retr_time = avg_retr_time http_start_index = all_packets.index(http.start_packet) http_end_index = all_packets.index(http.finish_packet) assert http_start_index < len( all_packets) and http_start_index is not None total_packets = 0 total_len = http.finish_packet.adjusted_seq - http.start_packet.adjusted_seq + len( http.finish_packet.data) real_len = 0 for tcp in all_packets[http_start_index:http_end_index + 1]: if len(tcp.data) > 0: total_packets += 1 real_len += len(tcp.data) if not hasattr(http.finish_packet, 'partof'): http.finish_packet.partof = defaultdict(str) if not hasattr(http.start_packet, 'partof'): http.start_packet.partof = defaultdict(str) http.start_packet.partof[http] += "<" http.finish_packet.partof[http] += ">" http.num_packets = total_packets http.total_len = total_len http.real_len = real_len yield http except (dpkt.UnpackError) as err: yield HTTPParsingError(what=str(err), packet_num=start_packet.num, etype=err.__class__.__name__) except (ValueError, TypeError, IndexError) as err: #TODO: check all this errors thrown from HTTP Parser yield HTTPParsingError(what=str(err), packet_num=start_packet.num, etype="ValueError") except HTTPParsingError as err: yield err
def parse_http_streams(tupl, data, reverse_content, content): if len(data)==0: return all_packets = [tcp for seq,tcp in content] for real, start_seq, stream, comment in data: if real == False: continue try: http = None pp = list() start_packet = TCPSession.in_packet(start_seq, content)[0] start_acked_by = start_packet.acked_by #assert start_packet is not None, "No begin packet. Wtf? Teardrop? seq=%d" % start_seq if start_packet is None: raise HTTPParsingError(what="No begin packet. Wtf? Teardrop?", packet_num=0, seq=start_seq, content=content) assert len(start_packet.data)>0 if stream[:len(start_packet.data)]!=start_packet.data: #could be double data in different packets - we don't really know whish one is in effect #print >> sys.stderr, tupl, "Diff packets(not error)", current_seq-1, start_packet.seq, len(start_packet.data), start_packet.data pass if stream[:4] == 'HTTP': http = HTTPResponse(stream,start_packet.num,tupl) elif stream[:4] == 'POST' or stream[:4] == 'HEAD' or stream[:3] == 'GET': http = HTTPRequest(stream,start_packet.num,tupl) else: raise HTTPParsingError(what="wtf is this data", packet_num=start_packet.num, seq=start_seq, stream=stream) if len(http.data)>0: raise HTTPParsingError(what="data after end of parsing", packet_num=start_packet.num, seq=start_seq+len(stream)-len(http.data), stream=http.data) end_seq = start_seq + len(stream)-1 finish_packet = TCPSession.in_packet(end_seq, content)[0] finish_acked_by = finish_packet.acked_by #assert finish_packet is not None, "No end packet. Wtf? Teardrop? seq=%s" % (current_seq-1) if finish_packet is None: raise HTTPParsingError(what="No end packet. Wtf? Teardrop?", packet_num=0, seq=end_seq, content=content) assert len(finish_packet.data)>0 http.start_acked_by=start_acked_by http.start_packet=start_packet http.finish_acked_by=finish_acked_by http.finish_packet=finish_packet http.start_seq=start_seq http.end_seq=end_seq rtt=TCPSession.rtt(content,start_seq,end_seq) if rtt is not None: http.min_rtt=rtt[0] http.median_rtt=rtt[1] http.max_rtt=rtt[2] else: http.min_rtt=None http.median_rtt=None http.max_rtt=None retransmits, false_retransmits, keepalive_retransmits, avg_retr_time, =TCPSession.retransmits(content,start_seq,end_seq,http.min_rtt) http.retransmits=retransmits http.false_retransmits=false_retransmits http.keepalive_retransmits = keepalive_retransmits http.avg_retr_time = avg_retr_time http_start_index = all_packets.index(http.start_packet) http_end_index = all_packets.index(http.finish_packet) assert http_start_index < len(all_packets) and http_start_index is not None total_packets = 0 total_len = http.finish_packet.adjusted_seq-http.start_packet.adjusted_seq+len(http.finish_packet.data) real_len = 0 for tcp in all_packets[http_start_index:http_end_index+1]: if len(tcp.data)>0: total_packets += 1 real_len += len(tcp.data) if not hasattr(http.finish_packet,'partof'): http.finish_packet.partof=defaultdict(str) if not hasattr(http.start_packet,'partof'): http.start_packet.partof=defaultdict(str) http.start_packet.partof[http] += "<" http.finish_packet.partof[http] += ">" http.num_packets=total_packets http.total_len=total_len http.real_len=real_len yield http except (dpkt.UnpackError) as err: yield HTTPParsingError(what=str(err), packet_num=start_packet.num, etype=err.__class__.__name__) except (ValueError,TypeError,IndexError) as err: #TODO: check all this errors thrown from HTTP Parser yield HTTPParsingError(what=str(err), packet_num=start_packet.num, etype="ValueError") except HTTPParsingError as err: yield err