def read_request(self, reader, message): """ read and output one http request. """ if message.expect_header and not utils.is_request(reader.fetchline()): req_header = message.expect_header message.expect_header = None else: req_header = self.read_http_req_header(reader) if req_header is None: # read header error, we skip all data. reader.skipall() return if req_header.expect: # it is expect:continue-100 post request message.expect_header = req_header # deal with body if not req_header.chunked: content = reader.read(req_header.content_len) else: content = self.read_chunked_body(reader) _filter = config.get_filter() show = _filter.by_domain(req_header.host) and _filter.by_uri( req_header.uri) message.filtered = not show if show: self.processor.on_http_req(req_header, content)
def read_request(self, reader, message): """ read and output one http request. """ if message.expect_header and not utils.is_request(reader.fetchline()): req_header = message.expect_header message.expect_header = None else: req_header = self.read_http_req_header(reader) if req_header is None: # read header error, we skip all data. reader.skipall() return if req_header.expect: # it is expect:continue-100 post request message.expect_header = req_header # deal with body if not req_header.chunked: content = reader.read(req_header.content_len) else: content = self.read_chunked_body(reader) _filter = config.get_filter() show = _filter.by_domain(req_header.host) and _filter.by_uri(req_header.uri) message.filtered = not show if show: self.processor.on_http_req(req_header, content)
def main(): parser = argparse.ArgumentParser() parser.add_argument("infile", nargs='?', help="the pcap file to parse") parser.add_argument("-i", "--ip", help="only parse packages with specified source OR dest ip") parser.add_argument("-p", "--port", type=int, help="only parse packages with specified source OR dest port") parser.add_argument("-v", "--verbosity", help="increase output verbosity(-vv is recommended)", action="count") parser.add_argument("-g", "--group", help="group http request/response by connection", action="store_true") parser.add_argument("-o", "--output", help="output to file instead of stdout") parser.add_argument("-e", "--encoding", help="decode the data use specified encodings.") parser.add_argument("-b", "--beauty", help="output json in a pretty way.", action="store_true") parser.add_argument("-d", "--domain", help="filter http data by request domain") parser.add_argument("-u", "--uri", help="filter http data by request uri pattern") args = parser.parse_args() file_path = "-" if args.infile is None else args.infile _filter = config.get_filter() _filter.ip = args.ip _filter.port = args.port _filter.domain = args.domain _filter.uri_pattern = args.uri # deal with configs parse_config = config.get_config() if args.verbosity: parse_config.level = args.verbosity if args.encoding: parse_config.encoding = args.encoding parse_config.pretty = args.beauty parse_config.group = args.group if args.output: output_file = open(args.output, "w+") else: output_file = sys.stdout config.out = output_file conn_dict = OrderedDict() try: if file_path != '-': infile = io.open(file_path, "rb") else: infile = sys.stdin try: pcap_file(conn_dict, infile) finally: infile.close() finally: for conn in conn_dict.values(): conn.finish() if args.output: output_file.close()
def handle_tcp(c): filter = config.get_filter() for tcp in get_tcpconn(c.infile): if filter.index != None and tcp.index not in filter.index: continue tcp_msg = "\033[31;2m%s [%s:%d] -- -- --> [%s:%d]\033[0m\n" % \ (tcp.index, tcp.con_tuple[0], tcp.con_tuple[1], tcp.con_tuple[2], tcp.con_tuple[3]) utils.print(tcp_msg)
def pcap_file(conn_dict, infile): """ :type conn_dict: dict :type infile:file """ file_format, head = get_file_format(infile) if file_format == FileFormat.PCAP: pcap_file = pcap.PcapFile(infile, head).read_packet elif file_format == FileFormat.PCAP_NG: pcap_file = pcapng.PcapngFile(infile, head).read_packet else: print("unknown file format.", file=sys.stderr) sys.exit(1) _filter = config.get_filter() for tcp_pac in packet_parser.read_package_r(pcap_file): # filter # get time if CTCore.activity_date_time == "": CTCore.activity_date_time = datetime.fromtimestamp( int(str(tcp_pac.micro_second)[:10])) if CTCore.client.headers["IP"] == "": CTCore.client.headers["IP"] = tcp_pac.source if CTCore.client.headers["MAC"] == "": CTCore.client.headers["MAC"] = tcp_pac.src_mac if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)): continue if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)): continue key = tcp_pac.gen_key() # we already have this conn if key in conn_dict: conn_dict[key].append(tcp_pac) # conn closed. if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE: conn_dict[key].finish() del conn_dict[key] # begin tcp connection. elif tcp_pac.pac_type == 1: conn_dict[key] = HttpConn(tcp_pac) elif tcp_pac.pac_type == 0: # tcp init before capture, we found a http request header, begin parse # if is a http request? if utils.is_request(tcp_pac.body): conn_dict[key] = HttpConn(tcp_pac)
def handle_http(c): def printheaders(headers): l = 0 for k in headers.keys(): if l < len(k): l = len(k) for k, v in headers.items(): utils.print(k.ljust(l)) utils.print(': ') utils.print(v) utils.print('\n') filter = config.get_filter() level = config.get_config().level for tcpcon in get_tcpconn(c.infile): if filter.index != None and tcpcon.index not in filter.index: continue if not (filter.by_con_tuple(tcpcon.con_tuple)): continue http = HttpParser(tcpcon) if not http.msgs: continue tcp = http.tcp tcp_msg = "\033[31;2m%s [%s:%d] -- -- --> [%s:%d]\033[0m\n" % \ (tcp.index, tcp.con_tuple[0], tcp.con_tuple[1], tcp.con_tuple[2], tcp.con_tuple[3]) utils.print(tcp_msg) if level == OutputLevel.ONLY_URL: for msg in http.msgs: if msg.is_request: utils.print(msg.reqline["method"] + ' ' + msg.URI()) utils.print('\n') else: for i, msg in enumerate(http.msgs): if msg.is_request and i != 0: utils.print('\033[31;2m') utils.print('-' * 80) utils.print('\033[0m') utils.print('\n') utils.print(''.join(msg.raw_headers)) utils.print('\n') if level == OutputLevel.ALL_BODY: utils.print(msg.body.getvalue())
def pcap_file(conn_dict, infile): """ :type conn_dict: dict :type infile:file """ file_format, head = get_file_format(infile) if file_format == FileFormat.PCAP: pcap_file = pcap.PcapFile(infile, head).read_packet elif file_format == FileFormat.PCAP_NG: pcap_file = pcapng.PcapngFile(infile, head).read_packet else: print("unknown file format.", file=sys.stderr) sys.exit(1) _filter = config.get_filter() for tcp_pac in packet_parser.read_package_r(pcap_file): # filter # get time if CTCore.activity_date_time == "": CTCore.activity_date_time = time.strftime('%a, %x %X', time.gmtime(int(str(tcp_pac.micro_second)[:10]))) if CTCore.client.headers["IP"] == "": CTCore.client.headers["IP"] = tcp_pac.source if CTCore.client.headers["MAC"] == "": CTCore.client.headers["MAC"] = tcp_pac.src_mac if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)): continue if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)): continue key = tcp_pac.gen_key() # we already have this conn if key in conn_dict: conn_dict[key].append(tcp_pac) # conn closed. if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE: conn_dict[key].finish() del conn_dict[key] # begin tcp connection. elif tcp_pac.pac_type == 1: conn_dict[key] = HttpConn(tcp_pac) elif tcp_pac.pac_type == 0: # tcp init before capture, we found a http request header, begin parse # if is a http request? if utils.is_request(tcp_pac.body): conn_dict[key] = HttpConn(tcp_pac)
def main(filename = None): if(filename == None): filename = '1.pcap' _filter = config.get_filter() conn_dict = OrderedDict() config.out = sys.stdout try: infile = open(filename, "rb") try: return process_pcap_file(conn_dict, infile) finally: infile.close() finally: for conn in conn_dict.values(): conn.finish()
def main(filename=None): if (filename == None): filename = '1.pcap' _filter = config.get_filter() conn_dict = OrderedDict() config.out = sys.stdout try: infile = open(filename, "rb") try: return process_pcap_file(conn_dict, infile) finally: infile.close() finally: for conn in conn_dict.values(): conn.finish()
def parse_pcap_file(infile): """ :type infile:file """ conn_dict = OrderedDict() file_format, head = get_file_format(infile) if file_format == FileFormat.PCAP: pcap_file = pcap.PcapFile(infile, head).read_packet elif file_format == FileFormat.PCAP_NG: pcap_file = pcapng.PcapngFile(infile, head).read_packet else: print("unknown file format.", file=sys.stderr) sys.exit(1) _filter = config.get_filter() for tcp_pac in packet_parser.read_tcp_packet(pcap_file): # filter if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)): continue if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)): continue key = tcp_pac.gen_key() # we already have this conn if key in conn_dict: conn_dict[key].on_packet(tcp_pac) # conn closed. if conn_dict[key].closed(): conn_dict[key].finish() del conn_dict[key] # begin tcp connection. elif tcp_pac.syn and not tcp_pac.ack: conn_dict[key] = TcpConnection(tcp_pac) elif utils.is_request(tcp_pac.body): # tcp init before capture, we start from a possible http request header. conn_dict[key] = TcpConnection(tcp_pac) # finish connection which not close yet for conn in conn_dict.values(): conn.finish()
def parse_pcap_file(conn_dict, infile): """ :type conn_dict: dict :type infile:file """ file_format, head = get_file_format(infile) if file_format == FileFormat.PCAP: pcap_file = pcap.PcapFile(infile, head).read_packet elif file_format == FileFormat.PCAP_NG: pcap_file = pcapng.PcapngFile(infile, head).read_packet else: print("unknown file format.", file=sys.stderr) sys.exit(1) _filter = config.get_filter() for tcp_pac in packet_parser.read_package_r(pcap_file): # filter if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)): continue if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)): continue key = tcp_pac.gen_key() # we already have this conn if key in conn_dict: conn_dict[key].append(tcp_pac) # conn closed. if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE: conn_dict[key].finish() del conn_dict[key] # begin tcp connection. elif tcp_pac.pac_type == 1: conn_dict[key] = HttpConn(tcp_pac) elif tcp_pac.pac_type == 0: # tcp init before capture, we found a http request header, begin parse # if is a http request? if utils.is_request(tcp_pac.body): conn_dict[key] = HttpConn(tcp_pac)
def main(): parser = argparse.ArgumentParser() parser.add_argument("infile", nargs='?', help="the pcap file to parse") parser.add_argument( "-i", "--ip", help="only parse packages with specified source OR dest ip") parser.add_argument( "-p", "--port", type=int, help="only parse packages with specified source OR dest port") parser.add_argument("-v", "--verbosity", help="increase output verbosity(-vv is recommended)", action="count") parser.add_argument("-g", "--group", help="group http request/response by connection", action="store_true") parser.add_argument("-o", "--output", help="output to file instead of stdout") parser.add_argument("-e", "--encoding", help="decode the data use specified encodings.") parser.add_argument("-b", "--beauty", help="output json in a pretty way.", action="store_true") parser.add_argument("-d", "--domain", help="filter http data by request domain") parser.add_argument("-u", "--uri", help="filter http data by request uri pattern") args = parser.parse_args() file_path = "-" if args.infile is None else args.infile _filter = config.get_filter() _filter.ip = args.ip _filter.port = args.port _filter.domain = args.domain _filter.uri_pattern = args.uri # deal with configs parse_config = config.get_config() if args.verbosity: parse_config.level = args.verbosity if args.encoding: parse_config.encoding = args.encoding parse_config.pretty = args.beauty parse_config.group = args.group if args.output: output_file = open(args.output, "w+") else: output_file = sys.stdout config.out = output_file try: if file_path != '-': infile = io.open(file_path, "rb") else: infile = sys.stdin try: parse_pcap_file(infile) finally: infile.close() finally: if args.output: output_file.close()
def process_pcap_file(conn_dict, infile): file_format, head = get_file_format(infile) if file_format == FileFormat.PCAP: pcap_file = pcap.PcapFile(infile, head).read_packet elif file_format == FileFormat.PCAP_NG: pcap_file = pcapng.PcapngFile(infile, head).read_packet else: print("unknown file format.") sys.exit(1) #used for diameter decode diameterConn = DiameterConn() _filter = config.get_filter() for tcp_pac in packet_parser.read_package_r(pcap_file): # filter if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)): continue if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)): continue #IP Protocol if(tcp_pac.source_port == 8083 or tcp_pac.dest_port == 8083): key = tcp_pac.gen_key() # we already have this conn if key in conn_dict: conn_dict[key].append(tcp_pac) # conn closed. if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE: for package in conn_dict[key].tcp_pac_list: msg = Message() msg.type = 'http' msg.sourceIP = package[1] msg.sourcePort = package[2] msg.destIP = package[3] msg.destPort = package[4] msg.body = package[0].body msg.seconds = package[0].seconds msg.suseconds = package[0].suseconds yield msg del conn_dict[key] # begin tcp connection. elif tcp_pac.pac_type == 1: conn_dict[key] = HttpConn(tcp_pac) elif tcp_pac.pac_type == 0: # tcp init before capture, we found a http request header, begin parse # if is a http request? if utils.is_request(tcp_pac.body): conn_dict[key] = HttpConn(tcp_pac) elif(tcp_pac.source_port in (6553,16553,3868) or tcp_pac.dest_port in (3868,6553,16553)): if len(tcp_pac.body) > 20: version = struct.unpack('!b',tcp_pac.body[0:1])[0] message_length = struct.unpack('!I',b'\x00' + tcp_pac.body[1:4])[0] Command_Flags = struct.unpack('!b',tcp_pac.body[4:5])[0] command_code = struct.unpack('!I',b'\x00' + tcp_pac.body[5:8])[0] application_ID, hop_by_hop_ID, end_to_end_ID = struct.unpack('!III', tcp_pac.body[8:20]) if(version == 1): #print(version, message_length, command_code) headerinfo, tree = diameterConn.decode(tcp_pac.body) body = headerinfo + '\n' for avp in tree: body += avp.AVPoutput() + '\n' msg = Message() msg.type = 'diameter' msg.sourceIP = tcp_pac.source msg.sourcePort = tcp_pac.source_port msg.destIP = tcp_pac.dest msg.destPort = tcp_pac.dest_port msg.body = body msg.seconds = tcp_pac.seconds msg.suseconds = tcp_pac.suseconds yield msg
def process_pcap_file(conn_dict, infile): file_format, head = get_file_format(infile) if file_format == FileFormat.PCAP: pcap_file = pcap.PcapFile(infile, head).read_packet elif file_format == FileFormat.PCAP_NG: pcap_file = pcapng.PcapngFile(infile, head).read_packet else: print("unknown file format.") sys.exit(1) #used for diameter decode diameterConn = DiameterConn() _filter = config.get_filter() for tcp_pac in packet_parser.read_package_r(pcap_file): # filter if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)): continue if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)): continue #IP Protocol if (tcp_pac.source_port == 8083 or tcp_pac.dest_port == 8083): key = tcp_pac.gen_key() # we already have this conn if key in conn_dict: conn_dict[key].append(tcp_pac) # conn closed. if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE: for package in conn_dict[key].tcp_pac_list: msg = Message() msg.type = 'http' msg.sourceIP = package[1] msg.sourcePort = package[2] msg.destIP = package[3] msg.destPort = package[4] msg.body = package[0].body msg.seconds = package[0].seconds msg.suseconds = package[0].suseconds yield msg del conn_dict[key] # begin tcp connection. elif tcp_pac.pac_type == 1: conn_dict[key] = HttpConn(tcp_pac) elif tcp_pac.pac_type == 0: # tcp init before capture, we found a http request header, begin parse # if is a http request? if utils.is_request(tcp_pac.body): conn_dict[key] = HttpConn(tcp_pac) elif (tcp_pac.source_port in (6553, 16553, 3868) or tcp_pac.dest_port in (3868, 6553, 16553)): if len(tcp_pac.body) > 20: version = struct.unpack('!b', tcp_pac.body[0:1])[0] message_length = struct.unpack('!I', b'\x00' + tcp_pac.body[1:4])[0] Command_Flags = struct.unpack('!b', tcp_pac.body[4:5])[0] command_code = struct.unpack('!I', b'\x00' + tcp_pac.body[5:8])[0] application_ID, hop_by_hop_ID, end_to_end_ID = struct.unpack( '!III', tcp_pac.body[8:20]) if (version == 1): #print(version, message_length, command_code) headerinfo, tree = diameterConn.decode(tcp_pac.body) body = headerinfo + '\n' for avp in tree: body += avp.AVPoutput() + '\n' msg = Message() msg.type = 'diameter' msg.sourceIP = tcp_pac.source msg.sourcePort = tcp_pac.source_port msg.destIP = tcp_pac.dest msg.destPort = tcp_pac.dest_port msg.body = body msg.seconds = tcp_pac.seconds msg.suseconds = tcp_pac.suseconds yield msg