Example #1
0
    def read_request(self, reader, message):
        """ read and output one http request. """
        if message.expect_header and not utils.is_request(reader.fetchline()):
            req_header = message.expect_header
            message.expect_header = None
        else:
            req_header = self.read_http_req_header(reader)
            if req_header is None:
                # read header error, we skip all data.
                reader.skipall()
                return
            if req_header.expect:
                # it is expect:continue-100 post request
                message.expect_header = req_header

        # deal with body
        if not req_header.chunked:
            content = reader.read(req_header.content_len)
        else:
            content = self.read_chunked_body(reader)

        _filter = config.get_filter()
        show = _filter.by_domain(req_header.host) and _filter.by_uri(
            req_header.uri)
        message.filtered = not show
        if show:
            self.processor.on_http_req(req_header, content)
Example #2
0
    def read_request(self, reader, message):
        """ read and output one http request. """
        if message.expect_header and not utils.is_request(reader.fetchline()):
            req_header = message.expect_header
            message.expect_header = None
        else:
            req_header = self.read_http_req_header(reader)
            if req_header is None:
                # read header error, we skip all data.
                reader.skipall()
                return
            if req_header.expect:
                # it is expect:continue-100 post request
                message.expect_header = req_header

        # deal with body
        if not req_header.chunked:
            content = reader.read(req_header.content_len)
        else:
            content = self.read_chunked_body(reader)

        _filter = config.get_filter()
        show = _filter.by_domain(req_header.host) and _filter.by_uri(req_header.uri)
        message.filtered = not show
        if show:
            self.processor.on_http_req(req_header, content)
Example #3
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("infile", nargs='?', help="the pcap file to parse")
    parser.add_argument("-i", "--ip", help="only parse packages with specified source OR dest ip")
    parser.add_argument("-p", "--port", type=int,
                        help="only parse packages with specified source OR dest port")
    parser.add_argument("-v", "--verbosity", help="increase output verbosity(-vv is recommended)",
                        action="count")
    parser.add_argument("-g", "--group", help="group http request/response by connection",
                        action="store_true")
    parser.add_argument("-o", "--output", help="output to file instead of stdout")
    parser.add_argument("-e", "--encoding", help="decode the data use specified encodings.")
    parser.add_argument("-b", "--beauty", help="output json in a pretty way.", action="store_true")
    parser.add_argument("-d", "--domain", help="filter http data by request domain")
    parser.add_argument("-u", "--uri", help="filter http data by request uri pattern")

    args = parser.parse_args()

    file_path = "-" if args.infile is None else args.infile

    _filter = config.get_filter()
    _filter.ip = args.ip
    _filter.port = args.port
    _filter.domain = args.domain
    _filter.uri_pattern = args.uri

    # deal with configs
    parse_config = config.get_config()
    if args.verbosity:
        parse_config.level = args.verbosity
    if args.encoding:
        parse_config.encoding = args.encoding
    parse_config.pretty = args.beauty
    parse_config.group = args.group

    if args.output:
        output_file = open(args.output, "w+")
    else:
        output_file = sys.stdout

    config.out = output_file

    conn_dict = OrderedDict()
    try:
        if file_path != '-':
            infile = io.open(file_path, "rb")
        else:
            infile = sys.stdin
        try:
            pcap_file(conn_dict, infile)
        finally:
            infile.close()
    finally:
        for conn in conn_dict.values():
            conn.finish()
        if args.output:
            output_file.close()
Example #4
0
def handle_tcp(c):
    filter = config.get_filter()
    for tcp in get_tcpconn(c.infile):
        if filter.index != None and tcp.index not in filter.index:
            continue
        tcp_msg = "\033[31;2m%s [%s:%d] -- -- --> [%s:%d]\033[0m\n" % \
                (tcp.index, tcp.con_tuple[0], tcp.con_tuple[1],
                        tcp.con_tuple[2], tcp.con_tuple[3])
        utils.print(tcp_msg)
Example #5
0
def pcap_file(conn_dict, infile):
    """
    :type conn_dict: dict
    :type infile:file
    """
    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.", file=sys.stderr)
        sys.exit(1)

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_package_r(pcap_file):
        # filter
        # get time
        if CTCore.activity_date_time == "":
            CTCore.activity_date_time = datetime.fromtimestamp(
                int(str(tcp_pac.micro_second)[:10]))

        if CTCore.client.headers["IP"] == "":
            CTCore.client.headers["IP"] = tcp_pac.source

        if CTCore.client.headers["MAC"] == "":
            CTCore.client.headers["MAC"] = tcp_pac.src_mac

        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port)
                or _filter.by_port(tcp_pac.dest_port)):
            continue

        key = tcp_pac.gen_key()
        # we already have this conn
        if key in conn_dict:
            conn_dict[key].append(tcp_pac)
            # conn closed.
            if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
                conn_dict[key].finish()
                del conn_dict[key]

        # begin tcp connection.
        elif tcp_pac.pac_type == 1:
            conn_dict[key] = HttpConn(tcp_pac)
        elif tcp_pac.pac_type == 0:
            # tcp init before capture, we found a http request header, begin parse
            # if is a http request?
            if utils.is_request(tcp_pac.body):
                conn_dict[key] = HttpConn(tcp_pac)
Example #6
0
def handle_http(c):
    def printheaders(headers):
        l = 0
        for k in headers.keys():
            if l < len(k):
                l = len(k)
        for k, v in headers.items():
            utils.print(k.ljust(l))
            utils.print(': ')
            utils.print(v)
            utils.print('\n')

    filter = config.get_filter()
    level = config.get_config().level

    for tcpcon in get_tcpconn(c.infile):
        if filter.index != None and tcpcon.index not in filter.index:
            continue

        if not (filter.by_con_tuple(tcpcon.con_tuple)):
            continue

        http = HttpParser(tcpcon)

        if not http.msgs:
            continue

        tcp = http.tcp
        tcp_msg = "\033[31;2m%s [%s:%d] -- -- --> [%s:%d]\033[0m\n" % \
                (tcp.index, tcp.con_tuple[0], tcp.con_tuple[1],
                        tcp.con_tuple[2], tcp.con_tuple[3])
        utils.print(tcp_msg)

        if level == OutputLevel.ONLY_URL:
            for msg in http.msgs:
                if msg.is_request:
                    utils.print(msg.reqline["method"] + ' ' + msg.URI())
                    utils.print('\n')
        else:
            for i, msg in enumerate(http.msgs):
                if msg.is_request and i != 0:
                        utils.print('\033[31;2m')
                        utils.print('-' * 80)
                        utils.print('\033[0m')
                        utils.print('\n')

                utils.print(''.join(msg.raw_headers))
                utils.print('\n')
                if level == OutputLevel.ALL_BODY:
                    utils.print(msg.body.getvalue())
Example #7
0
def pcap_file(conn_dict, infile):
    """
    :type conn_dict: dict
    :type infile:file
    """
    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.", file=sys.stderr)
        sys.exit(1)

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_package_r(pcap_file):
        # filter
        # get time
        if CTCore.activity_date_time == "":
            CTCore.activity_date_time = time.strftime('%a, %x %X', time.gmtime(int(str(tcp_pac.micro_second)[:10])))

        if CTCore.client.headers["IP"] == "":
            CTCore.client.headers["IP"] = tcp_pac.source

        if CTCore.client.headers["MAC"] == "":
            CTCore.client.headers["MAC"] = tcp_pac.src_mac

        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
            continue

        key = tcp_pac.gen_key()
        # we already have this conn
        if key in conn_dict:
            conn_dict[key].append(tcp_pac)
            # conn closed.
            if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
                conn_dict[key].finish()
                del conn_dict[key]

        # begin tcp connection.
        elif tcp_pac.pac_type == 1:
            conn_dict[key] = HttpConn(tcp_pac)
        elif tcp_pac.pac_type == 0:
            # tcp init before capture, we found a http request header, begin parse
            # if is a http request?
            if utils.is_request(tcp_pac.body):
                conn_dict[key] = HttpConn(tcp_pac)
def main(filename = None):

    if(filename == None):
        filename = '1.pcap'

    _filter = config.get_filter()
    conn_dict = OrderedDict()
    config.out = sys.stdout
    try:
        infile = open(filename, "rb")
        try:
            return process_pcap_file(conn_dict, infile)
        finally:
            infile.close()
    finally:
        for conn in conn_dict.values():
            conn.finish()
def main(filename=None):

    if (filename == None):
        filename = '1.pcap'

    _filter = config.get_filter()
    conn_dict = OrderedDict()
    config.out = sys.stdout
    try:
        infile = open(filename, "rb")
        try:
            return process_pcap_file(conn_dict, infile)
        finally:
            infile.close()
    finally:
        for conn in conn_dict.values():
            conn.finish()
Example #10
0
def parse_pcap_file(infile):
    """
    :type infile:file
    """

    conn_dict = OrderedDict()

    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.", file=sys.stderr)
        sys.exit(1)

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
        # filter
        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port)
                or _filter.by_port(tcp_pac.dest_port)):
            continue

        key = tcp_pac.gen_key()
        # we already have this conn
        if key in conn_dict:
            conn_dict[key].on_packet(tcp_pac)
            # conn closed.
            if conn_dict[key].closed():
                conn_dict[key].finish()
                del conn_dict[key]

        # begin tcp connection.
        elif tcp_pac.syn and not tcp_pac.ack:
            conn_dict[key] = TcpConnection(tcp_pac)
        elif utils.is_request(tcp_pac.body):
            # tcp init before capture, we start from a possible http request header.
            conn_dict[key] = TcpConnection(tcp_pac)

    # finish connection which not close yet
    for conn in conn_dict.values():
        conn.finish()
Example #11
0
def parse_pcap_file(infile):
    """
    :type infile:file
    """

    conn_dict = OrderedDict()

    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.", file=sys.stderr)
        sys.exit(1)

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_tcp_packet(pcap_file):
        # filter
        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
            continue

        key = tcp_pac.gen_key()
        # we already have this conn
        if key in conn_dict:
            conn_dict[key].on_packet(tcp_pac)
            # conn closed.
            if conn_dict[key].closed():
                conn_dict[key].finish()
                del conn_dict[key]

        # begin tcp connection.
        elif tcp_pac.syn and not tcp_pac.ack:
            conn_dict[key] = TcpConnection(tcp_pac)
        elif utils.is_request(tcp_pac.body):
            # tcp init before capture, we start from a possible http request header.
            conn_dict[key] = TcpConnection(tcp_pac)

    # finish connection which not close yet
    for conn in conn_dict.values():
        conn.finish()
Example #12
0
def parse_pcap_file(conn_dict, infile):
    """
    :type conn_dict: dict
    :type infile:file
    """
    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.", file=sys.stderr)
        sys.exit(1)

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_package_r(pcap_file):
        # filter
        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
            continue

        key = tcp_pac.gen_key()
        # we already have this conn
        if key in conn_dict:
            conn_dict[key].append(tcp_pac)
            # conn closed.
            if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
                conn_dict[key].finish()
                del conn_dict[key]

        # begin tcp connection.
        elif tcp_pac.pac_type == 1:
            conn_dict[key] = HttpConn(tcp_pac)
        elif tcp_pac.pac_type == 0:
            # tcp init before capture, we found a http request header, begin parse
            # if is a http request?
            if utils.is_request(tcp_pac.body):
                conn_dict[key] = HttpConn(tcp_pac)
Example #13
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("infile", nargs='?', help="the pcap file to parse")
    parser.add_argument(
        "-i",
        "--ip",
        help="only parse packages with specified source OR dest ip")
    parser.add_argument(
        "-p",
        "--port",
        type=int,
        help="only parse packages with specified source OR dest port")
    parser.add_argument("-v",
                        "--verbosity",
                        help="increase output verbosity(-vv is recommended)",
                        action="count")
    parser.add_argument("-g",
                        "--group",
                        help="group http request/response by connection",
                        action="store_true")
    parser.add_argument("-o",
                        "--output",
                        help="output to file instead of stdout")
    parser.add_argument("-e",
                        "--encoding",
                        help="decode the data use specified encodings.")
    parser.add_argument("-b",
                        "--beauty",
                        help="output json in a pretty way.",
                        action="store_true")
    parser.add_argument("-d",
                        "--domain",
                        help="filter http data by request domain")
    parser.add_argument("-u",
                        "--uri",
                        help="filter http data by request uri pattern")

    args = parser.parse_args()

    file_path = "-" if args.infile is None else args.infile

    _filter = config.get_filter()
    _filter.ip = args.ip
    _filter.port = args.port
    _filter.domain = args.domain
    _filter.uri_pattern = args.uri

    # deal with configs
    parse_config = config.get_config()
    if args.verbosity:
        parse_config.level = args.verbosity
    if args.encoding:
        parse_config.encoding = args.encoding
    parse_config.pretty = args.beauty
    parse_config.group = args.group

    if args.output:
        output_file = open(args.output, "w+")
    else:
        output_file = sys.stdout

    config.out = output_file

    try:
        if file_path != '-':
            infile = io.open(file_path, "rb")
        else:
            infile = sys.stdin
        try:
            parse_pcap_file(infile)
        finally:
            infile.close()
    finally:
        if args.output:
            output_file.close()
Example #14
0
def process_pcap_file(conn_dict, infile):
    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.")
        sys.exit(1)

    #used for diameter decode
    diameterConn = DiameterConn()

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_package_r(pcap_file):
        # filter
        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port) or _filter.by_port(tcp_pac.dest_port)):
            continue

        #IP Protocol
        if(tcp_pac.source_port == 8083 or tcp_pac.dest_port == 8083):
            key = tcp_pac.gen_key()
            # we already have this conn
            if key in conn_dict:
                conn_dict[key].append(tcp_pac)
                # conn closed.
                if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
                    for package in conn_dict[key].tcp_pac_list:
                        msg = Message()
                        msg.type = 'http'
                        msg.sourceIP = package[1]
                        msg.sourcePort = package[2]
                        msg.destIP = package[3]
                        msg.destPort = package[4]
                        msg.body = package[0].body
                        msg.seconds = package[0].seconds
                        msg.suseconds = package[0].suseconds

                        yield msg

                    del conn_dict[key]

            # begin tcp connection.
            elif tcp_pac.pac_type == 1:
                conn_dict[key] = HttpConn(tcp_pac)
            elif tcp_pac.pac_type == 0:
                # tcp init before capture, we found a http request header, begin parse
                # if is a http request?
                if utils.is_request(tcp_pac.body):
                    conn_dict[key] = HttpConn(tcp_pac)

        elif(tcp_pac.source_port in (6553,16553,3868) or tcp_pac.dest_port in (3868,6553,16553)):
            if len(tcp_pac.body) > 20:
                version = struct.unpack('!b',tcp_pac.body[0:1])[0]
                message_length = struct.unpack('!I',b'\x00' + tcp_pac.body[1:4])[0]
                Command_Flags = struct.unpack('!b',tcp_pac.body[4:5])[0]
                command_code = struct.unpack('!I',b'\x00' + tcp_pac.body[5:8])[0]
                application_ID, hop_by_hop_ID, end_to_end_ID = struct.unpack('!III', tcp_pac.body[8:20])

                if(version == 1):
                    #print(version, message_length, command_code)
                    headerinfo, tree = diameterConn.decode(tcp_pac.body)
                    body = headerinfo + '\n'
                    for avp in tree:
                        body += avp.AVPoutput() + '\n'

                    msg = Message()
                    msg.type = 'diameter'
                    msg.sourceIP = tcp_pac.source
                    msg.sourcePort = tcp_pac.source_port
                    msg.destIP = tcp_pac.dest
                    msg.destPort = tcp_pac.dest_port
                    msg.body = body
                    msg.seconds = tcp_pac.seconds
                    msg.suseconds = tcp_pac.suseconds

                    yield msg
Example #15
0
def process_pcap_file(conn_dict, infile):
    file_format, head = get_file_format(infile)
    if file_format == FileFormat.PCAP:
        pcap_file = pcap.PcapFile(infile, head).read_packet
    elif file_format == FileFormat.PCAP_NG:
        pcap_file = pcapng.PcapngFile(infile, head).read_packet
    else:
        print("unknown file format.")
        sys.exit(1)

    #used for diameter decode
    diameterConn = DiameterConn()

    _filter = config.get_filter()
    for tcp_pac in packet_parser.read_package_r(pcap_file):
        # filter
        if not (_filter.by_ip(tcp_pac.source) or _filter.by_ip(tcp_pac.dest)):
            continue
        if not (_filter.by_port(tcp_pac.source_port)
                or _filter.by_port(tcp_pac.dest_port)):
            continue

        #IP Protocol
        if (tcp_pac.source_port == 8083 or tcp_pac.dest_port == 8083):
            key = tcp_pac.gen_key()
            # we already have this conn
            if key in conn_dict:
                conn_dict[key].append(tcp_pac)
                # conn closed.
                if tcp_pac.pac_type == packet_parser.TcpPack.TYPE_CLOSE:
                    for package in conn_dict[key].tcp_pac_list:
                        msg = Message()
                        msg.type = 'http'
                        msg.sourceIP = package[1]
                        msg.sourcePort = package[2]
                        msg.destIP = package[3]
                        msg.destPort = package[4]
                        msg.body = package[0].body
                        msg.seconds = package[0].seconds
                        msg.suseconds = package[0].suseconds

                        yield msg

                    del conn_dict[key]

            # begin tcp connection.
            elif tcp_pac.pac_type == 1:
                conn_dict[key] = HttpConn(tcp_pac)
            elif tcp_pac.pac_type == 0:
                # tcp init before capture, we found a http request header, begin parse
                # if is a http request?
                if utils.is_request(tcp_pac.body):
                    conn_dict[key] = HttpConn(tcp_pac)

        elif (tcp_pac.source_port in (6553, 16553, 3868)
              or tcp_pac.dest_port in (3868, 6553, 16553)):
            if len(tcp_pac.body) > 20:
                version = struct.unpack('!b', tcp_pac.body[0:1])[0]
                message_length = struct.unpack('!I',
                                               b'\x00' + tcp_pac.body[1:4])[0]
                Command_Flags = struct.unpack('!b', tcp_pac.body[4:5])[0]
                command_code = struct.unpack('!I',
                                             b'\x00' + tcp_pac.body[5:8])[0]
                application_ID, hop_by_hop_ID, end_to_end_ID = struct.unpack(
                    '!III', tcp_pac.body[8:20])

                if (version == 1):
                    #print(version, message_length, command_code)
                    headerinfo, tree = diameterConn.decode(tcp_pac.body)
                    body = headerinfo + '\n'
                    for avp in tree:
                        body += avp.AVPoutput() + '\n'

                    msg = Message()
                    msg.type = 'diameter'
                    msg.sourceIP = tcp_pac.source
                    msg.sourcePort = tcp_pac.source_port
                    msg.destIP = tcp_pac.dest
                    msg.destPort = tcp_pac.dest_port
                    msg.body = body
                    msg.seconds = tcp_pac.seconds
                    msg.suseconds = tcp_pac.suseconds

                    yield msg