def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.epilog = textwrap.dedent("""
List the available API endpoints for this packet-cafe server.
::
$ lim cafe endpoints
+---------------------------------------------------------------------+
| Endpoint |
+---------------------------------------------------------------------+
| /api/v1 |
| /api/v1/id/{session_id}/{req_id}/{tool}/{pcap}/{counter}/{filename} |
| /api/v1/ids/{session_id} |
| /api/v1/info |
| /api/v1/raw/{tool}/{counter}/{session_id}/{req_id} |
| /api/v1/results/{tool}/{counter}/{session_id}/{req_id} |
| /api/v1/status/{session_id}/{req_id} |
| /api/v1/stop/{session_id}/{req_id} |
| /api/v1/tools |
| /api/v1/upload |
+---------------------------------------------------------------------+
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v1
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument('sess_id', nargs='?', default=None)
parser.epilog = textwrap.dedent("""
List current request IDs for a specific packet-cafe session ID. By default,
the last used session ID will be the default. Otherwise, specify the session ID
as an argument
::
$ lim cafe requests --fit-width
[+] implicitly reusing last session id bae5d69c-7180-445d-a8db-22a5ef0872e8
+--------------------------+--------------------------+-------------------+--------------------------+
| Id | Filename | Original_Filename | Tools |
+--------------------------+--------------------------+-------------------+--------------------------+
| 13394ad96ef3420094387a6a | trace_13394ad96ef3420094 | test.pcap | networkml,mercury,pcap- |
| a748490f | 387a6aa748490f_2020-05-1 | | stats,snort,p0f,pcapplot |
| | 5_07_25_48.pcap | | |
+--------------------------+--------------------------+-------------------+--------------------------+
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-ids-sess_id
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.epilog = textwrap.dedent("""
List the current session IDS in the packet-cafe service.
Returns shell exit code ``0`` if one or more sessiona are
present, or ``1`` if none are present.
Use the ``-q`` option to suppress the output table or error
message.
::
$ lim cafe admin sessions
+--------------------------------------+
| SessionId |
+--------------------------------------+
| 57b1484b-5502-4e3c-b6bc-854d4aeb2038 |
| 57be4843-32c0-4943-93d8-d1ec9bc0e792 |
| 2d222a53-5b01-4d5e-a659-7da7c21d3cf6 |
| 73d532d7-3b2b-4a93-9a68-ae7091af6a2f |
| 9a949fe6-6520-437f-89ec-e7af6925b1e0 |
| 7eedfd93-4f65-4422-8d70-a4edf47d7364 |
| a42ee6ab-d60b-4d8e-a1df-cb3dc6985c81 |
+--------------------------------------+
See https://iqtlabs.gitbook.io/packet-cafe/design/api#v-1-ids
""")
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument(
'--tree',
action='store_true',
dest='tree',
default=False,
help='Produce tree output rather than table (default: False)'
)
parser.epilog = textwrap.dedent("""
Lists all files uploaded into the packet-cafe server. This can produce
a large amount of output with very long lines, so you may want to use the
``--fit-width`` option to break lines to fit the screen.
You can get a tree listing of files, which is much more compact and
readable, with the ``--tree`` option.
::
$ lim cafe admin files --tree
files
└── 791e1034-fdb9-4fa4-a410-e1dedef7c0b8
└── dcfe1b4dd2a04d559f6600902847a11a
├── tcprewrite_dot1q-2020-06-21-21_44_49.215175-UTC
│ ├── pcap-node-splitter-2020-06-21-21_44_53.389934-UTC
│ │ ├── clients
│ │ │ ├── combined.csv.gz
│ │ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap
│ │ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap.csv.gz
│ │ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-79-147-32-84-165-147-32-84-79-data-udp-frame-eth-ip-port-0.pcap
│ │ │ └── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-79-147-32-84-165-147-32-84-79-data-udp-frame-eth-ip-port-0.pcap.csv.gz
│ │ └── servers
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-118-228-148-32-118-228-148-32-147-32-84-165-2-4-5-4-1-1-4-2-tcp-frame-eth-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-123-126-51-33-123-126-51-33-147-32-84-165-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-123-126-51-57-123-126-51-57-147-32-84-165-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-123-126-51-64-123-126-51-64-147-32-84-165-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-123-126-51-65-123-126-51-65-147-32-84-165-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-147-32-80-9-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-147-32-84-165-147-32-84-165-147-32-84-79-data-udp-frame-eth-ip-port-0.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-147-32-84-255-147-32-84-165-147-32-84-255-nbns-frame-eth-wsshort-udp-ip-port-137.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-147-32-84-79-147-32-84-165-147-32-84-79-icmp-wsshort-frame-eth-ip.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-195-113-232-73-147-32-84-165-195-113-232-73-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-209-85-149-160-147-32-84-165-209-85-149-160-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-218-29-42-137-147-32-84-165-218-29-42-137-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-220-181-111-147-147-32-84-165-220-181-111-147-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-220-181-69-213-147-32-84-165-2-4-5-4-1-1-4-2-220-181-69-213-tcp-frame-eth-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-61-135-188-157-147-32-84-165-2-4-5-4-1-1-4-2-61-135-188-157-tcp-frame-eth-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-61-135-188-210-147-32-84-165-61-135-188-210-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ ├── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-61-135-188-212-147-32-84-165-61-135-188-212-wsshort-eth-tcp-http-frame-ip-port-80.pcap
│ │ └── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-server-ip-61-135-189-50-147-32-84-165-2-4-5-4-1-1-4-2-61-135-189-50-tcp-frame-eth-ip-port-80.pcap
│ └── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45.pcap
├── test.pcap
└── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45.pcap
See https://iqtlabs.gitbook.io/packet-cafe/design/api#v-1-id-files
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument('sess_id', nargs='?', default=None)
parser.add_argument('req_id', nargs='?', default=None)
parser.add_argument(
'-t', '--tool',
metavar='<tool>',
dest='tool',
default=None,
help='Only show results for specified tool (default: None)'
)
parser.add_argument(
'-C', '--counter',
metavar='<counter>',
type=_valid_counter,
dest='counter',
default=1,
help=('Counter for selecting a specific file '
'from a set (default: 1)')
)
parser.epilog = textwrap.dedent("""
Get the results from a tool (in the form of HTML) for local storage
or rendering.
In this version, the contents are simply put on ``stdout`` and you must
redirect them to a file. (In future, this will be saved and a browser
opened to view the file, as if you had selected a result in the web UI.)
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-results-tool-counter-sess_id-req_id
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser = add_browser_options(parser)
parser.epilog = textwrap.dedent("""
Opens up the packet-cafe UI in a browser.
To see help information about how the browser option works and
how you can configure it, see ``lim about --help``.
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument('sess_id', nargs='?', default=None)
parser.add_argument('req_id', nargs='?', default=None)
parser.epilog = textwrap.dedent("""
Return the status of all tools for a session and request ID.
By default, the last session ID and request ID (if available)
are used.
::
$ lim cafe status
[+] implicitly reusing last session id bae5d69c-7180-445d-a8db-22a5ef0872e8
[+] implicitly reusing last request id c33c56abe4c743a8b77e0b76d9548c06
+---------------+----------+----------------------------------+
| Tool | State | Timestamp |
+---------------+----------+----------------------------------+
| snort | Complete | 2020-05-15T01:25:52.669640+00:00 |
| networkml | Complete | 2020-05-15T01:26:36.616426+00:00 |
| pcap-splitter | Complete | 2020-05-15T01:25:56.362483+00:00 |
| mercury | Complete | 2020-05-15T01:25:49.773921+00:00 |
| pcap-dot1q | Complete | 2020-05-15T01:25:47.988746+00:00 |
| ncapture | Complete | 2020-05-15T01:25:46.075214+00:00 |
| pcapplot | Complete | 2020-05-15T01:26:24.899752+00:00 |
| pcap_stats | Complete | 2020-05-15T01:25:48.251749+00:00 |
| p0f | Complete | 2020-05-15T01:26:48.456883+00:00 |
+---------------+----------+----------------------------------+
If no session ID is identified, you will be prompted to choose from
those that are available:
::
$ lim cafe status
Chose a session:
→ <CANCEL>
57b1484b-5502-4e3c-b6bc-854d4aeb2038
57be4843-32c0-4943-93d8-d1ec9bc0e792
2d222a53-5b01-4d5e-a659-7da7c21d3cf6
73d532d7-3b2b-4a93-9a68-ae7091af6a2f
9a949fe6-6520-437f-89ec-e7af6925b1e0
7eedfd93-4f65-4422-8d70-a4edf47d7364
a42ee6ab-d60b-4d8e-a1df-cb3dc6985c81
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-status-sess_id-req_id
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument(
'--definitions',
action='store_true',
dest='definitions',
default=False,
help=('Show definitions from workers.json file, not live '
'(default: False)'))
parser.epilog = textwrap.dedent("""
List tools used by workers in the packet-cafe server.
::
$ lim cafe tools --fit-width
+---------------+---------------------------+---------+--------+---------------+----------------+----------+---------------+
| Name | Image | Version | Labels | Stage | ViewableOutput | Outputs | Inputs |
+---------------+---------------------------+---------+--------+---------------+----------------+----------+---------------+
| pcapplot | iqtlabs/pcapplot | v0.1.5 | | analysis | True | file | pcap-splitter |
| pcap-splitter | iqtlabs/pcap_to_node_pcap | v0.11.8 | | preprocessing | False | pcap | pcap-dot1q |
| ncapture | iqtlabs/ncapture | v0.11.8 | | preprocessing | False | pcap | pcap,pcapng |
| pcap-dot1q | iqtlabs/tcprewrite_dot1q | v0.11.8 | | preprocessing | False | pcap | ncapture |
| networkml | iqtlabs/networkml | v0.5.3 | | analysis | True | rabbitmq | pcap-splitter |
| snort | iqtlabs/snort | v0.11.8 | | analysis | True | rabbitmq | pcap,pcapng |
| pcap_stats | iqtlabs/pcap_stats | v0.11.8 | | analysis | True | rabbitmq | pcap,pcapng |
| mercury | iqtlabs/mercury | v0.11.8 | | analysis | True | rabbitmq | pcap,pcapng |
| p0f | iqtlabs/p0f | v0.11.8 | | analysis | True | rabbitmq | pcap-splitter |
+---------------+---------------------------+---------+--------+---------------+----------------+----------+---------------+
The ``--definitions`` option will show the definitions as found in
the ``workers.json`` file from the repository directory, rather
than from the running system via the API. The ``--packet-cafe-repo-dir``
option controls which directory is used. This option is most useful
when developing and testing your own images to verify what images
will be used by workers after bringing up the stack.
::
$ lim cafe tools --definitions
[+] definitions from workers.json file in '/Users/dittrich/packet_cafe' (branch 'master')
. . .
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-tools
""") # noqa
parser = add_docker_global_options(parser)
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Text here also copied to docs/packet_cafe.rst
parser.epilog = textwrap.dedent("""
Produces a table listing the Docker containers associated with
Packet Café (by virtue of the ``com.docker.compose.project``
label being set to ``packet_cafe``).
::
$ lim cafe docker ps
+-------------------------+------------+--------------------------------------+---------+
| name | short_id | image | status |
+-------------------------+------------+--------------------------------------+---------+
| packet_cafe_messenger_1 | ce4eed9e01 | iqtlabs/packet_cafe_messenger:latest | running |
| packet_cafe_workers_1 | 43fff494f6 | iqtlabs/packet_cafe_workers:latest | running |
| packet_cafe_ui_1 | 794eb87ed6 | iqtlabs/packet_cafe_ui:latest | running |
| packet_cafe_web_1 | a1f8f5f7cc | iqtlabs/packet_cafe_web:latest | running |
| packet_cafe_mercury_1 | 882b12e31f | iqtlabs/mercury:v0.11.10 | running |
| packet_cafe_ncapture_1 | 5b1b10f3e0 | iqtlabs/ncapture:v0.11.10 | running |
| packet_cafe_admin_1 | 73304f16cf | iqtlabs/packet_cafe_admin:latest | running |
| packet_cafe_redis_1 | c893c408b5 | iqtlabs/packet_cafe_redis:latest | running |
| packet_cafe_lb_1 | 4530125e8e | iqtlabs/packet_cafe_lb:latest | running |
+-------------------------+------------+--------------------------------------+---------+
To just get a return value (``0`` for "all running" and ``1`` if not),
use the ``-q`` option.
::
$ lim -q cafe docker ps
$ echo $?
0
""") # noqa
parser = add_packet_cafe_global_options(parser)
return add_docker_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.epilog = textwrap.dedent("""
Return basic information about the packet-cafe service.
::
$ lim cafe admin info
+--------------+-------------------------------+
| Field | Value |
+--------------+-------------------------------+
| url | http://127.0.0.1:5001/v1/info |
| version | v0.1.0 |
| hostname | 5df1f9a14bff |
+--------------+-------------------------------+
Note that the last session ID and last request ID are found in the
output of ``lim cafe info`` (not ``lim cafe admin info``).
See https://iqtlabs.gitbook.io/packet-cafe/design/api#v-1-info
""")
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.epilog = textwrap.dedent("""
List the available admin endpoints for this packet-cafe server.
::
$ lim cafe admin endpoints
+-------------------+
| Endpoint |
+-------------------+
| /v1 |
| /v1/id/files |
| /v1/id/results |
| /v1/ids |
| /v1/info |
| /v1/logs/{req_id} |
+-------------------+
See https://iqtlabs.gitbook.io/packet-cafe/design/api#v1
""")
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.epilog = textwrap.dedent("""
Return basic information about the packet-cafe service.
Use this command to determine the last session ID and last
request ID, if available.
::
$ lim cafe info
+--------------+--------------------------------------+
| Field | Value |
+--------------+--------------------------------------+
| url | http://127.0.0.1:80/api/v1/info |
| last_session | 9a949fe6-6520-437f-89ec-e7af6925b1e0 |
| last_request | 81778bb8a9b946ba82659732baacdb44 |
| version | v0.1.0 |
| hostname | bf1456253115 |
+--------------+--------------------------------------+
To programmatically obtain the last session ID for use in other
scripts, do the following:
::
$ lim cafe info -f shell | grep last_
last_session="9a949fe6-6520-437f-89ec-e7af6925b1e0"
last_request="81778bb8a9b946ba82659732baacdb44"
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-info
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument(
'--all',
action='store_true',
dest='all',
default=False,
help=('Delete data for all sessions (careful with that '
'flag, Eugene! default: False)'))
parser.add_argument('sess_id', nargs='*', default=[])
parser.epilog = textwrap.dedent("""
Deletes all data and id directories for one or more
sessions.
As a safety feature, you must provide a session ID
on the command line or choose interactively. This command
will not default like other commands.
To select specific sessions, provide them as arguments.
You can select the desired session ID from a list of
available sessions with the ``--choose`` option, or
delete all sessions at once with ``--all``:
::
$ lim cafe admin delete --all
[+] deleted session 531f8bad-1f01-4b10-926b-a72aa27bcdba
[+] deleted session e6129371-ab97-4225-940e-5b18cd761da7
[+] deleted session 46d4f9a9-d5db-487e-a261-91764c044b44
[+] deleted session f44dc0e5-2ad0-4cbd-aac9-98a6c8233dff
[+] deleted session 5382b1b3-39f2-4563-9486-8efb99b56243
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument('sess_id', nargs='?', default=None)
parser.add_argument('req_id', nargs='?', default=None)
parser.add_argument(
'-t',
'--tool',
metavar='<tool>',
dest='tool',
default=None,
help='Only show results for specified tool (default: None)')
parser.add_argument('-P',
'--pprint',
action='store_true',
dest='pprint',
default=False,
help='Print with pprint module (default: False)')
parser.add_argument(
'-I',
'--indent',
type=_valid_counter,
dest='indent',
default=2,
help=('Indentation amount in characters (default: 2)'))
parser.add_argument(
'--no-color',
action='store_true',
dest='nocolor',
default=False,
help='Print without terminal coloring (default: False)')
parser.add_argument('-C',
'--counter',
metavar='<counter>',
type=_valid_counter,
dest='counter',
default=1,
help=('Counter for selecting a specific file '
'from a set (default: 1)'))
parser.epilog = textwrap.dedent("""
Get raw output from a specific tool, session, and request.
To select the tool from which you want output, use the ``--tool`` option.
You must select a tool (from the list produced by ``lim cafe tools``.)
::
$ lim cafe raw --tool networkml | head
[
{
"81778bb8a9b946ba82659732baacdb44": {
"valid": true,
"pcap_labels": "ip-147-32-84-79-147-32-84-165-147-32-84-79-data-udp-frame-eth-ip-port-0",
"decisions": {
"behavior": "normal",
"investigate": false
},
"classification": {
If there is more than one file, use ``--counter`` to select which one.
By default, JSON output is colored unless ``stdout`` is not a TTY (e.g.,
when piping output to another program, or redirecting output to a file.)
Disable colored output with ``--no-color``, select ``pprint`` style
pretty-printing with ``--pprint``, and control indentation with
``--indent``.
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-raw-tool-counter-sess_id-req_id
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument(
'-t',
'--tool',
metavar='<tool>',
dest='tool',
default=None,
help='Only show results for specified tool (default: None)')
parser.add_argument('sess_id', nargs='?', default=None)
parser.add_argument('req_id', nargs='?', default=None)
parser.epilog = textwrap.dedent("""
Produces a report of the results from one or more workers (tools) to
summarize the contents of a PCAP file.
If no tool(s) are specified, reports from all supported tools will
be produced.
This report is very high level and is intended to illustrate
how to get situational awareness about flows in a packet capture
to guide further more detailed analysis. It may not include all
details from a given tool. To see the full details from a worker,
use ``lim cafe raw --tool TOOL`` instead.
::
$ lim cafe report --tool p0f,networkml
[+] implicitly reusing last session id 46d4f9a9-d5db-487e-a261-91764c044b44
[+] implicitly reusing last request id a93591b554fe420ebbcf14b67fc8d298
************************************************************************************
Packet Cafe Report
Date produced: 2020-06-27T03:54:06.517174+00:00
Session ID: 46d4f9a9-d5db-487e-a261-91764c044b44
Request ID: a93591b554fe420ebbcf14b67fc8d298
File: trace_a93591b554fe420ebbcf14b67fc8d298_2020-06-21_21_44_45.pcap
Original File: test.pcap
************************************************************************************
Worker results: p0f
===================
+-----------------+----------------+----------+-------------------+---------+-------------------+
| source_ip | full_os | short_os | link | raw_mtu | mac |
+-----------------+----------------+----------+-------------------+---------+-------------------+
| 10.0.2.102 | Windows 7 or 8 | Windows | Ethernet or modem | 1500 | 08:00:27:5b:df:e1 |
| 202.44.54.4 | Windows XP | Windows | Ethernet or modem | 1500 | 52:54:00:12:35:02 |
| 190.110.121.202 | Windows XP | Windows | Ethernet or modem | 1500 | 52:54:00:12:35:02 |
| 112.213.89.90 | Windows XP | Windows | Ethernet or modem | 1500 | 52:54:00:12:35:02 |
+-----------------+----------------+----------+-------------------+---------+-------------------+
Worker results: networkml
=========================
+------------+-------------------+------------+-------------------+----------+-------------+
| source_ip | source_mac | role | confidence | behavior | investigate |
+------------+-------------------+------------+-------------------+----------+-------------+
| 10.0.2.102 | 08:00:27:5b:df:e1 | GPU laptop | 99.99999999539332 | normal | no |
+------------+-------------------+------------+-------------------+----------+-------------+
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument(
'--tree',
action='store_true',
dest='tree',
default=False,
help='Produce tree output rather than table (default: False)')
parser.add_argument('sess_id', nargs='?', default=None)
parser.add_argument('req_id', nargs='?', default=None)
parser.add_argument(
'-t',
'--tool',
metavar='<tool>',
dest='tool',
default=None,
help='Only show results for specified tool (default: None)')
parser.epilog = textwrap.dedent("""
List files produced as a result of processing uploaded files.
This can produce a large amount of output with very long lines, so
you may want to use the ``--fit-width`` option to break lines to
fit the screen.
You can get a tree listing of files, which is much more compact and
readable, with the ``--tree`` option.
::
$ lim cafe admin results --tree
id
└── 791e1034-fdb9-4fa4-a410-e1dedef7c0b8
└── dcfe1b4dd2a04d559f6600902847a11a
├── mercury
│ └── metadata.json
├── networkml
│ └── metadata.json
├── p0f
│ └── metadata.json
├── pcap_stats
│ └── metadata.json
├── pcapplot
│ ├── metadata.json
│ └── trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap
│ ├── 1
│ │ └── map_ASN-trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap.png
│ ├── 2
│ │ └── map_Private_RFC_1918-trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap.png
│ ├── 3
│ │ └── map_Source_Ports-trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap.png
│ └── 4
│ └── map_Destination_Ports-trace_dcfe1b4dd2a04d559f6600902847a11a_2020-06-21_21_44_45-client-ip-147-32-84-165-147-32-80-9-147-32-84-165-wsshort-frame-eth-dns-udp-ip-port-53.pcap.png
└── snort
└── metadata.json
You can filter results by session, by request, or by tool.
Filtering matches lines that contain all of the specified values.
To show results for a specific session or a specific request,
provide them as arguments to the command. To show only results
for a given tool, specify it with the ``-tool`` option.
::
$ lim cafe admin results --tool networkml
+---------------------------------------------------------------------------------------------------+
| Results |
+---------------------------------------------------------------------------------------------------+
| /id/6f080abf-ef71-461d-b754-a81a54fb5ad5/d709256a73b44f4e82d45f6e4ffd03e5/networkml/metadata.json |
| /id/86f71039-e6e5-44e2-90b4-3eaf27253d6d/fa142a055de24896923cc69407feeaba/networkml/metadata.json |
| /id/278adaae-df30-4d7d-883a-990ddcf6ce88/a383d781275f4dbe9e2c78ec4b8abda4/networkml/metadata.json |
| /id/bd976556-fbc3-4e2e-a808-7024c0c0f69b/6bb276459cba45b3abce9043d0bc0ad9/networkml/metadata.json |
| /id/bd976556-fbc3-4e2e-a808-7024c0c0f69b/9e74cc6f818c47ea9cd8c8ab94ce93db/networkml/metadata.json |
+---------------------------------------------------------------------------------------------------+
See https://iqtlabs.gitbook.io/packet-cafe/design/api#v-1-id-results
""") # noqa
return add_packet_cafe_global_options(parser)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.formatter_class = argparse.RawDescriptionHelpFormatter
parser.add_argument(
'--no-track',
action='store_true',
dest='no_track',
default=False,
help='Do not track worker status in real time (default: False)')
parser.add_argument(
'--ignore-errors',
action='store_true',
dest='ignore_errors',
default=False,
help=('Ignore job failures when tracking status (default: False)'))
parser.add_argument(
'--wait',
action='store_true',
dest='wait',
default=False,
help="Wait for processing to finish (default: False)")
parser.add_argument('pcap',
nargs=1,
default=None,
help='Path to PCAP file to upload')
session = parser.add_mutually_exclusive_group(required=False)
session.add_argument(
'sess_id',
nargs='?',
default=None,
help='Optional session ID (default is to generate)')
session.add_argument('--reuse-session',
action='store_true',
dest='reuse_session',
default=False,
help='Reuse the last session ID (default: False)')
parser.epilog = textwrap.dedent("""
Upload a file to the packet-cafe service for processing.
By default, the file is added to a new session. To instead
add this file to an existing session, you can (a) specify the
session ID as an argument on the command line, (b) add the
``--choose`` flag to interactively select the session ID from
existing sessions, (c) add the ``--reuse-session`` flag to
associate this file with the last session ID, or allow the
default behavior of generating a new session.
By default, basic status information is returned (including whether
the call succeeded and the session ID + request ID for this request)
and if the request was accepted, the progress of each worker is tracked
in real time similar to the web UI.
::
$ lim cafe upload ~/git/packet_cafe/notebooks/smallFlows.pcap
[+] Upload smallFlows.pcap: success
[+] Session ID (sess_id): 30b9ce67-75a4-49e6-b484-c4646b72fbd9
[+] Request ID (req_id): 4e058115ed19491193eadf58f105032b
[+] pcap_stats: complete 2020-05-23T17:29:56.982084+00:00
[+] pcap-dot1q: complete 2020-05-23T17:29:55.773211+00:00
[+] ncapture: complete 2020-05-23T17:29:53.333307+00:00
[+] mercury: complete 2020-05-23T17:29:59.330288+00:00
[+] snort: complete 2020-05-23T17:30:02.781840+00:00
[+] pcap-splitter: complete 2020-05-23T17:31:10.060056+00:00
[+] networkml: complete 2020-05-23T17:32:13.648982+00:00
[+] p0f: complete 2020-05-23T17:32:21.438466+00:00
[+] pcapplot: complete 2020-05-23T17:33:05.999342+00:00
If ``-v`` (or more) is given, even more information is produced and
tracking is performed as well.
Adding the ``--elapsed`` option includes elapsed lap time (per worker)
and total time for all workers.
::
$ lim cafe upload CTU-Malware-Capture-Botnet-114-1/2015-04-09_capture-win2.pcap --elapsed
[+] Upload 2015-04-09_capture-win2.pcap: success
[+] Session ID (sess_id): 46d4f9a9-d5db-487e-a261-91764c044b44
[+] Request ID (req_id): a93591b554fe420ebbcf14b67fc8d298
[+] ncapture: complete 2020-05-27T03:26:53.894222+00:00 (00:00:05.07)
[+] pcap_stats: complete 2020-05-27T03:26:56.531330+00:00 (00:00:05.07)
[+] pcap-dot1q: complete 2020-05-27T03:26:56.311676+00:00 (00:00:05.07)
[+] mercury: complete 2020-05-27T03:26:59.670225+00:00 (00:00:07.10)
[+] snort: complete 2020-05-27T03:27:03.241917+00:00 (00:00:11.16)
[+] pcap-splitter: complete 2020-05-27T03:27:03.122224+00:00 (00:00:11.16)
[+] p0f: complete 2020-05-27T03:27:07.341062+00:00 (00:00:15.22)
[+] networkml: complete 2020-05-27T03:27:08.732745+00:00 (00:00:17.25)
[+] pcapplot: complete 2020-05-27T03:27:10.634384+00:00 (00:00:19.27)
[+] Elapsed time 00:00:22.86
Adding the ``--no-track`` option will return the upload status and both
session and request IDs. You can then check on the status as needed
using ``lim cafe status``:
::
$ lim cafe upload test.pcap --no-track
[+] Upload test.pcap: success
[+] Session ID (sess_id): d7c9eaaa-6360-44d0-b821-097b17d1b4fb
[+] Request ID (req_id): 20c34e04b91a4fed9b4f876e67a218c9
$ lim cafe status
+------------+-------------+----------------------------------+
| Tool | State | Timestamp |
+------------+-------------+----------------------------------+
| snort | In progress | 2020-05-15T07:18:55.281469+00:00 |
| mercury | In progress | 2020-05-15T07:18:56.288996+00:00 |
| ncapture | Complete | 2020-05-15T07:18:56.881295+00:00 |
| pcap-dot1q | In progress | 2020-05-15T07:18:56.880669+00:00 |
| pcap_stats | In progress | 2020-05-15T07:18:56.923709+00:00 |
+------------+-------------+----------------------------------+
$ lim cafe status
+---------------+-------------+----------------------------------+
| Tool | State | Timestamp |
+---------------+-------------+----------------------------------+
| snort | Complete | 2020-05-15T07:19:02.913388+00:00 |
| networkml | In progress | 2020-05-15T07:19:07.484375+00:00 |
| pcap-splitter | Complete | 2020-05-15T07:19:07.994744+00:00 |
| mercury | Complete | 2020-05-15T07:19:00.197828+00:00 |
| pcap-dot1q | Complete | 2020-05-15T07:18:59.070603+00:00 |
| ncapture | Complete | 2020-05-15T07:18:56.881295+00:00 |
| pcapplot | In progress | 2020-05-15T07:19:07.046718+00:00 |
| pcap_stats | Complete | 2020-05-15T07:18:59.209291+00:00 |
| p0f | In progress | 2020-05-15T07:19:07.994061+00:00 |
+---------------+-------------+----------------------------------+
Using the ``-q`` flag will no produce any output and will also return
immediately without tracking processing. In circumstances where you are
performing lots of uploads, it may be better to wait until all processing
for each file is done is done before uploading the next file. Use the
``--wait`` flag to do this.
By default when waiting for the status of jobs, any failures result in
an error message and the program will exit. You can disable this by
using the ``--ignore-errors`` flag, but be aware that doing so may
cause the program to hang indefinitely.
See https://iqtlabs.gitbook.io/packet-cafe/design/api#api-v-1-upload
""") # noqa
return add_packet_cafe_global_options(parser)
