#!/usr/bin/env python
import kamene.all as kamene
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(
name='cap_icmp')
def capture_icmp_packets():
"""capture_icmp_packets
Capture ``ICMP`` packets and call the ``handle_packets`` method
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev(
"CAP_DEVICE",
"lo")
"""
Ignore ports for forwarding to consolidators:
Redis VM: 6379, 16379
RabbitMQ VM: 5672, 15672, 25672
"""
import socket
from spylunking.log.setup_logging import console_logger
log = console_logger(
name='create_l2_socket')
def create_layer_2_socket():
"""create_layer_2_socket"""
# create a socket for recording layer 2, 3 and 4 frames
s = None
try:
log.info("Creating l234 socket")
s = socket.socket(socket.AF_PACKET,
socket.SOCK_RAW,
socket.ntohs(0x0003))
except socket.error as msg:
log.error(("Socket could not be created ex={}")
.format(msg))
return s
# end of create_layer_2_socket
import os
import sys
import json
import argparse
import pandas as pd
from spylunking.log.setup_logging import console_logger
from antinex_client.utils import ev
from antinex_client.utils import ppj
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import SUCCESS
from antinex_client.consts import ERROR
from antinex_client.consts import FAILED
from antinex_client.build_ai_client_from_env import build_ai_client_from_env
from antinex_client.generate_ai_request import generate_ai_request
log = console_logger(name='env_predict')
def start_predictions():
"""start_predictions
Using environment variables, create an AntiNex AI Client.
You can also use command line args if you want.
This can train a new deep neural network if it does not
exist or it can use an existing pre-trained deep neural
network within the AntiNex Core to make new predictions.
"""
parser = argparse.ArgumentParser(
#!/usr/bin/env python
from spylunking.log.setup_logging import console_logger
from celery_connectors.kombu_subscriber import KombuSubscriber
from network_pipeline.consts import FORWARD_BROKER_URL
from network_pipeline.consts import FORWARD_SSL_OPTIONS
from network_pipeline.consts import FORWARD_QUEUE
from network_pipeline.record_packets_to_csv import RecordPacketsToCSV
name = 'packets_rabbitmq'
log = console_logger(name=name)
log.info("start - {}".format(name))
agg = RecordPacketsToCSV()
def recv_msg(body, message):
"""recv_msg
Handler method - fires when a messages is consumed from
the ``FORWARD_QUEUE`` queue running in the ``FORWARD_BROKER_URL``
broker.
:param body: message body
:param message: message object can ack, requeue or reject
"""
log.info(("callback received msg "))
agg.handle_msg(body=body, org_message=message)
import os
import json
import numpy as np
import pandas as pd
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.consts import VALID
from network_pipeline.consts import INVALID
from network_pipeline.consts import ERROR
from sklearn.model_selection import train_test_split
log = console_logger(name='build_training_request')
def build_training_request(
csv_file=ev("CSV_FILE", "/tmp/cleaned_attack_scans.csv"),
meta_file=ev("CSV_META_FILE", "/tmp/cleaned_metadata.json"),
predict_feature=ev("PREDICT_FEATURE", "label_value"),
ignore_features=[
"label_name",
"ip_src", # need to make this an int
"ip_dst", # need to make this an int
"eth_src", # need to make this an int
"eth_dst" # need to make this an int
],
seed=None,
test_size=float(ev("TEST_SIZE", "0.20")),
preproc_rules=None):
"""build_training_request
:param csv_file: csv file built with prepare_dataset.py
#!/usr/bin/env python
import os
import sys
import argparse
from spylunking.log.setup_logging import console_logger
from antinex_client.utils import ev
from antinex_client.utils import ppj
from antinex_client.ai_client import AIClient
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import SUCCESS
from antinex_client.consts import ERROR
from antinex_client.consts import FAILED
log = console_logger(name='get_prepared_dataset')
def get_prepared_dataset():
"""get_prepared_dataset
Get an ``MLPrepare`` by database id.
"""
parser = argparse.ArgumentParser(
description=("Python client get Prepared dataset by ID"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
help="user password",
required=False,
dest="password")
#!/usr/bin/env python
import scapy.all as scapy
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(name='cap_ssh')
def capture_tcp_packets_over_ssh():
"""capture_tcp_packets_over_ssh
Capture ``TCP`` packets over ssh
and call the ``handle_packets`` method
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev("CAP_DEVICE", "lo")
"""
Ignore ports for forwarding to consolidators:
Redis VM: 6379, 16379
RabbitMQ VM: 5672, 15672, 25672
"""
# http://biot.com/capstats/bpf.html
default_filter = ("tcp and ( port 22 )")
custom_filter = ev("NETWORK_FILTER", default_filter)
#!/usr/bin/env python
import os
import sys
import json
import argparse
from spylunking.log.setup_logging import console_logger
from antinex_client.utils import ev
from antinex_client.utils import ppj
from antinex_client.ai_client import AIClient
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import SUCCESS
from antinex_client.consts import ERROR
from antinex_client.consts import FAILED
log = console_logger(name='prepare_dataset')
def prepare_new_dataset():
"""prepare_new_dataset
Prepare a new ``MLPrepare`` record and dataset files on disk.
"""
parser = argparse.ArgumentParser(
description=("Python client to Prepare a dataset"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
help="user password",
required=False,
import socket
import time
from spylunking.log.setup_logging import console_logger
log = console_logger(name='connect_forwarder')
def connect_forwarder(forward_host=None,
forward_port=None,
max_retries=-1,
sleep_interval=1.0):
"""connect_forwarder
:param forward_host: host for receiving forwarded packets
:param forward_port: port for the forwarded packets
:param max_retries: retries, -1 = infinite
:param sleep_interval: how often to retry in this loop
"""
forward_skt = None
retry_count = 0
if max_retries == -1:
retry_count = -2
if forward_host and forward_port:
while not forward_skt and \
retry_count < max_retries:
try:
forward_skt = socket.socket()
log.info(("connecting to forward={}:{}").format(
forward_host, forward_port))
from spylunking.log.setup_logging import console_logger
from antinex_client.ai_client import AIClient
from antinex_client.consts import ANTINEX_PUBLISH_ENABLED
from antinex_client.consts import ANTINEX_URL
from antinex_client.consts import ANTINEX_CA_FILE
from antinex_client.consts import ANTINEX_KEY_FILE
from antinex_client.consts import ANTINEX_CERT_FILE
from antinex_client.consts import ANTINEX_USER
from antinex_client.consts import ANTINEX_EMAIL
from antinex_client.consts import ANTINEX_PASSWORD
from antinex_client.consts import ANTINEX_CLIENT_VERBOSE
from antinex_client.consts import ANTINEX_CLIENT_DEBUG
log = console_logger(name='build_ai_client_from_env')
def build_ai_client_from_env(verbose=ANTINEX_CLIENT_VERBOSE,
debug=ANTINEX_CLIENT_DEBUG,
ca_dir=None,
cert_file=None,
key_file=None):
"""build_ai_client_from_env
Use environment variables to build a client
:param verbose: verbose logging
:param debug: debug internal client calls
:param ca_dir: optional path to CA bundle dir
:param cert_file: optional path to x509 ssl cert file
:param key_file: optional path to x509 ssl key file
"""
from network_pipeline.consts import FILTERED
from network_pipeline.consts import UNKNOWN
from network_pipeline.consts import ETH_UNSUPPORTED
from network_pipeline.consts import IP_UNSUPPORTED
from network_pipeline.consts import TCP
from network_pipeline.consts import UDP
from network_pipeline.consts import ICMP
from network_pipeline.consts import ARP
from network_pipeline.consts import IP_PROTO_ETH
from network_pipeline.consts import ARP_PROTO_ETH
from network_pipeline.consts import TCP_PROTO_IP
from network_pipeline.consts import UDP_PROTO_IP
from network_pipeline.consts import ICMP_PROTO_IP
from spylunking.log.setup_logging import console_logger
log = console_logger(name='parse_network_data')
# Get string of 6 characters as ethernet address into dash seperated hex string
def eth_addr(f):
"""eth_addr
:param f: eth frame
"""
data = "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" % (f[0], f[1], f[2], f[3], f[4],
f[5])
return data
# end of eth_addr
import sys
import socket
import datetime
import time
import json
from spylunking.log.setup_logging import console_logger
from network_pipeline.consts import INCLUDED_IGNORE_KEY
from network_pipeline.consts import VALID
from network_pipeline.consts import TCP
from network_pipeline.consts import UDP
from network_pipeline.consts import ICMP
from network_pipeline.consts import ARP
from network_pipeline.utils import ppj
# consolidator - receives packets from network agents
log = console_logger(name='consolidate_packets')
def handle_processing_packets():
"""handle_processing_packets
Replacement packet processing engine. This is not done.
"""
host = os.getenv("LISTEN_ON_HOST", "127.0.0.1").strip().lstrip()
port = int(os.getenv("LISTEN_ON_PORT", "80").strip().lstrip())
backlog = int(os.getenv("LISTEN_BACKLOG", "5").strip().lstrip())
size = int(os.getenv("LISTEN_SIZE", "102400").strip().lstrip())
sleep_in_seconds = float(os.getenv("LISTEN_SLEEP", "0.5").strip().lstrip())
needs_response = bool(
import os
import sys
import json
import argparse
import pandas as pd
from spylunking.log.setup_logging import console_logger
from antinex_client.utils import ev
from antinex_client.utils import ppj
from antinex_client.ai_client import AIClient
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import SUCCESS
from antinex_client.consts import ERROR
from antinex_client.consts import FAILED
log = console_logger(name='ai_train')
def train_new_deep_neural_network():
"""train_new_deep_neural_network
Train a new deep neural network and store the results as a new:
``MLJob`` and ``MLJobResult`` database records.
"""
parser = argparse.ArgumentParser(
description=("Python client to Train a Deep Neural Network "
"with AntiNex Django Rest Framework"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
#!/usr/bin/env python
import scapy.all as scapy
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(name='base_capture')
def example_capture():
"""example_capture
An example capture script
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev("CAP_DEVICE", "lo")
"""
Ignore ports for forwarding to consolidators:
Redis Internal VM: 6379, 16379
RabbitMQ Internal VM: 5672, 15672, 25672
"""
# http://biot.com/capstats/bpf.html
custom_filter = ("(udp and portrange 10000-17001) "
"or (tcp and portrange 80) "
"or arp "
#!/usr/bin/env python
import kamene.all as kamene
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(name='cap_tcp')
def capture_tcp_packets():
"""capture_tcp_packets
Capture ``TCP`` packets and call the ``handle_packets`` method
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev("CAP_DEVICE", "lo")
"""
Ignore ports for forwarding to consolidators:
Redis VM: 6379, 16379
RabbitMQ VM: 5672, 15672, 25672
"""
# http://biot.com/capstats/bpf.html
default_filter = ("tcp and ( port 80 or port 443 "
"or port 8010 or port 8443 )")
custom_filter = ev("NETWORK_FILTER", default_filter)
import multiprocessing
from spylunking.log.setup_logging import console_logger
log = console_logger(name='start_consumers_for_queue')
def start_consumers_for_queue(prefix_name="worker",
num_workers=2,
tasks=None,
queue_to_consume=None,
shutdown_msg="SHUTDOWN",
consumer_class=None,
need_response=False,
callback=None):
"""start_consumers_for_queue
:param prefix_name:
:param num_workers:
:param tasks:
:param queue_to_consume:
:param shutdown_msg:
:param consumer_class:
:param need_response:
:param callback:
"""
consumers = []
if not consumer_class:
log.error("Please provide a consumer_class arg")
log.error(" like: network_pipeline.packet_consumer.PacketConsumer")
import json
from collections import defaultdict
from spylunking.log.setup_logging import console_logger
from network_pipeline.utils import ppj
from network_pipeline.consts import DEBUG_PACKETS
log = console_logger(name='ptoj')
def convert_pkt_to_json(pkg):
""" convert_pkt_to_json
Inspired by:
https://gist.githubusercontent.com/cr0hn/1b0c2e672cd0721d3a07/raw/9144676ceb12dbd545e6dce366822bbedde8de2c/pkg_to_json.py
This function convert a Scapy packet to JSON
:param pkg: A kamene package
:type pkg: objects
:return: A JSON data
:rtype: dict()
"""
results = defaultdict(dict)
try:
for index in range(0, len(pkg)):
layer = pkg[index]
# Get layer name
layer_tmp_name = str(layer.__dict__["aliastypes"][0])
#!/usr/bin/env python
import kamene.all as kamene
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(name='cap_telnet')
def capture_tcp_packets_over_telnet():
"""capture_tcp_packets_over_telnet
Capture ``TCP`` packets over telnet
and call the ``handle_packets`` method
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev("CAP_DEVICE", "lo")
"""
Ignore ports for forwarding to consolidators:
Redis VM: 6379, 16379
RabbitMQ VM: 5672, 15672, 25672
"""
# http://biot.com/capstats/bpf.html
default_filter = ("tcp and ( port 23 )")
custom_filter = ev("NETWORK_FILTER", default_filter)
from network_pipeline.consts import SOURCE
from network_pipeline.consts import FORWARD_BROKER_URL
from network_pipeline.consts import FORWARD_SSL_OPTIONS
from network_pipeline.consts import FORWARD_ENDPOINT_TYPE
from spylunking.log.setup_logging import console_logger
from celery_connectors.publisher import Publisher
log = console_logger(name='get_publisher')
def get_publisher():
"""get_publisher"""
log.info("initializing publisher")
pub = None
auth_url = ""
if FORWARD_ENDPOINT_TYPE == "redis":
auth_url = FORWARD_BROKER_URL
else:
auth_url = FORWARD_BROKER_URL
pub = Publisher(name="{}_{}".format(SOURCE, "-redis"),
auth_url=auth_url,
ssl_options=FORWARD_SSL_OPTIONS)
log.info("publisher={}".format(pub))
return pub
# end of get_publisher
pub = get_publisher()
#!/usr/bin/env python
import scapy.all as scapy
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(
name='cap_udp')
def capture_udp_packets():
"""capture_udp_packets
Capture ``UDP`` packets and call the ``handle_packets`` method
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev(
"CAP_DEVICE",
"lo")
"""
Ignore ports for forwarding to consolidators:
Redis VM: 6379, 16379
RabbitMQ VM: 5672, 15672, 25672
"""
from network_pipeline.build_packet_key import build_packet_key
from antinex_client.consts import SUCCESS
from antinex_client.consts import FAILED
from antinex_client.consts import ERROR
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import ANTINEX_PUBLISH_ENABLED
from antinex_client.consts import ANTINEX_PUBLISH_REQUEST_FILE
from antinex_client.consts import ANTINEX_USE_MODEL_NAME
from antinex_client.consts import ANTINEX_URL
from antinex_client.consts import ANTINEX_USER
from antinex_client.consts import ANTINEX_MISSING_VALUE
from antinex_client.build_ai_client_from_env import build_ai_client_from_env
from antinex_client.generate_ai_request import generate_ai_request
log = console_logger(
name='csv')
class RecordPacketsToCSV:
"""RecordPacketsToCSV"""
def __init__(self):
"""__init__"""
self.recv_msgs = []
# save every nth number of messages
self.save_after_num = int(
ev("SAVE_AFTER_NUM",
"100"))
import multiprocessing
from spylunking.log.setup_logging import console_logger
log = console_logger(
name='worker_to_process_packets')
class WorkerToProcessPackets(multiprocessing.Process):
def __init__(self,
name,
task_queue,
result_queue,
shutdown_msg="SHUTDOWN",
need_response=False,
callback=None):
"""__init__
:param name: name of the consumer
:param task_queue: queue for tasks to process
:param result_queue: send results to back here
:param shutdown_msg: custom poison pill shutdown msg
:param need_response: send a response back on result_queue
:param callback: method for processing packets
"""
multiprocessing.Process.__init__(self)
self.name = name
self.response_name = 'pcktr'
self.task_queue = task_queue
self.need_response = need_response
from antinex_client.consts import ANTINEX_OPTIMIZER
from antinex_client.consts import ANTINEX_METRICS
from antinex_client.consts import ANTINEX_HISTORIES
from antinex_client.consts import FILTER_FEATURES_DICT
from antinex_client.consts import FILTER_FEATURES
from antinex_client.consts import ANTINEX_CONVERT_DATA
from antinex_client.consts import ANTINEX_CONVERT_DATA_TYPE
from antinex_client.consts import ANTINEX_INCLUDE_FAILED_CONVERSIONS
from antinex_client.consts import ANTINEX_PUBLISH_TO_CORE
from antinex_client.consts import ANTINEX_CHECK_MISSING_PREDICT
from antinex_client.consts import ANTINEX_MISSING_VALUE
from antinex_client.consts import ANTINEX_VERSION
from antinex_client.consts import ANTINEX_CLIENT_DEBUG
log = console_logger(
name='gen_ai_req')
def generate_ai_request(
predict_rows,
req_dict=None,
req_file=ANTINEX_PUBLISH_REQUEST_FILE,
features=ANTINEX_FEATURES_TO_PROCESS,
ignore_features=ANTINEX_IGNORE_FEATURES,
sort_values=ANTINEX_SORT_VALUES,
ml_type=ANTINEX_ML_TYPE,
use_model_name=ANTINEX_USE_MODEL_NAME,
predict_feature=ANTINEX_PREDICT_FEATURE,
seed=ANTINEX_SEED,
test_size=ANTINEX_TEST_SIZE,
batch_size=ANTINEX_BATCH_SIZE,
#!/usr/bin/env python
import os
import sys
import argparse
from spylunking.log.setup_logging import console_logger
from antinex_client.utils import ev
from antinex_client.utils import ppj
from antinex_client.ai_client import AIClient
from antinex_client.consts import LOGIN_FAILED
from antinex_client.consts import SUCCESS
from antinex_client.consts import ERROR
from antinex_client.consts import FAILED
log = console_logger(name='get_job')
def get_ml_job():
"""get_ml_job
Get an ``MLJob`` by database id.
"""
parser = argparse.ArgumentParser(
description=("Python client get AI Job by ID"))
parser.add_argument("-u", help="username", required=False, dest="user")
parser.add_argument("-p",
help="user password",
required=False,
dest="password")
from network_pipeline.consts import SOURCE
from network_pipeline.consts import FORWARD_EXCHANGE
from network_pipeline.consts import FORWARD_ROUTING_KEY
from network_pipeline.consts import FORWARD_QUEUE
from spylunking.log.setup_logging import console_logger
from network_pipeline.utils import rnow
from network_pipeline.convert_pkt_to_json import convert_pkt_to_json
from network_pipeline.publisher import pub
import kamene.all as kamene
log = console_logger(
name='proc')
def handle_packets(pk):
"""handle_packets
:param pk: data packet that kamene sends in
"""
log.info(("processing with pub={}")
.format(pub))
# get the lowest layer
eth = pk.getlayer(kamene.Ether)
should_forward = False
send_msg = {"data": {},
"created": rnow(),
"source": SOURCE}
#!/usr/bin/env python
import scapy.all as scapy
from spylunking.log.setup_logging import console_logger
from celery_connectors.utils import ev
from network_pipeline.handle_packets import handle_packets
log = console_logger(name='cap_arp')
def capture_arp_packets():
"""capture_arp_packets
Capture ``ARP`` packets and call the ``handle_packets`` method
Change the network interface by ``export CAP_DEVICE=eth0``
"""
dev = ev("CAP_DEVICE", "lo")
"""
Ignore ports for forwarding to consolidators:
Redis VM: 6379, 16379
RabbitMQ VM: 5672, 15672, 25672
"""
# http://biot.com/capstats/bpf.html
default_filter = "arp"
custom_filter = ev("NETWORK_FILTER", default_filter)
from antinex_client.consts import NOT_SET
log_level = logging.INFO
log_level_str = ev("AI_CLIENT_LEVEL", "info").lower()
if log_level_str == "info":
log_level = logging.INFO
elif log_level_str == "debug":
log_level = logging.DEBUG
elif log_level_str == "silent":
log_level = logging.CRITICAL
elif log_level_str == "critical":
log_level = logging.CRITICAL
elif log_level_str == "error":
log_level = logging.ERROR
log = console_logger(name='ai_client', log_level=log_level)
class AIClient:
"""
AntiNex Python AI Client
This can use either environment variables or keyword arguments
to create a valid client.
"""
def __init__(self,
user=ev("API_USER", "user-not-set"),
password=ev("API_PASSWORD", "password-not-set"),
url=ev("API_URL", "http://localhost:8010"),
