#!/usr/bin/env python import kamene.all as kamene from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger( name='cap_icmp') def capture_icmp_packets(): """capture_icmp_packets Capture ``ICMP`` packets and call the ``handle_packets`` method Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev( "CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis VM: 6379, 16379 RabbitMQ VM: 5672, 15672, 25672 """
import socket from spylunking.log.setup_logging import console_logger log = console_logger( name='create_l2_socket') def create_layer_2_socket(): """create_layer_2_socket""" # create a socket for recording layer 2, 3 and 4 frames s = None try: log.info("Creating l234 socket") s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.ntohs(0x0003)) except socket.error as msg: log.error(("Socket could not be created ex={}") .format(msg)) return s # end of create_layer_2_socket
import os import sys import json import argparse import pandas as pd from spylunking.log.setup_logging import console_logger from antinex_client.utils import ev from antinex_client.utils import ppj from antinex_client.consts import LOGIN_FAILED from antinex_client.consts import SUCCESS from antinex_client.consts import ERROR from antinex_client.consts import FAILED from antinex_client.build_ai_client_from_env import build_ai_client_from_env from antinex_client.generate_ai_request import generate_ai_request log = console_logger(name='env_predict') def start_predictions(): """start_predictions Using environment variables, create an AntiNex AI Client. You can also use command line args if you want. This can train a new deep neural network if it does not exist or it can use an existing pre-trained deep neural network within the AntiNex Core to make new predictions. """ parser = argparse.ArgumentParser(
#!/usr/bin/env python from spylunking.log.setup_logging import console_logger from celery_connectors.kombu_subscriber import KombuSubscriber from network_pipeline.consts import FORWARD_BROKER_URL from network_pipeline.consts import FORWARD_SSL_OPTIONS from network_pipeline.consts import FORWARD_QUEUE from network_pipeline.record_packets_to_csv import RecordPacketsToCSV name = 'packets_rabbitmq' log = console_logger(name=name) log.info("start - {}".format(name)) agg = RecordPacketsToCSV() def recv_msg(body, message): """recv_msg Handler method - fires when a messages is consumed from the ``FORWARD_QUEUE`` queue running in the ``FORWARD_BROKER_URL`` broker. :param body: message body :param message: message object can ack, requeue or reject """ log.info(("callback received msg ")) agg.handle_msg(body=body, org_message=message)
import os import json import numpy as np import pandas as pd from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.consts import VALID from network_pipeline.consts import INVALID from network_pipeline.consts import ERROR from sklearn.model_selection import train_test_split log = console_logger(name='build_training_request') def build_training_request( csv_file=ev("CSV_FILE", "/tmp/cleaned_attack_scans.csv"), meta_file=ev("CSV_META_FILE", "/tmp/cleaned_metadata.json"), predict_feature=ev("PREDICT_FEATURE", "label_value"), ignore_features=[ "label_name", "ip_src", # need to make this an int "ip_dst", # need to make this an int "eth_src", # need to make this an int "eth_dst" # need to make this an int ], seed=None, test_size=float(ev("TEST_SIZE", "0.20")), preproc_rules=None): """build_training_request :param csv_file: csv file built with prepare_dataset.py
#!/usr/bin/env python import os import sys import argparse from spylunking.log.setup_logging import console_logger from antinex_client.utils import ev from antinex_client.utils import ppj from antinex_client.ai_client import AIClient from antinex_client.consts import LOGIN_FAILED from antinex_client.consts import SUCCESS from antinex_client.consts import ERROR from antinex_client.consts import FAILED log = console_logger(name='get_prepared_dataset') def get_prepared_dataset(): """get_prepared_dataset Get an ``MLPrepare`` by database id. """ parser = argparse.ArgumentParser( description=("Python client get Prepared dataset by ID")) parser.add_argument("-u", help="username", required=False, dest="user") parser.add_argument("-p", help="user password", required=False, dest="password")
#!/usr/bin/env python import scapy.all as scapy from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger(name='cap_ssh') def capture_tcp_packets_over_ssh(): """capture_tcp_packets_over_ssh Capture ``TCP`` packets over ssh and call the ``handle_packets`` method Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev("CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis VM: 6379, 16379 RabbitMQ VM: 5672, 15672, 25672 """ # http://biot.com/capstats/bpf.html default_filter = ("tcp and ( port 22 )") custom_filter = ev("NETWORK_FILTER", default_filter)
#!/usr/bin/env python import os import sys import json import argparse from spylunking.log.setup_logging import console_logger from antinex_client.utils import ev from antinex_client.utils import ppj from antinex_client.ai_client import AIClient from antinex_client.consts import LOGIN_FAILED from antinex_client.consts import SUCCESS from antinex_client.consts import ERROR from antinex_client.consts import FAILED log = console_logger(name='prepare_dataset') def prepare_new_dataset(): """prepare_new_dataset Prepare a new ``MLPrepare`` record and dataset files on disk. """ parser = argparse.ArgumentParser( description=("Python client to Prepare a dataset")) parser.add_argument("-u", help="username", required=False, dest="user") parser.add_argument("-p", help="user password", required=False,
import socket import time from spylunking.log.setup_logging import console_logger log = console_logger(name='connect_forwarder') def connect_forwarder(forward_host=None, forward_port=None, max_retries=-1, sleep_interval=1.0): """connect_forwarder :param forward_host: host for receiving forwarded packets :param forward_port: port for the forwarded packets :param max_retries: retries, -1 = infinite :param sleep_interval: how often to retry in this loop """ forward_skt = None retry_count = 0 if max_retries == -1: retry_count = -2 if forward_host and forward_port: while not forward_skt and \ retry_count < max_retries: try: forward_skt = socket.socket() log.info(("connecting to forward={}:{}").format( forward_host, forward_port))
from spylunking.log.setup_logging import console_logger from antinex_client.ai_client import AIClient from antinex_client.consts import ANTINEX_PUBLISH_ENABLED from antinex_client.consts import ANTINEX_URL from antinex_client.consts import ANTINEX_CA_FILE from antinex_client.consts import ANTINEX_KEY_FILE from antinex_client.consts import ANTINEX_CERT_FILE from antinex_client.consts import ANTINEX_USER from antinex_client.consts import ANTINEX_EMAIL from antinex_client.consts import ANTINEX_PASSWORD from antinex_client.consts import ANTINEX_CLIENT_VERBOSE from antinex_client.consts import ANTINEX_CLIENT_DEBUG log = console_logger(name='build_ai_client_from_env') def build_ai_client_from_env(verbose=ANTINEX_CLIENT_VERBOSE, debug=ANTINEX_CLIENT_DEBUG, ca_dir=None, cert_file=None, key_file=None): """build_ai_client_from_env Use environment variables to build a client :param verbose: verbose logging :param debug: debug internal client calls :param ca_dir: optional path to CA bundle dir :param cert_file: optional path to x509 ssl cert file :param key_file: optional path to x509 ssl key file """
from network_pipeline.consts import FILTERED from network_pipeline.consts import UNKNOWN from network_pipeline.consts import ETH_UNSUPPORTED from network_pipeline.consts import IP_UNSUPPORTED from network_pipeline.consts import TCP from network_pipeline.consts import UDP from network_pipeline.consts import ICMP from network_pipeline.consts import ARP from network_pipeline.consts import IP_PROTO_ETH from network_pipeline.consts import ARP_PROTO_ETH from network_pipeline.consts import TCP_PROTO_IP from network_pipeline.consts import UDP_PROTO_IP from network_pipeline.consts import ICMP_PROTO_IP from spylunking.log.setup_logging import console_logger log = console_logger(name='parse_network_data') # Get string of 6 characters as ethernet address into dash seperated hex string def eth_addr(f): """eth_addr :param f: eth frame """ data = "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x" % (f[0], f[1], f[2], f[3], f[4], f[5]) return data # end of eth_addr
import sys import socket import datetime import time import json from spylunking.log.setup_logging import console_logger from network_pipeline.consts import INCLUDED_IGNORE_KEY from network_pipeline.consts import VALID from network_pipeline.consts import TCP from network_pipeline.consts import UDP from network_pipeline.consts import ICMP from network_pipeline.consts import ARP from network_pipeline.utils import ppj # consolidator - receives packets from network agents log = console_logger(name='consolidate_packets') def handle_processing_packets(): """handle_processing_packets Replacement packet processing engine. This is not done. """ host = os.getenv("LISTEN_ON_HOST", "127.0.0.1").strip().lstrip() port = int(os.getenv("LISTEN_ON_PORT", "80").strip().lstrip()) backlog = int(os.getenv("LISTEN_BACKLOG", "5").strip().lstrip()) size = int(os.getenv("LISTEN_SIZE", "102400").strip().lstrip()) sleep_in_seconds = float(os.getenv("LISTEN_SLEEP", "0.5").strip().lstrip()) needs_response = bool(
import os import sys import json import argparse import pandas as pd from spylunking.log.setup_logging import console_logger from antinex_client.utils import ev from antinex_client.utils import ppj from antinex_client.ai_client import AIClient from antinex_client.consts import LOGIN_FAILED from antinex_client.consts import SUCCESS from antinex_client.consts import ERROR from antinex_client.consts import FAILED log = console_logger(name='ai_train') def train_new_deep_neural_network(): """train_new_deep_neural_network Train a new deep neural network and store the results as a new: ``MLJob`` and ``MLJobResult`` database records. """ parser = argparse.ArgumentParser( description=("Python client to Train a Deep Neural Network " "with AntiNex Django Rest Framework")) parser.add_argument("-u", help="username", required=False, dest="user") parser.add_argument("-p",
#!/usr/bin/env python import scapy.all as scapy from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger(name='base_capture') def example_capture(): """example_capture An example capture script Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev("CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis Internal VM: 6379, 16379 RabbitMQ Internal VM: 5672, 15672, 25672 """ # http://biot.com/capstats/bpf.html custom_filter = ("(udp and portrange 10000-17001) " "or (tcp and portrange 80) " "or arp "
#!/usr/bin/env python import kamene.all as kamene from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger(name='cap_tcp') def capture_tcp_packets(): """capture_tcp_packets Capture ``TCP`` packets and call the ``handle_packets`` method Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev("CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis VM: 6379, 16379 RabbitMQ VM: 5672, 15672, 25672 """ # http://biot.com/capstats/bpf.html default_filter = ("tcp and ( port 80 or port 443 " "or port 8010 or port 8443 )") custom_filter = ev("NETWORK_FILTER", default_filter)
import multiprocessing from spylunking.log.setup_logging import console_logger log = console_logger(name='start_consumers_for_queue') def start_consumers_for_queue(prefix_name="worker", num_workers=2, tasks=None, queue_to_consume=None, shutdown_msg="SHUTDOWN", consumer_class=None, need_response=False, callback=None): """start_consumers_for_queue :param prefix_name: :param num_workers: :param tasks: :param queue_to_consume: :param shutdown_msg: :param consumer_class: :param need_response: :param callback: """ consumers = [] if not consumer_class: log.error("Please provide a consumer_class arg") log.error(" like: network_pipeline.packet_consumer.PacketConsumer")
import json from collections import defaultdict from spylunking.log.setup_logging import console_logger from network_pipeline.utils import ppj from network_pipeline.consts import DEBUG_PACKETS log = console_logger(name='ptoj') def convert_pkt_to_json(pkg): """ convert_pkt_to_json Inspired by: https://gist.githubusercontent.com/cr0hn/1b0c2e672cd0721d3a07/raw/9144676ceb12dbd545e6dce366822bbedde8de2c/pkg_to_json.py This function convert a Scapy packet to JSON :param pkg: A kamene package :type pkg: objects :return: A JSON data :rtype: dict() """ results = defaultdict(dict) try: for index in range(0, len(pkg)): layer = pkg[index] # Get layer name layer_tmp_name = str(layer.__dict__["aliastypes"][0])
#!/usr/bin/env python import kamene.all as kamene from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger(name='cap_telnet') def capture_tcp_packets_over_telnet(): """capture_tcp_packets_over_telnet Capture ``TCP`` packets over telnet and call the ``handle_packets`` method Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev("CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis VM: 6379, 16379 RabbitMQ VM: 5672, 15672, 25672 """ # http://biot.com/capstats/bpf.html default_filter = ("tcp and ( port 23 )") custom_filter = ev("NETWORK_FILTER", default_filter)
from network_pipeline.consts import SOURCE from network_pipeline.consts import FORWARD_BROKER_URL from network_pipeline.consts import FORWARD_SSL_OPTIONS from network_pipeline.consts import FORWARD_ENDPOINT_TYPE from spylunking.log.setup_logging import console_logger from celery_connectors.publisher import Publisher log = console_logger(name='get_publisher') def get_publisher(): """get_publisher""" log.info("initializing publisher") pub = None auth_url = "" if FORWARD_ENDPOINT_TYPE == "redis": auth_url = FORWARD_BROKER_URL else: auth_url = FORWARD_BROKER_URL pub = Publisher(name="{}_{}".format(SOURCE, "-redis"), auth_url=auth_url, ssl_options=FORWARD_SSL_OPTIONS) log.info("publisher={}".format(pub)) return pub # end of get_publisher pub = get_publisher()
#!/usr/bin/env python import scapy.all as scapy from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger( name='cap_udp') def capture_udp_packets(): """capture_udp_packets Capture ``UDP`` packets and call the ``handle_packets`` method Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev( "CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis VM: 6379, 16379 RabbitMQ VM: 5672, 15672, 25672 """
from network_pipeline.build_packet_key import build_packet_key from antinex_client.consts import SUCCESS from antinex_client.consts import FAILED from antinex_client.consts import ERROR from antinex_client.consts import LOGIN_FAILED from antinex_client.consts import ANTINEX_PUBLISH_ENABLED from antinex_client.consts import ANTINEX_PUBLISH_REQUEST_FILE from antinex_client.consts import ANTINEX_USE_MODEL_NAME from antinex_client.consts import ANTINEX_URL from antinex_client.consts import ANTINEX_USER from antinex_client.consts import ANTINEX_MISSING_VALUE from antinex_client.build_ai_client_from_env import build_ai_client_from_env from antinex_client.generate_ai_request import generate_ai_request log = console_logger( name='csv') class RecordPacketsToCSV: """RecordPacketsToCSV""" def __init__(self): """__init__""" self.recv_msgs = [] # save every nth number of messages self.save_after_num = int( ev("SAVE_AFTER_NUM", "100"))
import multiprocessing from spylunking.log.setup_logging import console_logger log = console_logger( name='worker_to_process_packets') class WorkerToProcessPackets(multiprocessing.Process): def __init__(self, name, task_queue, result_queue, shutdown_msg="SHUTDOWN", need_response=False, callback=None): """__init__ :param name: name of the consumer :param task_queue: queue for tasks to process :param result_queue: send results to back here :param shutdown_msg: custom poison pill shutdown msg :param need_response: send a response back on result_queue :param callback: method for processing packets """ multiprocessing.Process.__init__(self) self.name = name self.response_name = 'pcktr' self.task_queue = task_queue self.need_response = need_response
from antinex_client.consts import ANTINEX_OPTIMIZER from antinex_client.consts import ANTINEX_METRICS from antinex_client.consts import ANTINEX_HISTORIES from antinex_client.consts import FILTER_FEATURES_DICT from antinex_client.consts import FILTER_FEATURES from antinex_client.consts import ANTINEX_CONVERT_DATA from antinex_client.consts import ANTINEX_CONVERT_DATA_TYPE from antinex_client.consts import ANTINEX_INCLUDE_FAILED_CONVERSIONS from antinex_client.consts import ANTINEX_PUBLISH_TO_CORE from antinex_client.consts import ANTINEX_CHECK_MISSING_PREDICT from antinex_client.consts import ANTINEX_MISSING_VALUE from antinex_client.consts import ANTINEX_VERSION from antinex_client.consts import ANTINEX_CLIENT_DEBUG log = console_logger( name='gen_ai_req') def generate_ai_request( predict_rows, req_dict=None, req_file=ANTINEX_PUBLISH_REQUEST_FILE, features=ANTINEX_FEATURES_TO_PROCESS, ignore_features=ANTINEX_IGNORE_FEATURES, sort_values=ANTINEX_SORT_VALUES, ml_type=ANTINEX_ML_TYPE, use_model_name=ANTINEX_USE_MODEL_NAME, predict_feature=ANTINEX_PREDICT_FEATURE, seed=ANTINEX_SEED, test_size=ANTINEX_TEST_SIZE, batch_size=ANTINEX_BATCH_SIZE,
#!/usr/bin/env python import os import sys import argparse from spylunking.log.setup_logging import console_logger from antinex_client.utils import ev from antinex_client.utils import ppj from antinex_client.ai_client import AIClient from antinex_client.consts import LOGIN_FAILED from antinex_client.consts import SUCCESS from antinex_client.consts import ERROR from antinex_client.consts import FAILED log = console_logger(name='get_job') def get_ml_job(): """get_ml_job Get an ``MLJob`` by database id. """ parser = argparse.ArgumentParser( description=("Python client get AI Job by ID")) parser.add_argument("-u", help="username", required=False, dest="user") parser.add_argument("-p", help="user password", required=False, dest="password")
from network_pipeline.consts import SOURCE from network_pipeline.consts import FORWARD_EXCHANGE from network_pipeline.consts import FORWARD_ROUTING_KEY from network_pipeline.consts import FORWARD_QUEUE from spylunking.log.setup_logging import console_logger from network_pipeline.utils import rnow from network_pipeline.convert_pkt_to_json import convert_pkt_to_json from network_pipeline.publisher import pub import kamene.all as kamene log = console_logger( name='proc') def handle_packets(pk): """handle_packets :param pk: data packet that kamene sends in """ log.info(("processing with pub={}") .format(pub)) # get the lowest layer eth = pk.getlayer(kamene.Ether) should_forward = False send_msg = {"data": {}, "created": rnow(), "source": SOURCE}
#!/usr/bin/env python import scapy.all as scapy from spylunking.log.setup_logging import console_logger from celery_connectors.utils import ev from network_pipeline.handle_packets import handle_packets log = console_logger(name='cap_arp') def capture_arp_packets(): """capture_arp_packets Capture ``ARP`` packets and call the ``handle_packets`` method Change the network interface by ``export CAP_DEVICE=eth0`` """ dev = ev("CAP_DEVICE", "lo") """ Ignore ports for forwarding to consolidators: Redis VM: 6379, 16379 RabbitMQ VM: 5672, 15672, 25672 """ # http://biot.com/capstats/bpf.html default_filter = "arp" custom_filter = ev("NETWORK_FILTER", default_filter)
from antinex_client.consts import NOT_SET log_level = logging.INFO log_level_str = ev("AI_CLIENT_LEVEL", "info").lower() if log_level_str == "info": log_level = logging.INFO elif log_level_str == "debug": log_level = logging.DEBUG elif log_level_str == "silent": log_level = logging.CRITICAL elif log_level_str == "critical": log_level = logging.CRITICAL elif log_level_str == "error": log_level = logging.ERROR log = console_logger(name='ai_client', log_level=log_level) class AIClient: """ AntiNex Python AI Client This can use either environment variables or keyword arguments to create a valid client. """ def __init__(self, user=ev("API_USER", "user-not-set"), password=ev("API_PASSWORD", "password-not-set"), url=ev("API_URL", "http://localhost:8010"),