def test_monitor_eof(model_config, output_config):
config = {
"filter": {
"type": "ip_list",
"value": ["10.0.0.10"],
},
"model": model_config,
"output": output_config,
}
monitor = Monitor(config)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:55:00.000000",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 53,
"protocol": 17,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
monitor.on_message({"type": "eof", "data": {}})
monitor.on_message({"type": "eof", "data": {}})
monitor.on_message({"type": "eof", "data": {}})
assert monitor.eof_count == 3
def test_monitor_init(model_config, output_config):
config = {
"filter": {
"type": "ip_list",
"value": ["10.0.0.10"],
},
"model": model_config,
"output": output_config,
}
Monitor(config)
assert TorClassifier.relays_filter.items == [
250128189,
1192492057,
1601169074,
1659551192,
1754629750,
2734115529,
2919223544,
2987502196,
3168175153,
55828179287476426650588888404756276908,
]
cnc_steps = [x[0] for x in CNCClassifier.model.steps]
assert cnc_steps == ["smote", "scaler", "classifier"]
def test_monitor_verdict_mac(model_config, output_config):
config = {
"filter": {
"type": "mac_list",
"value": ["11:11:11:11:11:11", "22:22:22:22:22:22"],
},
"model": model_config,
"output": output_config,
}
monitor = Monitor(config)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:55:00.000000",
"dst_mac": "11:11:11:11:11:11",
"src_mac": "ee:ee:ee:ee:ee:ee",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 53,
"protocol": 17,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
monitor.endpoints["11:11:11:11:11:11"].classifiers["dht"].positive = True
monitor.endpoints["11:11:11:11:11:11"].classifiers["dht"].reason = "test"
monitor.endpoints["11:11:11:11:11:11"].classifiers[
"stratum"].positive = True
monitor.endpoints["11:11:11:11:11:11"].classifiers[
"stratum"].reason = "test"
verdict = monitor._process_window(datetime.now())
assert verdict == {
"11:11:11:11:11:11": {
"positive": True,
"reason": {
"dht": "test",
"stratum": "test"
},
}
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("config", help="config file path")
parser.add_argument("interface", help="network interface")
args = parser.parse_args()
with open(args.config) as f:
config = json.load(f)
plugins = [x["type"] for x in config["interfaces"]]
interfaces = [
x["interface"] if x["type"] != "pstats" else x["interface"] + "_pre"
for x in config["interfaces"]
]
interfaces = []
pstats_pre = None
pstats_post = None
for x in config["interfaces"]:
if x["type"] == "pstats":
pstats_pre = x["interface"] + "_pre"
pstats_post = x["interface"]
interfaces.append(pstats_pre + ":buffer=off")
else:
interfaces.append(x["interface"] + ":buffer=off")
# Nemea setup
log.info("starting aggregator...")
p_agg = aggregator(config["nemea"], [pstats_pre, pstats_post])
log.info("starting ipfixprobe...")
p_probe = ipfixprobe(config["nemea"], args.interface, plugins, interfaces)
# BOTA setup
log.info("starting BOTA...")
monitor = Monitor(config)
collector = Collector(config["interfaces"], monitor.on_message)
collector.start()
# signal handling
signal.signal(signal.SIGINT,
partial(signal_handler, monitor, p_agg, p_probe))
# main loop
agg_rc = None
probe_rc = None
while agg_rc == None or probe_rc == None:
time.sleep(0.25)
agg_rc = p_agg.poll()
probe_rc = p_probe.poll()
while not monitor.end:
time.sleep(0.25)
time.sleep(1)
def test_monitor_verdict(model_config, output_config):
config = {
"filter": {
"type": "ip_range",
"value": "10.0.0.0/24",
},
"model": model_config,
"output": output_config,
}
monitor = Monitor(config)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T16:03:21.000000",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 80,
"protocol": 6,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
monitor.endpoints["10.0.0.10"].classifiers["dht"].positive = True
monitor.endpoints["10.0.0.10"].classifiers["dht"].reason = "test"
monitor.endpoints["10.0.0.10"].classifiers["stratum"].positive = True
monitor.endpoints["10.0.0.10"].classifiers["stratum"].reason = "test"
verdict = monitor._process_window(datetime.now())
assert verdict == {
"10.0.0.10": {
"positive": True,
"reason": {
"dht": "test",
"stratum": "test"
}
}
}
def test_monitor_invalid(model_config, output_config):
config = {
"filter": {
"type": "error",
"value": ["x", "y"],
},
"model": model_config,
"output": output_config,
}
with pytest.raises(ValueError) as e:
Monitor(config)
assert e.type == ValueError
def test_monitor_basic(model_config, output_config):
config = {
"filter": {
"type": "ip_range",
"value": "10.0.0.0/24",
},
"model": model_config,
"output": output_config,
}
monitor = Monitor(config)
assert monitor.time_start == None
assert monitor.time_next == None
assert monitor.endpoints == {}
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:55:00.000000",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 53,
"protocol": 17,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:57:01.000000",
"src_ip": "2.2.2.2",
"dst_ip": "10.0.0.20",
"src_port": 443,
"dst_port": 42000,
"protocol": 6,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:58:01.000000",
"src_ip": "10.0.0.30",
"dst_ip": "3.3.3.3",
"src_port": 42000,
"dst_port": 80,
"protocol": 6,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
assert str(monitor.time_start) == "2021-03-03 15:55:00"
assert str(monitor.time_next) == "2021-03-03 16:03:20"
assert sorted(
monitor.endpoints.keys()) == ["10.0.0.10", "10.0.0.20", "10.0.0.30"]
t_1 = monitor.endpoints["10.0.0.10"].classifiers["bytes"].time_next
t_2 = monitor.endpoints["10.0.0.20"].classifiers["bytes"].time_next
t_3 = monitor.endpoints["10.0.0.30"].classifiers["bytes"].time_next
assert str(t_1) == "2021-03-03 15:56:00"
assert str(t_2) == "2021-03-03 15:56:00"
assert str(t_3) == "2021-03-03 15:59:00"
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T16:03:21.000000",
"src_ip": "10.0.0.30",
"dst_ip": "4.4.4.4",
"src_port": 42000,
"dst_port": 80,
"protocol": 6,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
assert str(monitor.time_start) == "2021-03-03 16:03:20"
assert str(monitor.time_next) == "2021-03-03 16:11:40"
def test_monitor_mac_list(model_config, output_config):
config = {
"filter": {
"type": "mac_list",
"value": ["11:11:11:11:11:11", "22:22:22:22:22:22"],
},
"model": model_config,
"output": output_config,
}
monitor = Monitor(config)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:55:00.000000",
"dst_mac": "11:11:11:11:11:11",
"src_mac": "ee:ee:ee:ee:ee:ee",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 53,
"protocol": 17,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:55:00.000000",
"src_mac": "22:22:22:22:22:22",
"dst_mac": "ff:ff:ff:ff:ff:ff",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 53,
"protocol": 17,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
message = {
"type": "basic",
"data": {
"time_last": "2021-03-03T15:55:00.000000",
"src_mac": "33:33:33:33:33:33",
"dst_mac": "ff:ff:ff:ff:ff:ff",
"src_ip": "10.0.0.10",
"dst_ip": "1.1.1.1",
"src_port": 42000,
"dst_port": 53,
"protocol": 17,
"bytes": 100,
"packets": 100,
},
}
monitor.on_message(message)
assert sorted(monitor.endpoints.keys()) == [
"11:11:11:11:11:11",
"22:22:22:22:22:22",
]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("config", help="config file path")
parser.add_argument("dir", help="pcap path")
args = parser.parse_args()
with open(args.config) as f:
config = json.load(f)
plugins = [x["type"] for x in config["interfaces"]]
interfaces = [
x["interface"] if x["type"] != "pstats" else x["interface"] + "_pre"
for x in config["interfaces"]
]
interfaces = []
pstats_pre = None
pstats_post = None
for x in config["interfaces"]:
if x["type"] == "pstats":
pstats_pre = x["interface"] + "_pre"
pstats_post = x["interface"]
interfaces.append(pstats_pre + ":buffer=off")
else:
interfaces.append(x["interface"] + ":buffer=off")
prefix = "siliconHillLong4"
log.info(f"analyzing {prefix}")
# Nemea setup
log.info("starting traffic_repeater 1")
p_tr_1 = traffic_repeater(
[f"f:{args.dir}/{prefix}-basic.trapcap", f"u:collector_basic"])
log.info("starting traffic_repeater 2")
p_tr_2 = traffic_repeater([
f"f:{args.dir}/{prefix}-idpcontent.trapcap", f"u:collector_idpcontent"
])
log.info("starting traffic_repeater 3")
p_tr_3 = traffic_repeater(
[f"f:{args.dir}/{prefix}-pstats.trapcap", pstats_pre])
log.info("starting aggregator...")
p_agg = aggregator(config["nemea"], [pstats_pre, pstats_post])
# BOTA setup
log.info("starting BOTA...")
monitor = Monitor(config)
collector = Collector(config["interfaces"], monitor.on_message)
collector.start()
# signal handling
signal.signal(
signal.SIGINT,
partial(signal_handler, monitor, [p_agg, p_tr_1, p_tr_2, p_tr_3]),
)
# main loop
agg_rc = None
tr_1_rc = None
tr_2_rc = None
tr_3_rc = None
while agg_rc == None or tr_1_rc == None or tr_2_rc == None or tr_3_rc == None:
time.sleep(0.25)
agg_rc = p_agg.poll()
tr_1_rc = p_tr_1.poll()
tr_2_rc = p_tr_2.poll()
tr_3_rc = p_tr_3.poll()
time.sleep(1)