def new_connection_test():
ret = True
# CONFIG
connection_filter = Connection()
connection_filter.init_data(
["42.42.42.1;42.42.42.2;1", "42.42.42.1;42.42.42.2;43;17"])
connection_filter.configure()
# START FILTER
if not connection_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=connection_filter.socket,
socket_type="unix",
)
results = darwin_api.bulk_call(
[
["42.42.42.2", "42.42.42.3", "42", "6"],
],
filter_code="CONNECTION",
response_type="back",
)
# VERIFY RESULTS
certitudes = results.get('certitude_list')
expected_certitudes = [100]
expected_certitudes_size = 1
if certitudes is None:
ret = False
logging.error("Connection Test : No certitude list found in result")
if len(certitudes) != 1:
ret = False
logging.error(
"Connection Test : Unexpected certitude size of {} instead of {}".
format(len(certitudes), expected_certitudes_size))
if certitudes != expected_certitudes:
ret = False
logging.error(
"Connection Test : Unexpected certitude of {} instead of {}".
format(certitudes, expected_certitudes))
# CLEAN
darwin_api.close()
connection_filter.clean_files()
# ret = connection_filter.valgrind_stop() or connection_filter.valgrind_stop()
# would erase upper ret if this function return True
if not connection_filter.valgrind_stop():
ret = False
return ret
def test(test_name, init_data, data, expected_certitudes,
expected_certitudes_size):
ret = True
# CONFIG
hostlookup_filter = HostLookup()
# All the trusted hosts
hostlookup_filter.init_data(init_data)
hostlookup_filter.configure()
# START FILTER
if not hostlookup_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=hostlookup_filter.socket,
socket_type="unix",
)
results = darwin_api.bulk_call(
data,
filter_code="HOSTLOOKUP",
response_type="back",
)
# VERIFY RESULTS
certitudes = results.get('certitude_list')
if certitudes is None:
ret = False
logging.error(
"Hostlookup Test : {} : No certitude list found in result".format(
test_name))
if len(certitudes) != expected_certitudes_size:
ret = False
logging.error(
"HostLookup Test : {} : Unexpected certitude size of {} instead of {}"
.format(test_name, len(certitudes), expected_certitudes_size))
if certitudes != expected_certitudes:
ret = False
logging.error(
"HostLookup Test : {} : Unexpected certitude of {} instead of {}".
format(test_name, certitudes, expected_certitudes))
# CLEAN
darwin_api.close()
hostlookup_filter.clean_files()
# ret = hostlookup_filter.valgrind_stop() or hostlookup_filter.valgrind_stop()
# would erase upper ret if this function return True
if not hostlookup_filter.valgrind_stop():
ret = False
return ret
def send_single(self, line):
"""
Send a single line.
"""
api = DarwinApi(socket_type="unix", socket_path=self.socket)
ret = api.call([line], response_type="back")
api.close()
return ret
def send(self, data):
header = DarwinPacket(packet_type="other",
response_type="no",
filter_code=0x544D4C59,
event_id=uuid.uuid4().hex,
body_size=len(data))
api = DarwinApi(socket_type="unix", socket_path=self.socket)
api.socket.sendall(header)
api.socket.sendall(data)
api.close()
def test(test_name, data, expected_certitudes, expected_certitudes_size):
ret = True
# CONFIG
anomaly_filter = Anomaly()
# All the trusted hosts
anomaly_filter.configure()
# START FILTER
if not anomaly_filter.start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=anomaly_filter.socket,
socket_type="unix",
)
results = darwin_api.bulk_call(
data,
filter_code="ANOMALY",
response_type="back",
)
# VERIFY RESULTS
certitudes = results.get('certitude_list')
if certitudes is None:
ret = False
logging.error(
"Anomaly Test : {} : No certitude list found in result".format(
test_name))
if len(certitudes) != expected_certitudes_size:
ret = False
logging.error(
"Anomaly Test : {} : Unexpected certitude size of {} instead of {}"
.format(test_name, len(certitudes), expected_certitudes_size))
if certitudes != expected_certitudes:
ret = False
logging.error(
"Anomaly Test : {} : Unexpected certitude of {} instead of {}".
format(test_name, certitudes, expected_certitudes))
# CLEAN
darwin_api.close()
anomaly_filter.clean_files()
if not anomaly_filter.stop():
ret = False
return ret
def python_false_return_test():
ret = True
# CONFIG
sofa_filter = Sofa()
sofa_filter.configure()
# START FILTER
if not sofa_filter.start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=sofa_filter.socket,
socket_type="unix",
)
results = darwin_api.bulk_call(
[["192.168.1.31", "trigger_false", "Linux 2.6.39", "tcp", "80"],
["192.168.1.31", "", "Microsoft Windows Server 2003 R2", "tcp", "21"]
],
filter_code="SOFA",
response_type="back",
)
# VERIFY RESULTS
results = results.get('body')
if results is not '':
logging.error(
"Sofa test : python_false_return_test : expected no result but got {} instead"
.format(results))
ret = False
# CLEAN
darwin_api.close()
sofa_filter.clean_files()
# ret = hostlookup_filter.valgrind_stop() or hostlookup_filter.valgrind_stop()
# would erase upper ret if this function return True
if not sofa_filter.stop():
ret = False
# Test if the filter didn't crash and can still process entries
if ret and not good_format_data_test():
ret = False
return ret
def log(self, log_line):
"""
Send a single log line.
"""
header = DarwinPacket(
packet_type="other",
response_type="no",
filter_code=0x4C4F4753,
event_id=uuid.uuid4().hex,
body_size=len(log_line)
)
api = DarwinApi(socket_type="unix", socket_path=self.socket)
api.socket.sendall(header)
api.socket.sendall(log_line)
api.close()
def redis_test(test_name, data, expectations, config=None, bulk=True):
def data_to_bytes(data):
res = set()
for d in data:
d = map(str, d)
res.add(str.encode(';'.join(d)))
return res
ret = True
# CONFIG
buffer_filter = Buffer()
buffer_filter.configure(config)
# START FILTER
if not buffer_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=buffer_filter.socket,
socket_type="unix",
)
if bulk:
darwin_api.bulk_call(
data,
response_type="back",
)
else:
for line in data:
darwin_api.call(line, response_type="back")
for expect_pair in expectations:
redis_data = buffer_filter.get_internal_redis_set_data(expect_pair[0])
expected_data = data_to_bytes(expect_pair[1])
if redis_data != expected_data:
logging.error(
"{}: Expected this data : {} but got {} in redis".format(
test_name, expected_data, redis_data))
ret = False
# CLEAN
darwin_api.close()
# ret = buffer_filter.valgrind_stop() or buffer_filter.valgrind_stop()
# would erase upper ret if this function return True
if not buffer_filter.valgrind_stop():
ret = False
return ret
def alert_in_file_test():
ret = True
# CONFIG
tanomaly_filter = TAnomaly()
tanomaly_filter.configure()
# START FILTER
if not tanomaly_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=tanomaly_filter.socket,
socket_type="unix",
)
darwin_api.bulk_call(
tanomaly_filter.get_test_data(),
filter_code="TANOMALY",
response_type="back",
)
# We wait for the thread to activate
# Wait more to let detection happen with valgrind
sleep(30)
redis_alerts = tanomaly_filter.get_file_alerts()
redis_alerts = [
format_alert(a, "alerts_in_file_test") for a in redis_alerts
]
if redis_alerts is None:
ret = False
logging.error("alert_in_file_test : No alerts writing in alert file")
if len(redis_alerts) != len(EXPECTED_ALERTS):
ret = False
logging.error(
"alerts_in_file_test : Not the expected data in file. Got : {}, expected : {}"
.format(redis_alerts, EXPECTED_ALERTS))
for a in redis_alerts:
if a not in EXPECTED_ALERTS:
ret = False
logging.error(
"alerts_in_file_test : Not the expected data in file. Got : {}, expected : {}"
.format(redis_alerts, EXPECTED_ALERTS))
# CLEAN
darwin_api.close()
tanomaly_filter.clean_files()
# ret = tanomaly_filter.valgrind_stop() or tanomaly_filter.valgrind_stop()
# would erase upper ret if this function return True
if not tanomaly_filter.valgrind_stop():
ret = False
return ret
def thread_working_test():
ret = True
# CONFIG
tanomaly_filter = TAnomaly()
tanomaly_filter.configure()
# START FILTER
if not tanomaly_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=tanomaly_filter.socket,
socket_type="unix",
)
darwin_api.bulk_call(
[["73.90.76.52", "99.184.81.66", "1017", "17"],
["250.230.92.234", "54.220.65.198", "2922", "6"],
["171.104.231.132", "0.127.226.192", "467", "17"],
["42.214.30.108", "246.163.54.146", "2979", "1"],
["79.187.169.202", "46.126.241.248", "2677", "1"],
["57.126.101.247", "255.171.17.199", "2468", "17"],
["102.27.128.38", "75.125.227.149", "2249", "1"],
["116.145.214.73", "182.20.121.254", "1687", "1"],
["248.78.140.91", "112.67.123.34", "1119", "1"],
["47.159.155.135", "117.9.1.88", "1740", "6"]],
filter_code="TANOMALY",
response_type="back",
)
# We wait for the thread to activate
sleep(12)
redis_data = tanomaly_filter.get_internal_redis_data()
if redis_data != set():
logging.error(
"thread_working_test : Expected no data in Redis but got {}".
format(redis_data))
ret = False
# CLEAN
darwin_api.close()
tanomaly_filter.clean_files()
# ret = tanomaly_filter.valgrind_stop() or tanomaly_filter.valgrind_stop()
# would erase upper ret if this function return True
if not tanomaly_filter.valgrind_stop():
ret = False
return ret
def redis_test(test_name, data, expected_data):
def data_to_bytes(data):
res = set()
for d in data:
res.add(str.encode(";".join(d)))
return res
ret = True
# CONFIG
tanomaly_filter = TAnomaly()
tanomaly_filter.configure()
# START FILTER
if not tanomaly_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=tanomaly_filter.socket,
socket_type="unix",
)
darwin_api.bulk_call(
data,
filter_code="TANOMALY",
response_type="back",
)
redis_data = tanomaly_filter.get_internal_redis_data()
expected_data = data_to_bytes(expected_data)
if redis_data != expected_data:
logging.error("{}: Expected this data : {} but got {} in redis".format(
test_name, expected_data, redis_data))
ret = False
# CLEAN
darwin_api.close()
tanomaly_filter.clean_files()
# ret = tanomaly_filter.valgrind_stop() or tanomaly_filter.valgrind_stop()
# would erase upper ret if this function return True
if not tanomaly_filter.valgrind_stop():
ret = False
return ret
def check_socket_connection():
filter = Filter(filter_name="logs")
pid = -1
filter.configure(FLOGS_CONFIG)
filter.valgrind_start()
try:
api = DarwinApi(socket_path=filter.socket, socket_type="unix")
api.call("test\n", filter_code="logs", response_type="back")
api.close()
except Exception as e:
logging.error("check_socket_connection_back: Error connecting to socket: {}".format(e))
return False
filter.stop()
return True
"""examples.py: Provide some examples when using the Darwin API"""
__author__ = "gcatto"
__version__ = "1.0"
__date__ = "05/02/19"
__license__ = "GPLv3"
__copyright__ = "Copyright (c) 2019 Advens. All rights reserved."
from darwin import DarwinApi
if __name__ == "__main__":
darwin_api = DarwinApi(
socket_host="10.59.10.28",
socket_port=8006,
socket_type="tcp",
timeout=1,
)
darwin_api.call(
["google.com"],
filter_code="DGA",
response_type="back",
)
darwin_api.close()
darwin_api = DarwinApi(
socket_host="10.59.10.28",
socket_port=8007,
socket_type="tcp",
def send(self, data):
api = DarwinApi(socket_type="unix", socket_path=self.socket)
api.call([data], response_type='no')
def test_filter(filter_name,
socket_type=None,
socket_path=None,
socket_host=None,
socket_port=None,
filter_code=None,
response_type="back",
verbose=True,
debug_mode=False,
display_time=False,
**kwargs):
try:
call_args = kwargs.get("call_args", None)
bulk_call_args = kwargs.get("bulk_call_args", None)
expected_bulk_results = kwargs.get("expected_bulk_results", None)
# expected_call_result could REALLY be None, so we have to make the distinction
try:
expected_call_result = kwargs["expected_call_result"]
is_expected_call_result = True
except KeyError:
expected_call_result = None
is_expected_call_result = False
if debug_mode:
log(
"test_filter",
filter_name,
"tests will begin",
color=colors.OKBLUE,
)
# if not filter code is provided, we try to get it from the filter name
if filter_code is None:
if debug_mode:
log(
"test_filter",
filter_name,
"no filter code provided, so it will be extracted from the filter name \"{filter_name}\"",
color=colors.WARNING,
is_indent=True,
)
filter_code = filter_name
try:
if socket_type == "unix":
darwin_api = DarwinApi(
socket_path=socket_path,
socket_type=socket_type,
verbose=verbose,
)
elif socket_type == "tcp":
darwin_api = DarwinApi(
socket_host=socket_host,
socket_port=socket_port,
socket_type=socket_type,
verbose=verbose,
)
else:
if debug_mode:
log(
"test_filter",
filter_name,
"Invalid socket type provided: \"{socket_type}\"".
format(socket_type=socket_type),
color=colors.FAIL,
is_indent=True,
)
log(
"test_filter",
filter_name,
"[NOT OK]",
color=colors.FAIL,
is_indent=True,
)
return
except DarwinConnectionError:
if debug_mode:
log(
"test_filter",
filter_name,
"the filter does not appear to be running",
color=colors.WARNING,
is_indent=True,
)
log(
"test_filter",
filter_name,
"[NOT RUNNING]",
color=colors.WARNING,
)
return
is_ok = True
if call_args is not None:
if debug_mode:
log(
"test_filter",
filter_name,
"calling call function",
color=colors.HEADER,
is_indent=True,
)
if display_time:
start = time.time()
darwin_result = darwin_api.call(
call_args,
filter_code=filter_code,
response_type=response_type,
)
if display_time:
end = time.time()
log(
"test_filter",
filter_name,
"call function: took {duration} ms".format(
duration=(end - start) * 1000, ),
color=colors.OKBLUE,
)
if is_expected_call_result:
if expected_call_result != darwin_result:
if debug_mode:
log(
"test_filter",
filter_name,
"call function: expected result not matching Darwin: "
"{expected_result} != {darwin_result}".format(
expected_result=expected_call_result,
darwin_result=darwin_result,
),
color=colors.FAIL,
is_indent=True,
)
is_ok = False
else:
if debug_mode:
log(
"test_filter",
filter_name,
"call function: expected result matches Darwin: {expected_result}"
.format(expected_result=expected_call_result, ),
color=colors.OKGREEN,
is_indent=True,
)
else:
if debug_mode:
log(
"test_filter",
filter_name,
"call function: expected_call_result is not provided: "
"nothing to check. Darwin result obtained: {darwin_result}"
.format(darwin_result=darwin_result, ),
color=colors.OKGREEN,
is_indent=True,
)
if debug_mode:
log(
"test_filter",
filter_name,
"call function ended",
color=colors.OKBLUE,
is_indent=True,
)
else:
if debug_mode:
log(
"test_filter",
filter_name,
"call_args is None: nothing to do".format(
expected_result=expected_call_result, ),
color=colors.HEADER,
is_indent=True,
)
if bulk_call_args is not None:
if debug_mode:
log(
"test_filter",
filter_name,
"calling bulk_call function".format(
expected_result=expected_call_result, ),
color=colors.HEADER,
is_indent=True,
)
if display_time:
start = time.time()
darwin_result = darwin_api.bulk_call(
bulk_call_args,
filter_code=filter_code,
response_type=response_type,
)
if display_time:
end = time.time()
log(
"test_filter",
filter_name,
"bulk_call function: took {duration} ms".format(
duration=(end - start) * 1000, ),
color=colors.OKBLUE,
)
if isinstance(darwin_result, dict):
darwin_result = darwin_result["certitude_list"]
if expected_bulk_results is not None:
if expected_bulk_results != darwin_result:
if debug_mode:
log(
"test_filter",
filter_name,
"bulk_call function: "
"expected result not matching Darwin: {expected_result} != {darwin_result}"
.format(
expected_result=expected_bulk_results,
darwin_result=darwin_result,
),
color=colors.FAIL,
is_indent=True,
)
is_ok = False
else:
if debug_mode:
log(
"test_filter",
filter_name,
"bulk_call function: "
"expected result matches Darwin: {expected_result}"
.format(expected_result=expected_bulk_results, ),
color=colors.OKGREEN,
is_indent=True,
)
else:
if debug_mode:
log(
"test_filter",
filter_name,
"bulk_call function: "
"expected_bulk_results is None: nothing to check. "
"Darwin result obtained: {darwin_result}".format(
darwin_result=darwin_result, ),
color=colors.OKGREEN,
is_indent=True,
)
if debug_mode:
log(
"test_filter",
filter_name,
"bulk_call function ended",
color=colors.HEADER,
is_indent=True,
)
else:
if debug_mode:
log(
"test_filter",
filter_name,
"bulk_call_args is None: nothing to do",
color=colors.HEADER,
is_indent=True,
)
darwin_api.close()
if debug_mode:
log(
"test_filter",
filter_name,
"tests are finished",
color=colors.OKBLUE,
)
if is_ok:
log(
"test_filter",
filter_name,
"[OK]",
color=colors.OKGREEN,
)
else:
log(
"test_filter",
filter_name,
"[NOT OK]",
color=colors.FAIL,
)
except Exception as error:
darwin_api.close()
log(
"test_filter",
filter_name,
"[ERROR] {error}".format(error=error, ),
color=colors.FAIL,
)
if debug_mode:
raise
def alert_published_test(legacy=False):
ret = True
# CONFIG
tanomaly_filter = TAnomaly()
tanomaly_filter.configure(legacy)
# START FILTER
if not tanomaly_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=tanomaly_filter.socket,
socket_type="unix",
)
darwin_api.bulk_call(
tanomaly_filter.get_test_data(),
filter_code="TANOMALY",
response_type="back",
)
# Too hard to test with "time" field, so it's removed,
# but we check in alert received if this field is present
expected_alerts = [{
"filter": "tanomaly",
"anomaly": {
"ip": "213.211.198.58",
"udp_nb_host": 0.000000,
"udp_nb_port": 0.000000,
"tcp_nb_host": 126.000000,
"tcp_nb_port": 126.000000,
"icmp_nb_host": 0.000000,
"distance": 122.697636
}
}, {
"filter": "tanomaly",
"anomaly": {
"ip": "192.168.110.2",
"udp_nb_host": 252.000000,
"udp_nb_port": 252.000000,
"tcp_nb_host": 0.000000,
"tcp_nb_port": 0.000000,
"icmp_nb_host": 0.000000,
"distance": 349.590273
}
}]
try:
r = redis.Redis(unix_socket_path=tanomaly_filter.redis.unix_socket,
db=0)
pubsub = r.pubsub()
pubsub.subscribe([REDIS_ALERT_CHANNEL])
# We wait for the thread to activate
sleep(280)
alert_received = 0
timeout = 30 # in seconds
timeout_start = time()
# We stop waiting for the data fter 30 seconds
while (time() < timeout_start + timeout) or (alert_received < 2):
message = pubsub.get_message()
if message:
if message["type"] == "message":
alert = format_alert(message["data"].decode(),
"alert_published_test")
if alert not in expected_alerts:
ret = False
logging.error(
"alert_published_test: Not the expected alert received in redis. Got {}"
.format(alert))
alert_received += 1
sleep(0.001)
if alert_received != len(expected_alerts):
ret = False
logging.error(
"alert_published_test: Not the expected alerts number received on the channel"
)
r.close()
except Exception as e:
ret = False
logging.error(
"alert_published_test: Error when trying to redis subscribe: {}".
format(e))
# CLEAN
darwin_api.close()
tanomaly_filter.clean_files()
# ret = tanomaly_filter.valgrind_stop() or tanomaly_filter.valgrind_stop()
# would erase upper ret if this function return True
if not tanomaly_filter.valgrind_stop():
ret = False
return ret
def alert_in_file_test(legacy=False):
ret = True
# CONFIG
tanomaly_filter = TAnomaly()
tanomaly_filter.configure(legacy)
# START FILTER
if not tanomaly_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=tanomaly_filter.socket,
socket_type="unix",
)
darwin_api.bulk_call(
tanomaly_filter.get_test_data(),
filter_code="TANOMALY",
response_type="back",
)
# We wait for the thread to activate
sleep(302)
# Too hard to test with "time" field, so it's removed,
# but we check in alert received if this field is present
expected_alerts = [{
"filter": "tanomaly",
"anomaly": {
"ip": "213.211.198.58",
"udp_nb_host": 0.000000,
"udp_nb_port": 0.000000,
"tcp_nb_host": 126.000000,
"tcp_nb_port": 126.000000,
"icmp_nb_host": 0.000000,
"distance": 122.697636
}
}, {
"filter": "tanomaly",
"anomaly": {
"ip": "192.168.110.2",
"udp_nb_host": 252.000000,
"udp_nb_port": 252.000000,
"tcp_nb_host": 0.000000,
"tcp_nb_port": 0.000000,
"icmp_nb_host": 0.000000,
"distance": 349.590273
}
}]
redis_alerts = tanomaly_filter.get_file_alerts()
redis_alerts = [
format_alert(a, "alerts_in_file_test") for a in redis_alerts
]
if redis_alerts is None:
ret = False
logging.error("alert_in_file_test : No alerts writing in alert file")
if len(redis_alerts) != len(expected_alerts):
ret = False
logging.error(
"alerts_in_file_test : Not the expected data in file. Got : {}, expected : {}"
.format(redis_alerts, expected_alerts))
for a in redis_alerts:
if a not in expected_alerts:
ret = False
logging.error(
"alerts_in_file_test : Not the expected data in file. Got : {}, expected : {}"
.format(redis_alerts, expected_alerts))
# CLEAN
darwin_api.close()
tanomaly_filter.clean_files()
# ret = tanomaly_filter.valgrind_stop() or tanomaly_filter.valgrind_stop()
# would erase upper ret if this function return True
if not tanomaly_filter.valgrind_stop():
ret = False
return ret
def input_with_several_rules(data, expected_certitudes=[0]):
ret = True
yara_filter = Yara()
yara_filter.generate_rule_file("""rule niwraD
{
strings:
$s1 = { 6e 69 77 72 61 44 }
condition:
all of them
}
rule Darwin
{
strings:
$s1 = { 44 61 72 77 69 6e }
condition:
all of them
}""")
yara_filter.configure()
if not yara_filter.valgrind_start():
logging.error("one_valid_rule_file: filter didn't start")
return False
# SEND TEST
darwin_api = DarwinApi(socket_path=yara_filter.socket,
socket_type="unix")
results = darwin_api.bulk_call(
data,
response_type="back",
)
# VERIFY RESULTS
certitudes = results.get('certitude_list')
if certitudes is None:
ret = False
logging.error("yara_scan Test : No certitude list found in result")
if len(certitudes) != len(expected_certitudes):
ret = False
logging.error("yara_scan Test : Unexpected certitude size of {} instead of {}"
.format(len(certitudes), len(expected_certitudes)))
if certitudes != expected_certitudes:
ret = False
logging.error("yara_scan Test : Unexpected certitude of {} instead of {}"
.format(certitudes, expected_certitudes))
# CLEAN
darwin_api.close()
if not yara_filter.valgrind_stop():
logging.error("one_valid_rule_file: filter didn't stop properly")
return False
return ret
def fanomaly_connector_and_send_test():
test_name = "fanomaly_connector_and_send_test"
ret = True
config_test = '{{' \
'"redis_socket_path": "{redis_socket}",' \
'"alert_redis_list_name": "{redis_alert}",' \
'"log_file_path": "/var/log/darwin/alerts.log",' \
'"alert_redis_channel_name": "darwin.alerts"' \
'}}'.format(redis_socket=REDIS_SOCKET, redis_alert=REDIS_ALERT_LIST)
config_buffer = '{{' \
'"redis_socket_path": "{redis_socket}",' \
'"input_format": [' \
'{{"name": "net_src_ip", "type": "string"}},' \
'{{"name": "net_dst_ip", "type": "string"}},' \
'{{"name": "net_dst_port", "type": "string"}},' \
'{{"name": "ip_proto", "type": "string"}}' \
'],' \
'"outputs": [' \
'{{' \
'"filter_type": "fanomaly",' \
'"filter_socket_path": "/tmp/anomaly.sock",' \
'"interval": 10,' \
'"required_log_lines": 5,' \
'"redis_lists": [{{' \
'"source": "",' \
'"name": "darwin_buffer_test"' \
'}}]' \
'}}' \
']' \
'}}'.format(redis_socket=REDIS_SOCKET)
# CONFIG
buffer_filter = Buffer()
buffer_filter.configure(config_buffer)
test_filter = Filter(filter_name="anomaly",
socket_path="/tmp/anomaly.sock")
test_filter.configure(config_test)
# START FILTER
if not buffer_filter.valgrind_start():
print("Buffer did not start")
return False
if not test_filter.start():
print("Anomaly did not start")
return False
# SEND TEST
data = buffer_filter.get_test_data()
darwin_api = DarwinApi(socket_path=buffer_filter.socket,
socket_type="unix")
darwin_api.bulk_call(
data,
response_type="back",
)
sleep(15)
# GET REDIS DATA AND COMPARE
redis_data = buffer_filter.get_internal_redis_list_data(REDIS_ALERT_LIST)
expected_data = '"details": {"ip": "192.168.110.2","udp_nb_host": 1.000000,"udp_nb_port": 252.000000,"tcp_nb_host": 0.000000,"tcp_nb_port": 0.000000,"icmp_nb_host": 0.000000,"distance": 246.193959}'
if len(redis_data) != 1:
logging.error("{}: Expecting a single element list.".format(test_name))
ret = False
redis_data = [a.decode() for a in redis_data]
if (expected_data not in redis_data[0]):
logging.error("{}: Expected this data : {} but got {} in redis".format(
test_name, expected_data, redis_data))
ret = False
# CLEAN
darwin_api.close()
test_filter.clean_files()
# ret = buffer_filter.valgrind_stop() or buffer_filter.valgrind_stop()
# would erase upper ret if this function return True
if not buffer_filter.valgrind_stop():
ret = False
if not test_filter.stop():
ret = False
return ret
def thread_working_test():
ret = True
config_buffer = '{{' \
'"redis_socket_path": "{redis_socket}",' \
'"input_format": [' \
'{{"name": "net_src_ip", "type": "string"}},' \
'{{"name": "net_dst_ip", "type": "string"}},' \
'{{"name": "net_dst_port", "type": "string"}},' \
'{{"name": "ip_proto", "type": "string"}}' \
'],' \
'"outputs": [' \
'{{' \
'"filter_type": "fanomaly",' \
'"filter_socket_path": "/tmp/anomaly.sock",' \
'"interval": 10,' \
'"required_log_lines": 5,' \
'"redis_lists": [{{' \
'"source": "",' \
'"name": "darwin_buffer_anomaly"' \
'}}]' \
'}}' \
']' \
'}}'.format(redis_socket=REDIS_SOCKET)
config_test = '{{' \
'"redis_socket_path": "{redis_socket}",' \
'"alert_redis_list_name": "{redis_alert}",' \
'"log_file_path": "/var/log/darwin/alerts.log",' \
'"alert_redis_channel_name": "darwin.alerts"' \
'}}'.format(redis_socket=REDIS_SOCKET, redis_alert=REDIS_ALERT_LIST)
# CONFIG
buffer_filter = Buffer()
buffer_filter.configure(config_buffer)
test_filter = Filter(filter_name="anomaly",
socket_path="/tmp/anomaly.sock")
test_filter.configure(config_test)
# START FILTER
if not buffer_filter.valgrind_start():
return False
if not test_filter.start():
print("Anomaly did not start")
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=buffer_filter.socket,
socket_type="unix",
)
data = buffer_filter.get_test_data()
darwin_api.bulk_call(
data,
response_type="back",
)
# We wait for the thread to activate
sleep(15)
redis_data = buffer_filter.get_internal_redis_set_data(
"darwin_buffer_anomaly")
if redis_data != set():
logging.error(
"thread_working_test : Expected no data in Redis but got {}".
format(redis_data))
ret = False
# CLEAN
darwin_api.close()
# ret = buffer_filter.valgrind_stop() or buffer_filter.valgrind_stop()
# would erase upper ret if this function return True
if not buffer_filter.valgrind_stop():
ret = False
return ret
def test(data, expected_data, test_name, expect_result=True):
ret = True
# CONFIG
sofa_filter = Sofa()
sofa_filter.configure()
# START FILTER
if not sofa_filter.start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=sofa_filter.socket,
socket_type="unix",
)
results = darwin_api.bulk_call(
data,
filter_code="SOFA",
response_type="back",
)
# VERIFY RESULTS
results = results.get('body')
if results is '' and expect_result:
logging.error(
"Sofa test : {} : No body found in result".format(test_name))
darwin_api.close()
sofa_filter.clean_files()
sofa_filter.stop()
return False
if expect_result:
results = results.splitlines()
header = results[0]
body = results[1:]
else:
header = EXPECTED_HEADER
body = []
if header != EXPECTED_HEADER:
ret = False
logging.error(
"Sofa test: {} : Unexpected header -> {}, expected {}".format(
test_name, header, EXPECTED_HEADER))
expected_body_size = len(expected_data)
if len(body) != expected_body_size:
ret = False
logging.error(
"Sofa test : {} : Unexpected body size of {} instead of {}".format(
test_name, len(body), expected_body_size))
if body != expected_data:
ret = False
logging.error(
"Sofa test : {} : Unexpected body {} instead of {}".format(
test_name, body, expected_data))
# CLEAN
darwin_api.close()
# ret = hostlookup_filter.valgrind_stop() or hostlookup_filter.valgrind_stop()
# would erase upper ret if this function return True
if not sofa_filter.stop():
ret = False
for root, dirs, files in os.walk("/var/tmp"):
for file in files:
if file.endswith(".csv") or file.endswith(".json"):
logging.error("Sofa test: {} : remaining file {}".format(
test_name, file))
ret = False
sofa_filter.clean_files()
return ret
def alert_published_test():
ret = True
# CONFIG
tanomaly_filter = TAnomaly()
tanomaly_filter.configure()
# START FILTER
if not tanomaly_filter.valgrind_start():
return False
# SEND TEST
darwin_api = DarwinApi(
socket_path=tanomaly_filter.socket,
socket_type="unix",
)
darwin_api.bulk_call(
tanomaly_filter.get_test_data(),
filter_code="TANOMALY",
response_type="back",
)
try:
r = redis.Redis(unix_socket_path=tanomaly_filter.redis.unix_socket,
db=0)
pubsub = r.pubsub()
pubsub.subscribe([REDIS_ALERT_CHANNEL])
alert_received = 0
timeout = 30 # in seconds
timeout_start = time()
# We stop waiting for the data after 10 seconds
while (time() < timeout_start + timeout) and (alert_received < 3):
message = pubsub.get_message()
if message:
if message["type"] == "message":
alert = format_alert(message["data"].decode(),
"alert_published_test")
if alert not in EXPECTED_ALERTS:
ret = False
logging.error(
"alert_published_test: Not the expected alert received in redis. Got {}"
.format(alert))
logging.error(alert)
alert_received += 1
sleep(0.001)
if alert_received != len(EXPECTED_ALERTS):
ret = False
logging.error(
"alert_published_test: Not the expected alerts number received on the channel"
)
logging.error(
"alert_published_test: got {} alerts".format(alert_received))
r.close()
except Exception as e:
ret = False
logging.error(
"alert_published_test: Error when trying to redis subscribe: {}".
format(e))
# CLEAN
darwin_api.close()
tanomaly_filter.clean_files()
# ret = tanomaly_filter.valgrind_stop() or tanomaly_filter.valgrind_stop()
# would erase upper ret if this function return True
if not tanomaly_filter.valgrind_stop():
ret = False
return ret