def test_terms_small_batch_treat_all(self):
dummy_doc_generate = DummyDocumentsGenerate()
# Init the list of user
nbr_doc_per_hours = 5
nbr_hours = 10
nbr_doc_generated_per_hours = [
nbr_doc_per_hours for _ in range(nbr_hours)
]
# If the number of document per hours is not a divisor of the batch limit, all document will not be detected
# Generate documents
self.test_es.add_multiple_docs(
dummy_doc_generate.generate_doc_time_variable_sensitivity(
nbr_doc_generated_per_hours))
# Run analyzer
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/terms_test_small_batch_eval.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_float.conf"
)
analyzer.evaluate_model()
self.assertEqual(analyzer.total_outliers,
nbr_doc_per_hours * nbr_hours)
def test_sudden_extra_outlier_infos_all_present(self):
# Generate documents
dummy_doc_generate = DummyDocumentsGenerate()
list_delta_hour = [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 15]
field_1_name = "user_id"
list_field_1_value = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
field_2_name = "hostname"
list_field_2_value = []
for _ in range(len(list_delta_hour)):
list_field_2_value.append("host1")
generated_docs = dummy_doc_generate.generate_doc_time_variable_witt_custom_fields(
list_delta_hour, field_1_name, list_field_1_value, field_2_name,
list_field_2_value)
self.test_es.add_multiple_docs(generated_docs)
self.test_settings.change_configuration_path(test_conf_file_01)
analyzer = AnalyzerFactory.create(
root_test_use_case_files + "sudden_appearance_dummy_test_03.conf")
set_new_current_date(analyzer)
analyzer.evaluate_model()
list_outlier = list()
for elem in es._scan():
if "outliers" in elem["_source"]:
list_outlier.append(elem)
all_fields_exists = [
elem in EXTRA_OUTLIERS_KEY_FIELDS + DEFAULT_OUTLIERS_KEY_FIELDS
for elem in list_outlier[0]['_source']['outliers']
]
self.assertTrue(all(all_fields_exists))
def test_whitelist_literals_per_model_not_removed_by_housekeeping(self):
# Init
doc_generate = DummyDocumentsGenerate()
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/housekeeping.conf")
housekeeping = HousekeepingJob()
# Generate document
document = doc_generate.generate_document({
"hostname":
"NOT-WHITELISTED",
"create_outlier":
True,
"outlier.model_name":
"dummy_test",
"outlier.model_type":
"simplequery"
})
self.assertTrue("outliers" in document["_source"])
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/housekeeping/analyzer_dummy_test_with_whitelist.conf"
)
housekeeping.update_analyzer_list([analyzer])
self.test_es.add_doc(document)
housekeeping.execute_housekeeping()
result = [elem for elem in self.test_es._scan()][0]
self.assertTrue("outliers" in result["_source"])
def test_non_outliers_present_in_metrics(self):
dummy_doc_generate = DummyDocumentsGenerate()
# Generate documents
# Outlier document
self.test_es.add_doc(
dummy_doc_generate.generate_document({"user_id": 11}))
# Non outlier document
self.test_es.add_doc(
dummy_doc_generate.generate_document({"user_id": 8}))
# Outlier document
self.test_es.add_doc(
dummy_doc_generate.generate_document({"user_id": 12}))
# Run analyzer
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/metrics_test_02.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/metrics/metrics_dummy_test_not_derived.conf"
)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][2]
self.assertEqual(
result["_source"]["outliers"]["non_outlier_values_sample"],
["8.0"])
def test_metrics_whitelist_work_test_es_result(self):
dummy_doc_generate = DummyDocumentsGenerate()
command_query = "SELECT * FROM dummy_table" # must be bigger than the trigger value (here 3)
nbr_generated_documents = 5
# Generate document that match outlier
for _ in range(nbr_generated_documents):
self.test_es.add_doc(
dummy_doc_generate.generate_document(
{"command_query": command_query}))
# Generate whitelist document
self.test_es.add_doc(
dummy_doc_generate.generate_document({
"hostname": "whitelist_hostname",
"command_query": command_query
}))
# Run analyzer
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/metrics_test_01.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/metrics/metrics_length_dummy_test.conf"
)
analyzer.evaluate_model()
nbr_outliers = 0
for elem in es._scan():
if "outliers" in elem["_source"]:
nbr_outliers += 1
self.assertEqual(nbr_outliers, nbr_generated_documents)
def test_notification_on_outlier_already_detected_but_not_in_queue(self):
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/notifications_test.conf")
self.test_notifier = TestStubNotifier()
doc_generate = DummyDocumentsGenerate()
# Create outliers
doc = doc_generate.generate_document()
# Full the queue (3 elements)
outlier1 = Outlier("dummy type", "dummy reason", "dummy summary1", doc)
es.notifier.notify_on_outlier(outlier1)
outlier2 = Outlier("dummy type2", "dummy reason2", "dummy summary2",
doc)
es.notifier.notify_on_outlier(outlier2)
outlier3 = Outlier("dummy type3", "dummy reason3", "dummy summary3",
doc)
es.notifier.notify_on_outlier(outlier3)
# Add a new one that will remove the first
outlier4 = Outlier("dummy type4", "dummy reason4", "dummy summary4",
doc)
es.notifier.notify_on_outlier(outlier4)
# Add again the first one
es.notifier.notify_on_outlier(outlier1)
# All outliers notify need to be present (so 5)
self.assertEqual(len(self.test_notifier.get_list_email()), 5)
self.test_notifier.restore_notifier()
def test_sudden_appearance_detect_no_outlier_es_check(self):
# Generate documents
dummy_doc_generate = DummyDocumentsGenerate()
list_delta_hour = [1, 1, 1, 3, 3, 3, 4, 5, 5, 5, 15, 15]
field_1_name = "user_id"
list_field_1_value = [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2]
field_2_name = "hostname"
list_field_2_value = []
for _ in range(len(list_delta_hour)):
list_field_2_value.append("host1")
generated_docs = dummy_doc_generate.generate_doc_time_variable_witt_custom_fields(
list_delta_hour, field_1_name, list_field_1_value, field_2_name,
list_field_2_value)
self.test_es.add_multiple_docs(generated_docs)
# Run analyzer
self.test_settings.change_configuration_path(test_conf_file_01)
analyzer = AnalyzerFactory.create(
root_test_use_case_files + "sudden_appearance_dummy_test_02.conf")
set_new_current_date(analyzer)
analyzer.evaluate_model()
nbr_outliers = 0
for elem in es._scan():
if "outliers" in elem["_source"]:
nbr_outliers += 1
self.assertEqual(nbr_outliers, 0)
def test_remove_all_whitelisted_outliers(self):
self.test_settings.change_configuration_path(
test_file_whitelist_path_config)
doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(
doc_generate.generate_document({
"create_outlier":
True,
"outlier_observation":
"dummy observation",
"outlier.model_name":
"dummy_test",
"outlier.model_type":
"analyzer",
"command_query":
"osquery_get_all_processes_with_listening_conns.log"
}))
# Check that outlier correctly generated
result = [doc for doc in es._scan()][0]
self.assertTrue("outliers" in result["_source"])
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/analyzer/analyzer_dummy_test.conf"
)
# Remove whitelisted outlier
es.remove_all_whitelisted_outliers({"analyzer_dummy_test": analyzer})
# Check that outlier is correctly remove
result = [doc for doc in es._scan()][0]
self.assertFalse("outliers" in result["_source"])
def test_add_document_to_batch_empty_aggergator(self):
dummy_doc_generate = DummyDocumentsGenerate()
dummy_doc = dummy_doc_generate.generate_document()
current_batch = {"dummy_key": "dummy_value"}
result = TermsAnalyzer._add_document_to_batch(current_batch,
["dummy_target"], list(),
dummy_doc)
self.assertEqual(result, current_batch)
def test_min_target_buckets_dont_detect_outlier(self):
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/terms_test_whitelist_batch.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_batch_whitelist_within_float.conf"
)
# Recap:
# min_target_buckets=4
# trigger_sensitivity=5
# trigger_on=high
# trigger_method=float
# Dont encode with a matrix to keep order of document
doc_to_generate = [
# New batch:
# 0 1
# agg1 [6, 1]
# agg2 [1, 2]
("agg1", 0),
("agg2", 0),
("agg1", 0),
("agg1", 0),
("agg1", 0),
("agg1", 0),
("agg1", 0),
("agg2", 1),
("agg2", 1),
("agg1", 1),
# New Batch
# 2
# agg1 [0]
# agg2 [1]
("agg1", 2)
]
# At the end:
# 0 1 2
# agg1 [6, 1, 1]
# agg2 [1, 2]
# Normally agg1 - 0 must be flagged, but here they doesn't have enough buckets values
dummy_doc_gen = DummyDocumentsGenerate()
for aggregator, target_value in doc_to_generate:
user_id = target_value
hostname = aggregator
doc_generated = dummy_doc_gen.generate_document({
"user_id":
user_id,
"hostname":
hostname
})
self.test_es.add_doc(doc_generated)
analyzer.evaluate_model()
self.assertEqual(analyzer.total_outliers, 0)
def test_whitelist_literal_match(self):
self.test_settings.change_configuration_path(
test_file_outliers_path_config)
# Contain: "C:\Windows\system32\msfeedssync.exe sync"
dummy_doc_gen = DummyDocumentsGenerate()
doc = dummy_doc_gen.generate_document(
{"command_query": r'C:\Windows\system32\msfeedssync.exe sync'})
result = Outlier.is_whitelisted_doc(doc)
self.assertTrue(result)
def test_simplequery_not_use_derived_fields_in_doc(self):
dummy_doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(dummy_doc_generate.generate_document())
self.test_settings.change_configuration_path(
config_file_simplequery_test_02)
analyzer = AnalyzerFactory.create(
use_case_simplequery_dummy_test_not_derived)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertFalse("timestamp_year" in result['_source'])
def test_simplequry_use_matched_values_in_outlier(self):
dummy_doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(dummy_doc_generate.generate_document())
self.test_settings.change_configuration_path(
config_file_simplequery_test_02)
analyzer = AnalyzerFactory.create(
use_case_simplequery_dummy_test_highlight_match_activated)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertTrue("matched_values" in result['_source']['outliers'])
def test_terms_not_use_derived_fields_in_doc(self):
dummy_doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(dummy_doc_generate.generate_document())
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/terms_test_01.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_not_derived.conf"
)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertFalse("timestamp_year" in result['_source'])
def test_sudden_appearance_derived_fields_in_doc(self):
dummy_doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(dummy_doc_generate.generate_document())
self.test_settings.change_configuration_path(test_conf_file_01)
analyzer = AnalyzerFactory.create(
root_test_use_case_files +
"sudden_appearance_derived_fields_01.conf")
set_new_current_date(analyzer)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertTrue("timestamp_year" in result['_source'])
def test_simplequery_not_use_derived_fields_but_present_in_outlier(self):
dummy_doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(dummy_doc_generate.generate_document())
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/simplequery_test_02.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/simplequery/simplequery_dummy_test_not_derived.conf"
)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertTrue(
"derived_timestamp_year" in result['_source']['outliers'])
def test_whitelist_correctly_reload_after_update_config(self):
self.test_settings.change_configuration_path(test_whitelist_single_literal_file)
dummy_doc_gen = DummyDocumentsGenerate()
doc = dummy_doc_gen.generate_document({"create_outlier": True, "outlier_observation": "dummy observation",
"filename": "osquery_get_all_processes_with_listening_conns.log"})
# With this configuration, outlier is not whitlisted
self.assertFalse(Outlier.is_whitelisted_doc(doc))
# Update configuration
self.test_settings.change_configuration_path(test_whitelist_multiple_literal_file)
# Now outlier is whitelisted
self.assertTrue(Outlier.is_whitelisted_doc(doc))
def test_batch_whitelist_work_doent_match_outlier_in_across(self):
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/terms_test_whitelist_batch.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_batch_whitelist_across_float.conf"
)
doc_to_generate = [
# agg1 (0, 1, 2) -> 3 but with whitelist: (0, 2) -> 2
# agg2 (0, 3, 4) -> 3
("agg1", 0, False),
("agg1", 1, True),
("agg2", 0, False),
("agg2", 0, False),
("agg1", 2, False),
("agg2", 3, False),
("agg2", 4, False)
]
dummy_doc_gen = DummyDocumentsGenerate()
for aggregator, target_value, is_whitelist in doc_to_generate:
deployment_name = None
if is_whitelist:
deployment_name = "whitelist-deployment"
user_id = target_value
hostname = aggregator
doc_generated = dummy_doc_gen.generate_document({
"deployment_name":
deployment_name,
"user_id":
user_id,
"hostname":
hostname
})
self.test_es.add_doc(doc_generated)
analyzer.evaluate_model()
list_outliers = []
for doc in es._scan():
if "outliers" in doc["_source"]:
list_outliers.append(
(doc["_source"]["outliers"]["aggregator"][0],
doc["_source"]["outliers"]["term"][0]))
# We detect agg2 but not agg1
self.assertEqual(list_outliers, [("agg2", "0"), ("agg2", "0"),
("agg2", "3"), ("agg2", "4")])
def test_bulk_update_do_not_remove_values(self):
dummy_doc_gen = DummyDocumentsGenerate()
doc = dummy_doc_gen.generate_document({"create_outlier": True})
self.test_es.add_doc(doc)
test_doc = copy.deepcopy(doc)
# Remove outlier
test_doc["_source"].pop("outliers")
# Update the document (without outliers)
es.add_update_bulk_action(test_doc)
# Result in ES is the same that the original document (outliers wasn't removed)
result = [elem for elem in es._scan()][0]
self.assertEqual(doc, result)
def test_arbitrary_key_config_present_in_outlier(self):
self.test_settings.change_configuration_path(
config_file_simplequery_test_01)
analyzer = AnalyzerFactory.create(
use_case_simplequery_arbitrary_dummy_test)
dummy_doc_generate = DummyDocumentsGenerate()
# Generate document
self.test_es.add_doc(dummy_doc_generate.generate_document())
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertEquals(result["_source"]["outliers"]["test_arbitrary_key"],
["arbitrary_value"])
def test_arbitrary_key_config_not_present_int_other_model(self):
# Dictionary and list could be share between different instance. This test check that a residual value is not
# present in the dictionary
self.test_settings.change_configuration_path(
config_file_simplequery_test_01)
analyzer = AnalyzerFactory.create(use_case_simplequery_dummy_test)
dummy_doc_generate = DummyDocumentsGenerate()
# Generate document
self.test_es.add_doc(dummy_doc_generate.generate_document())
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertFalse("test_arbitrary_key" in result["_source"]["outliers"])
def test_terms_not_use_derived_fields_but_present_in_outlier(self):
dummy_doc_generate = DummyDocumentsGenerate()
self.test_es.add_doc(
dummy_doc_generate.generate_document({"user_id": 11}))
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/terms_test_01.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_not_derived.conf"
)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
# The parameter use_derived_fields haven't any impact on outliers keys
self.assertTrue(
"derived_timestamp_year" in result['_source']['outliers'])
def test_whitelist_regex_per_model_match_whitelist(self):
doc_generate = DummyDocumentsGenerate()
# Generate document
self.test_es.add_doc(
doc_generate.generate_document({"hostname": "AAA-WHITELISTED"}))
# Run analyzer
self.test_settings.change_configuration_path(
config_file_simplequery_test_01)
analyzer = AnalyzerFactory.create(
use_case_whitelist_tests_model_whitelist_02)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertFalse("outliers" in result["_source"])
def test_email_dict_key(self):
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/notifications_test.conf")
self.test_notifier = TestStubNotifier()
doc_generate = DummyDocumentsGenerate()
# Create outlier
doc = doc_generate.generate_document()
outlier = Outlier("dummy type", "dummy reason", "dummy summary", doc)
# execute notification
es.notifier.notify_on_outlier(outlier)
email_dict = self.test_notifier.get_list_email()[0]
self.assertEqual(list(email_dict.keys()), ["subject", "body"])
self.test_notifier.restore_notifier()
def test_notification_on_outlier_match_metrics(self):
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/notifications_test.conf")
self.test_notifier = TestStubNotifier()
doc_generate = DummyDocumentsGenerate()
# Create document that's an outlier
doc = doc_generate.generate_document({"user_id": 11})
self.test_es.add_doc(doc)
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/notifications/metrics_numerical_value_dummy_test.conf"
)
analyzer.evaluate_model()
self.assertEqual(len(self.test_notifier.get_list_email()), 1)
self.test_notifier.restore_notifier()
def test_whitelist_regex_per_model_not_match_whitelist(self):
doc_generate = DummyDocumentsGenerate()
# Generate document
self.test_es.add_doc(
doc_generate.generate_document(
{"hostname": "Not-work-WHITELISTED"}))
# Run analyzer
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/simplequery_test_01.conf")
analyzer = AnalyzerFactory.create(
"/app/tests/unit_tests/files/use_cases/simplequery/whitelist_tests_model_whitelist_02.conf"
)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
self.assertTrue("outliers" in result["_source"])
def test_simplequery_no_extra_outlier_infos(self):
dummy_doc_generate = DummyDocumentsGenerate()
# Generate document
self.test_es.add_doc(dummy_doc_generate.generate_document())
# Run analyzer
self.test_settings.change_configuration_path(
config_file_simplequery_test_01)
analyzer = AnalyzerFactory.create(use_case_simplequery_dummy_test)
analyzer.evaluate_model()
result = [elem for elem in es._scan()][0]
all_fields_exists = [
elem in DEFAULT_OUTLIERS_KEY_FIELDS
for elem in result['_source']['outliers']
]
self.assertTrue(all(all_fields_exists))
def _generate_metrics_doc_with_whitelist(self, doc_to_generate):
# Use list of tuple (and not dict) to keep order
dummy_doc_gen = DummyDocumentsGenerate()
for aggregator, target_value, is_whitelist in doc_to_generate:
deployment_name = None
if is_whitelist:
deployment_name = "whitelist-deployment"
user_id = target_value
hostname = aggregator
doc_generated = dummy_doc_gen.generate_document({
"deployment_name":
deployment_name,
"user_id":
user_id,
"hostname":
hostname
})
self.test_es.add_doc(doc_generated)
def test_add_document_to_batch_one_aggregator_and_one_target(self):
dummy_doc_generate = DummyDocumentsGenerate()
dummy_doc = dummy_doc_generate.generate_document()
target_value = "dummy_target"
aggregator_value = "dummy_aggregator"
current_batch = {"dummy_key": "dummy_value"}
result = TermsAnalyzer._add_document_to_batch(current_batch,
[target_value],
[aggregator_value],
dummy_doc)
expected_batch = current_batch.copy()
expected_batch[aggregator_value] = defaultdict(list)
expected_batch[aggregator_value]["targets"].append(target_value)
expected_batch[aggregator_value]["observations"].append(dict())
expected_batch[aggregator_value]["raw_docs"].append(dummy_doc)
self.assertEqual(result, expected_batch)
def test_notify_on_outlier_correctly_create_email(self):
import logging, sys
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
self.test_settings.change_configuration_path(
"/app/tests/unit_tests/files/notifications_test.conf")
self.test_notifier = TestStubNotifier()
doc_generate = DummyDocumentsGenerate()
# Create outlier
doc = doc_generate.generate_document()
outlier = Outlier("dummy type", "dummy reason", "dummy summary", doc)
# execute notification
es.notifier.notify_on_outlier(outlier)
self.assertEqual(len(self.test_notifier.get_list_email()), 1)
self.test_notifier.restore_notifier()
