def test_non_outliers_present_in_metrics(self): dummy_doc_generate = DummyDocumentsGenerate() # Generate documents # Outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 11})) # Non outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 8})) # Outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 12})) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/metrics_test_02.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/metrics/metrics_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][2] self.assertEqual( result["_source"]["outliers"]["non_outlier_values_sample"], ["8.0"])
def test_metrics_whitelist_work_test_es_result(self): dummy_doc_generate = DummyDocumentsGenerate() command_query = "SELECT * FROM dummy_table" # must be bigger than the trigger value (here 3) nbr_generated_documents = 5 # Generate document that match outlier for _ in range(nbr_generated_documents): self.test_es.add_doc( dummy_doc_generate.generate_document( {"command_query": command_query})) # Generate whitelist document self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "whitelist_hostname", "command_query": command_query })) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/metrics_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/metrics/metrics_length_dummy_test.conf" ) analyzer.evaluate_model() nbr_outliers = 0 for elem in es._scan(): if "outliers" in elem["_source"]: nbr_outliers += 1 self.assertEqual(nbr_outliers, nbr_generated_documents)
def test_non_outlier_values_not_present_in_terms_within(self): dummy_doc_generate = DummyDocumentsGenerate() # Generate documents # Outlier document # index: 0 # Non outlier self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "one", "deployment_name": "one" })) # index: 1 self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "one", "deployment_name": "two" })) # index: 2 self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "one", "deployment_name": "two" })) # index: 3 # Outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "one", "deployment_name": "two" })) # index: 4 self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "one", "deployment_name": "three" })) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_float_low.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][4] self.assertEqual( result["_source"]["outliers"]["non_outlier_values_sample"], ["two"])
def test_whitelist_literals_per_model_not_removed_by_housekeeping(self): # Init doc_generate = DummyDocumentsGenerate() self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/housekeeping.conf") housekeeping = HousekeepingJob() # Generate document document = doc_generate.generate_document({ "hostname": "NOT-WHITELISTED", "create_outlier": True, "outlier.model_name": "dummy_test", "outlier.model_type": "simplequery" }) self.assertTrue("outliers" in document["_source"]) analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/housekeeping/analyzer_dummy_test_with_whitelist.conf" ) housekeeping.update_analyzer_list([analyzer]) self.test_es.add_doc(document) housekeeping.execute_housekeeping() result = [elem for elem in self.test_es._scan()][0] self.assertTrue("outliers" in result["_source"])
def test_notification_on_outlier_already_detected_but_not_in_queue(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outliers doc = doc_generate.generate_document() # Full the queue (3 elements) outlier1 = Outlier("dummy type", "dummy reason", "dummy summary1", doc) es.notifier.notify_on_outlier(outlier1) outlier2 = Outlier("dummy type2", "dummy reason2", "dummy summary2", doc) es.notifier.notify_on_outlier(outlier2) outlier3 = Outlier("dummy type3", "dummy reason3", "dummy summary3", doc) es.notifier.notify_on_outlier(outlier3) # Add a new one that will remove the first outlier4 = Outlier("dummy type4", "dummy reason4", "dummy summary4", doc) es.notifier.notify_on_outlier(outlier4) # Add again the first one es.notifier.notify_on_outlier(outlier1) # All outliers notify need to be present (so 5) self.assertEqual(len(self.test_notifier.get_list_email()), 5) self.test_notifier.restore_notifier()
def test_remove_all_whitelisted_outliers(self): self.test_settings.change_configuration_path( test_file_whitelist_path_config) doc_generate = DummyDocumentsGenerate() self.test_es.add_doc( doc_generate.generate_document({ "create_outlier": True, "outlier_observation": "dummy observation", "outlier.model_name": "dummy_test", "outlier.model_type": "analyzer", "command_query": "osquery_get_all_processes_with_listening_conns.log" })) # Check that outlier correctly generated result = [doc for doc in es._scan()][0] self.assertTrue("outliers" in result["_source"]) analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/analyzer/analyzer_dummy_test.conf" ) # Remove whitelisted outlier es.remove_all_whitelisted_outliers({"analyzer_dummy_test": analyzer}) # Check that outlier is correctly remove result = [doc for doc in es._scan()][0] self.assertFalse("outliers" in result["_source"])
def test_add_document_to_batch_empty_aggergator(self): dummy_doc_generate = DummyDocumentsGenerate() dummy_doc = dummy_doc_generate.generate_document() current_batch = {"dummy_key": "dummy_value"} result = TermsAnalyzer._add_document_to_batch(current_batch, ["dummy_target"], list(), dummy_doc) self.assertEqual(result, current_batch)
def test_min_target_buckets_dont_detect_outlier(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_whitelist_batch.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_batch_whitelist_within_float.conf" ) # Recap: # min_target_buckets=4 # trigger_sensitivity=5 # trigger_on=high # trigger_method=float # Dont encode with a matrix to keep order of document doc_to_generate = [ # New batch: # 0 1 # agg1 [6, 1] # agg2 [1, 2] ("agg1", 0), ("agg2", 0), ("agg1", 0), ("agg1", 0), ("agg1", 0), ("agg1", 0), ("agg1", 0), ("agg2", 1), ("agg2", 1), ("agg1", 1), # New Batch # 2 # agg1 [0] # agg2 [1] ("agg1", 2) ] # At the end: # 0 1 2 # agg1 [6, 1, 1] # agg2 [1, 2] # Normally agg1 - 0 must be flagged, but here they doesn't have enough buckets values dummy_doc_gen = DummyDocumentsGenerate() for aggregator, target_value in doc_to_generate: user_id = target_value hostname = aggregator doc_generated = dummy_doc_gen.generate_document({ "user_id": user_id, "hostname": hostname }) self.test_es.add_doc(doc_generated) analyzer.evaluate_model() self.assertEqual(analyzer.total_outliers, 0)
def test_whitelist_literal_match(self): self.test_settings.change_configuration_path( test_file_outliers_path_config) # Contain: "C:\Windows\system32\msfeedssync.exe sync" dummy_doc_gen = DummyDocumentsGenerate() doc = dummy_doc_gen.generate_document( {"command_query": r'C:\Windows\system32\msfeedssync.exe sync'}) result = Outlier.is_whitelisted_doc(doc) self.assertTrue(result)
def test_notification_on_two_different_outliers(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outliers doc1 = doc_generate.generate_document() outlier1 = Outlier("dummy type", "dummy reason", "dummy summary", doc1) doc2 = doc_generate.generate_document() outlier2 = Outlier("dummy type2", "dummy reason2", "dummy summary2", doc2) # execute notification es.notifier.notify_on_outlier(outlier1) es.notifier.notify_on_outlier(outlier2) self.assertEqual(len(self.test_notifier.get_list_email()), 2) self.test_notifier.restore_notifier()
def test_simplequery_not_use_derived_fields_in_doc(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( config_file_simplequery_test_02) analyzer = AnalyzerFactory.create( use_case_simplequery_dummy_test_not_derived) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("timestamp_year" in result['_source'])
def test_simplequry_use_matched_values_in_outlier(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( config_file_simplequery_test_02) analyzer = AnalyzerFactory.create( use_case_simplequery_dummy_test_highlight_match_activated) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue("matched_values" in result['_source']['outliers'])
def test_terms_not_use_derived_fields_in_doc(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("timestamp_year" in result['_source'])
def test_sudden_appearance_derived_fields_in_doc(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path(test_conf_file_01) analyzer = AnalyzerFactory.create( root_test_use_case_files + "sudden_appearance_derived_fields_01.conf") set_new_current_date(analyzer) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue("timestamp_year" in result['_source'])
def test_sudden_appearance_whitelist_work_test_es_result(self): dummy_doc_generate = DummyDocumentsGenerate() command_query = "SELECT * FROM dummy_table" # must be bigger than the trigger value (here 3) nbr_generated_documents = 5 # Generate document that match outlier command_name = "default_name_" for i in range(nbr_generated_documents): dummy_doc_generated = dummy_doc_generate.generate_document({ "command_query": command_query, "command_name": command_name + str(i) }) self.test_es.add_doc(dummy_doc_generated) whitelist_doc_generated = dummy_doc_generate.generate_document({ "hostname": "whitelist_hostname", "command_query": command_query, "command_name": command_name + str(nbr_generated_documents) }) self.test_es.add_doc(whitelist_doc_generated) # Run analyzer self.test_settings.change_configuration_path( test_conf_file_with_whitelist) analyzer = AnalyzerFactory.create( root_test_use_case_files + "sudden_appearance_dummy_test_01.conf") set_new_current_date(analyzer) analyzer.evaluate_model() nbr_outliers = 0 for elem in es._scan(): if "outliers" in elem["_source"]: nbr_outliers += 1 self.assertEqual(nbr_outliers, nbr_generated_documents)
def test_whitelist_correctly_reload_after_update_config(self): self.test_settings.change_configuration_path(test_whitelist_single_literal_file) dummy_doc_gen = DummyDocumentsGenerate() doc = dummy_doc_gen.generate_document({"create_outlier": True, "outlier_observation": "dummy observation", "filename": "osquery_get_all_processes_with_listening_conns.log"}) # With this configuration, outlier is not whitlisted self.assertFalse(Outlier.is_whitelisted_doc(doc)) # Update configuration self.test_settings.change_configuration_path(test_whitelist_multiple_literal_file) # Now outlier is whitelisted self.assertTrue(Outlier.is_whitelisted_doc(doc))
def test_simplequery_not_use_derived_fields_but_present_in_outlier(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/simplequery_test_02.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/simplequery/simplequery_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue( "derived_timestamp_year" in result['_source']['outliers'])
def test_batch_whitelist_work_doent_match_outlier_in_across(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_whitelist_batch.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_batch_whitelist_across_float.conf" ) doc_to_generate = [ # agg1 (0, 1, 2) -> 3 but with whitelist: (0, 2) -> 2 # agg2 (0, 3, 4) -> 3 ("agg1", 0, False), ("agg1", 1, True), ("agg2", 0, False), ("agg2", 0, False), ("agg1", 2, False), ("agg2", 3, False), ("agg2", 4, False) ] dummy_doc_gen = DummyDocumentsGenerate() for aggregator, target_value, is_whitelist in doc_to_generate: deployment_name = None if is_whitelist: deployment_name = "whitelist-deployment" user_id = target_value hostname = aggregator doc_generated = dummy_doc_gen.generate_document({ "deployment_name": deployment_name, "user_id": user_id, "hostname": hostname }) self.test_es.add_doc(doc_generated) analyzer.evaluate_model() list_outliers = [] for doc in es._scan(): if "outliers" in doc["_source"]: list_outliers.append( (doc["_source"]["outliers"]["aggregator"][0], doc["_source"]["outliers"]["term"][0])) # We detect agg2 but not agg1 self.assertEqual(list_outliers, [("agg2", "0"), ("agg2", "0"), ("agg2", "3"), ("agg2", "4")])
def test_bulk_update_do_not_remove_values(self): dummy_doc_gen = DummyDocumentsGenerate() doc = dummy_doc_gen.generate_document({"create_outlier": True}) self.test_es.add_doc(doc) test_doc = copy.deepcopy(doc) # Remove outlier test_doc["_source"].pop("outliers") # Update the document (without outliers) es.add_update_bulk_action(test_doc) # Result in ES is the same that the original document (outliers wasn't removed) result = [elem for elem in es._scan()][0] self.assertEqual(doc, result)
def test_arbitrary_key_config_present_in_outlier(self): self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create( use_case_simplequery_arbitrary_dummy_test) dummy_doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc(dummy_doc_generate.generate_document()) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertEquals(result["_source"]["outliers"]["test_arbitrary_key"], ["arbitrary_value"])
def test_arbitrary_key_config_not_present_int_other_model(self): # Dictionary and list could be share between different instance. This test check that a residual value is not # present in the dictionary self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create(use_case_simplequery_dummy_test) dummy_doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc(dummy_doc_generate.generate_document()) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("test_arbitrary_key" in result["_source"]["outliers"])
def test_whitelist_regex_per_model_match_whitelist(self): doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc( doc_generate.generate_document({"hostname": "AAA-WHITELISTED"})) # Run analyzer self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create( use_case_whitelist_tests_model_whitelist_02) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("outliers" in result["_source"])
def test_terms_not_use_derived_fields_but_present_in_outlier(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 11})) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] # The parameter use_derived_fields haven't any impact on outliers keys self.assertTrue( "derived_timestamp_year" in result['_source']['outliers'])
def test_email_dict_key(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outlier doc = doc_generate.generate_document() outlier = Outlier("dummy type", "dummy reason", "dummy summary", doc) # execute notification es.notifier.notify_on_outlier(outlier) email_dict = self.test_notifier.get_list_email()[0] self.assertEqual(list(email_dict.keys()), ["subject", "body"]) self.test_notifier.restore_notifier()
def test_notification_on_outlier_match_metrics(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create document that's an outlier doc = doc_generate.generate_document({"user_id": 11}) self.test_es.add_doc(doc) analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/notifications/metrics_numerical_value_dummy_test.conf" ) analyzer.evaluate_model() self.assertEqual(len(self.test_notifier.get_list_email()), 1) self.test_notifier.restore_notifier()
def test_simplequery_no_extra_outlier_infos(self): dummy_doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc(dummy_doc_generate.generate_document()) # Run analyzer self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create(use_case_simplequery_dummy_test) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] all_fields_exists = [ elem in DEFAULT_OUTLIERS_KEY_FIELDS for elem in result['_source']['outliers'] ] self.assertTrue(all(all_fields_exists))
def test_whitelist_regex_per_model_not_match_whitelist(self): doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc( doc_generate.generate_document( {"hostname": "Not-work-WHITELISTED"})) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/simplequery_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/simplequery/whitelist_tests_model_whitelist_02.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue("outliers" in result["_source"])
def test_add_document_to_batch_one_aggregator_and_one_target(self): dummy_doc_generate = DummyDocumentsGenerate() dummy_doc = dummy_doc_generate.generate_document() target_value = "dummy_target" aggregator_value = "dummy_aggregator" current_batch = {"dummy_key": "dummy_value"} result = TermsAnalyzer._add_document_to_batch(current_batch, [target_value], [aggregator_value], dummy_doc) expected_batch = current_batch.copy() expected_batch[aggregator_value] = defaultdict(list) expected_batch[aggregator_value]["targets"].append(target_value) expected_batch[aggregator_value]["observations"].append(dict()) expected_batch[aggregator_value]["raw_docs"].append(dummy_doc) self.assertEqual(result, expected_batch)
def _generate_metrics_doc_with_whitelist(self, doc_to_generate): # Use list of tuple (and not dict) to keep order dummy_doc_gen = DummyDocumentsGenerate() for aggregator, target_value, is_whitelist in doc_to_generate: deployment_name = None if is_whitelist: deployment_name = "whitelist-deployment" user_id = target_value hostname = aggregator doc_generated = dummy_doc_gen.generate_document({ "deployment_name": deployment_name, "user_id": user_id, "hostname": hostname }) self.test_es.add_doc(doc_generated)
def test_notify_on_outlier_correctly_create_email(self): import logging, sys logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outlier doc = doc_generate.generate_document() outlier = Outlier("dummy type", "dummy reason", "dummy summary", doc) # execute notification es.notifier.notify_on_outlier(outlier) self.assertEqual(len(self.test_notifier.get_list_email()), 1) self.test_notifier.restore_notifier()