def test_terms_small_batch_treat_all(self): dummy_doc_generate = DummyDocumentsGenerate() # Init the list of user nbr_doc_per_hours = 5 nbr_hours = 10 nbr_doc_generated_per_hours = [ nbr_doc_per_hours for _ in range(nbr_hours) ] # If the number of document per hours is not a divisor of the batch limit, all document will not be detected # Generate documents self.test_es.add_multiple_docs( dummy_doc_generate.generate_doc_time_variable_sensitivity( nbr_doc_generated_per_hours)) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_small_batch_eval.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_float.conf" ) analyzer.evaluate_model() self.assertEqual(analyzer.total_outliers, nbr_doc_per_hours * nbr_hours)
def test_sudden_extra_outlier_infos_all_present(self): # Generate documents dummy_doc_generate = DummyDocumentsGenerate() list_delta_hour = [1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 15] field_1_name = "user_id" list_field_1_value = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] field_2_name = "hostname" list_field_2_value = [] for _ in range(len(list_delta_hour)): list_field_2_value.append("host1") generated_docs = dummy_doc_generate.generate_doc_time_variable_witt_custom_fields( list_delta_hour, field_1_name, list_field_1_value, field_2_name, list_field_2_value) self.test_es.add_multiple_docs(generated_docs) self.test_settings.change_configuration_path(test_conf_file_01) analyzer = AnalyzerFactory.create( root_test_use_case_files + "sudden_appearance_dummy_test_03.conf") set_new_current_date(analyzer) analyzer.evaluate_model() list_outlier = list() for elem in es._scan(): if "outliers" in elem["_source"]: list_outlier.append(elem) all_fields_exists = [ elem in EXTRA_OUTLIERS_KEY_FIELDS + DEFAULT_OUTLIERS_KEY_FIELDS for elem in list_outlier[0]['_source']['outliers'] ] self.assertTrue(all(all_fields_exists))
def test_whitelist_literals_per_model_not_removed_by_housekeeping(self): # Init doc_generate = DummyDocumentsGenerate() self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/housekeeping.conf") housekeeping = HousekeepingJob() # Generate document document = doc_generate.generate_document({ "hostname": "NOT-WHITELISTED", "create_outlier": True, "outlier.model_name": "dummy_test", "outlier.model_type": "simplequery" }) self.assertTrue("outliers" in document["_source"]) analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/housekeeping/analyzer_dummy_test_with_whitelist.conf" ) housekeeping.update_analyzer_list([analyzer]) self.test_es.add_doc(document) housekeeping.execute_housekeeping() result = [elem for elem in self.test_es._scan()][0] self.assertTrue("outliers" in result["_source"])
def test_non_outliers_present_in_metrics(self): dummy_doc_generate = DummyDocumentsGenerate() # Generate documents # Outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 11})) # Non outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 8})) # Outlier document self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 12})) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/metrics_test_02.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/metrics/metrics_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][2] self.assertEqual( result["_source"]["outliers"]["non_outlier_values_sample"], ["8.0"])
def test_metrics_whitelist_work_test_es_result(self): dummy_doc_generate = DummyDocumentsGenerate() command_query = "SELECT * FROM dummy_table" # must be bigger than the trigger value (here 3) nbr_generated_documents = 5 # Generate document that match outlier for _ in range(nbr_generated_documents): self.test_es.add_doc( dummy_doc_generate.generate_document( {"command_query": command_query})) # Generate whitelist document self.test_es.add_doc( dummy_doc_generate.generate_document({ "hostname": "whitelist_hostname", "command_query": command_query })) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/metrics_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/metrics/metrics_length_dummy_test.conf" ) analyzer.evaluate_model() nbr_outliers = 0 for elem in es._scan(): if "outliers" in elem["_source"]: nbr_outliers += 1 self.assertEqual(nbr_outliers, nbr_generated_documents)
def test_notification_on_outlier_already_detected_but_not_in_queue(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outliers doc = doc_generate.generate_document() # Full the queue (3 elements) outlier1 = Outlier("dummy type", "dummy reason", "dummy summary1", doc) es.notifier.notify_on_outlier(outlier1) outlier2 = Outlier("dummy type2", "dummy reason2", "dummy summary2", doc) es.notifier.notify_on_outlier(outlier2) outlier3 = Outlier("dummy type3", "dummy reason3", "dummy summary3", doc) es.notifier.notify_on_outlier(outlier3) # Add a new one that will remove the first outlier4 = Outlier("dummy type4", "dummy reason4", "dummy summary4", doc) es.notifier.notify_on_outlier(outlier4) # Add again the first one es.notifier.notify_on_outlier(outlier1) # All outliers notify need to be present (so 5) self.assertEqual(len(self.test_notifier.get_list_email()), 5) self.test_notifier.restore_notifier()
def test_sudden_appearance_detect_no_outlier_es_check(self): # Generate documents dummy_doc_generate = DummyDocumentsGenerate() list_delta_hour = [1, 1, 1, 3, 3, 3, 4, 5, 5, 5, 15, 15] field_1_name = "user_id" list_field_1_value = [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2] field_2_name = "hostname" list_field_2_value = [] for _ in range(len(list_delta_hour)): list_field_2_value.append("host1") generated_docs = dummy_doc_generate.generate_doc_time_variable_witt_custom_fields( list_delta_hour, field_1_name, list_field_1_value, field_2_name, list_field_2_value) self.test_es.add_multiple_docs(generated_docs) # Run analyzer self.test_settings.change_configuration_path(test_conf_file_01) analyzer = AnalyzerFactory.create( root_test_use_case_files + "sudden_appearance_dummy_test_02.conf") set_new_current_date(analyzer) analyzer.evaluate_model() nbr_outliers = 0 for elem in es._scan(): if "outliers" in elem["_source"]: nbr_outliers += 1 self.assertEqual(nbr_outliers, 0)
def test_remove_all_whitelisted_outliers(self): self.test_settings.change_configuration_path( test_file_whitelist_path_config) doc_generate = DummyDocumentsGenerate() self.test_es.add_doc( doc_generate.generate_document({ "create_outlier": True, "outlier_observation": "dummy observation", "outlier.model_name": "dummy_test", "outlier.model_type": "analyzer", "command_query": "osquery_get_all_processes_with_listening_conns.log" })) # Check that outlier correctly generated result = [doc for doc in es._scan()][0] self.assertTrue("outliers" in result["_source"]) analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/analyzer/analyzer_dummy_test.conf" ) # Remove whitelisted outlier es.remove_all_whitelisted_outliers({"analyzer_dummy_test": analyzer}) # Check that outlier is correctly remove result = [doc for doc in es._scan()][0] self.assertFalse("outliers" in result["_source"])
def test_add_document_to_batch_empty_aggergator(self): dummy_doc_generate = DummyDocumentsGenerate() dummy_doc = dummy_doc_generate.generate_document() current_batch = {"dummy_key": "dummy_value"} result = TermsAnalyzer._add_document_to_batch(current_batch, ["dummy_target"], list(), dummy_doc) self.assertEqual(result, current_batch)
def test_min_target_buckets_dont_detect_outlier(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_whitelist_batch.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_batch_whitelist_within_float.conf" ) # Recap: # min_target_buckets=4 # trigger_sensitivity=5 # trigger_on=high # trigger_method=float # Dont encode with a matrix to keep order of document doc_to_generate = [ # New batch: # 0 1 # agg1 [6, 1] # agg2 [1, 2] ("agg1", 0), ("agg2", 0), ("agg1", 0), ("agg1", 0), ("agg1", 0), ("agg1", 0), ("agg1", 0), ("agg2", 1), ("agg2", 1), ("agg1", 1), # New Batch # 2 # agg1 [0] # agg2 [1] ("agg1", 2) ] # At the end: # 0 1 2 # agg1 [6, 1, 1] # agg2 [1, 2] # Normally agg1 - 0 must be flagged, but here they doesn't have enough buckets values dummy_doc_gen = DummyDocumentsGenerate() for aggregator, target_value in doc_to_generate: user_id = target_value hostname = aggregator doc_generated = dummy_doc_gen.generate_document({ "user_id": user_id, "hostname": hostname }) self.test_es.add_doc(doc_generated) analyzer.evaluate_model() self.assertEqual(analyzer.total_outliers, 0)
def test_whitelist_literal_match(self): self.test_settings.change_configuration_path( test_file_outliers_path_config) # Contain: "C:\Windows\system32\msfeedssync.exe sync" dummy_doc_gen = DummyDocumentsGenerate() doc = dummy_doc_gen.generate_document( {"command_query": r'C:\Windows\system32\msfeedssync.exe sync'}) result = Outlier.is_whitelisted_doc(doc) self.assertTrue(result)
def test_simplequery_not_use_derived_fields_in_doc(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( config_file_simplequery_test_02) analyzer = AnalyzerFactory.create( use_case_simplequery_dummy_test_not_derived) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("timestamp_year" in result['_source'])
def test_simplequry_use_matched_values_in_outlier(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( config_file_simplequery_test_02) analyzer = AnalyzerFactory.create( use_case_simplequery_dummy_test_highlight_match_activated) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue("matched_values" in result['_source']['outliers'])
def test_terms_not_use_derived_fields_in_doc(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("timestamp_year" in result['_source'])
def test_sudden_appearance_derived_fields_in_doc(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path(test_conf_file_01) analyzer = AnalyzerFactory.create( root_test_use_case_files + "sudden_appearance_derived_fields_01.conf") set_new_current_date(analyzer) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue("timestamp_year" in result['_source'])
def test_simplequery_not_use_derived_fields_but_present_in_outlier(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc(dummy_doc_generate.generate_document()) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/simplequery_test_02.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/simplequery/simplequery_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue( "derived_timestamp_year" in result['_source']['outliers'])
def test_whitelist_correctly_reload_after_update_config(self): self.test_settings.change_configuration_path(test_whitelist_single_literal_file) dummy_doc_gen = DummyDocumentsGenerate() doc = dummy_doc_gen.generate_document({"create_outlier": True, "outlier_observation": "dummy observation", "filename": "osquery_get_all_processes_with_listening_conns.log"}) # With this configuration, outlier is not whitlisted self.assertFalse(Outlier.is_whitelisted_doc(doc)) # Update configuration self.test_settings.change_configuration_path(test_whitelist_multiple_literal_file) # Now outlier is whitelisted self.assertTrue(Outlier.is_whitelisted_doc(doc))
def test_batch_whitelist_work_doent_match_outlier_in_across(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_whitelist_batch.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_batch_whitelist_across_float.conf" ) doc_to_generate = [ # agg1 (0, 1, 2) -> 3 but with whitelist: (0, 2) -> 2 # agg2 (0, 3, 4) -> 3 ("agg1", 0, False), ("agg1", 1, True), ("agg2", 0, False), ("agg2", 0, False), ("agg1", 2, False), ("agg2", 3, False), ("agg2", 4, False) ] dummy_doc_gen = DummyDocumentsGenerate() for aggregator, target_value, is_whitelist in doc_to_generate: deployment_name = None if is_whitelist: deployment_name = "whitelist-deployment" user_id = target_value hostname = aggregator doc_generated = dummy_doc_gen.generate_document({ "deployment_name": deployment_name, "user_id": user_id, "hostname": hostname }) self.test_es.add_doc(doc_generated) analyzer.evaluate_model() list_outliers = [] for doc in es._scan(): if "outliers" in doc["_source"]: list_outliers.append( (doc["_source"]["outliers"]["aggregator"][0], doc["_source"]["outliers"]["term"][0])) # We detect agg2 but not agg1 self.assertEqual(list_outliers, [("agg2", "0"), ("agg2", "0"), ("agg2", "3"), ("agg2", "4")])
def test_bulk_update_do_not_remove_values(self): dummy_doc_gen = DummyDocumentsGenerate() doc = dummy_doc_gen.generate_document({"create_outlier": True}) self.test_es.add_doc(doc) test_doc = copy.deepcopy(doc) # Remove outlier test_doc["_source"].pop("outliers") # Update the document (without outliers) es.add_update_bulk_action(test_doc) # Result in ES is the same that the original document (outliers wasn't removed) result = [elem for elem in es._scan()][0] self.assertEqual(doc, result)
def test_arbitrary_key_config_present_in_outlier(self): self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create( use_case_simplequery_arbitrary_dummy_test) dummy_doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc(dummy_doc_generate.generate_document()) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertEquals(result["_source"]["outliers"]["test_arbitrary_key"], ["arbitrary_value"])
def test_arbitrary_key_config_not_present_int_other_model(self): # Dictionary and list could be share between different instance. This test check that a residual value is not # present in the dictionary self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create(use_case_simplequery_dummy_test) dummy_doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc(dummy_doc_generate.generate_document()) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("test_arbitrary_key" in result["_source"]["outliers"])
def test_terms_not_use_derived_fields_but_present_in_outlier(self): dummy_doc_generate = DummyDocumentsGenerate() self.test_es.add_doc( dummy_doc_generate.generate_document({"user_id": 11})) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/terms_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/terms/terms_dummy_test_not_derived.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] # The parameter use_derived_fields haven't any impact on outliers keys self.assertTrue( "derived_timestamp_year" in result['_source']['outliers'])
def test_whitelist_regex_per_model_match_whitelist(self): doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc( doc_generate.generate_document({"hostname": "AAA-WHITELISTED"})) # Run analyzer self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create( use_case_whitelist_tests_model_whitelist_02) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertFalse("outliers" in result["_source"])
def test_email_dict_key(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outlier doc = doc_generate.generate_document() outlier = Outlier("dummy type", "dummy reason", "dummy summary", doc) # execute notification es.notifier.notify_on_outlier(outlier) email_dict = self.test_notifier.get_list_email()[0] self.assertEqual(list(email_dict.keys()), ["subject", "body"]) self.test_notifier.restore_notifier()
def test_notification_on_outlier_match_metrics(self): self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create document that's an outlier doc = doc_generate.generate_document({"user_id": 11}) self.test_es.add_doc(doc) analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/notifications/metrics_numerical_value_dummy_test.conf" ) analyzer.evaluate_model() self.assertEqual(len(self.test_notifier.get_list_email()), 1) self.test_notifier.restore_notifier()
def test_whitelist_regex_per_model_not_match_whitelist(self): doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc( doc_generate.generate_document( {"hostname": "Not-work-WHITELISTED"})) # Run analyzer self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/simplequery_test_01.conf") analyzer = AnalyzerFactory.create( "/app/tests/unit_tests/files/use_cases/simplequery/whitelist_tests_model_whitelist_02.conf" ) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] self.assertTrue("outliers" in result["_source"])
def test_simplequery_no_extra_outlier_infos(self): dummy_doc_generate = DummyDocumentsGenerate() # Generate document self.test_es.add_doc(dummy_doc_generate.generate_document()) # Run analyzer self.test_settings.change_configuration_path( config_file_simplequery_test_01) analyzer = AnalyzerFactory.create(use_case_simplequery_dummy_test) analyzer.evaluate_model() result = [elem for elem in es._scan()][0] all_fields_exists = [ elem in DEFAULT_OUTLIERS_KEY_FIELDS for elem in result['_source']['outliers'] ] self.assertTrue(all(all_fields_exists))
def _generate_metrics_doc_with_whitelist(self, doc_to_generate): # Use list of tuple (and not dict) to keep order dummy_doc_gen = DummyDocumentsGenerate() for aggregator, target_value, is_whitelist in doc_to_generate: deployment_name = None if is_whitelist: deployment_name = "whitelist-deployment" user_id = target_value hostname = aggregator doc_generated = dummy_doc_gen.generate_document({ "deployment_name": deployment_name, "user_id": user_id, "hostname": hostname }) self.test_es.add_doc(doc_generated)
def test_add_document_to_batch_one_aggregator_and_one_target(self): dummy_doc_generate = DummyDocumentsGenerate() dummy_doc = dummy_doc_generate.generate_document() target_value = "dummy_target" aggregator_value = "dummy_aggregator" current_batch = {"dummy_key": "dummy_value"} result = TermsAnalyzer._add_document_to_batch(current_batch, [target_value], [aggregator_value], dummy_doc) expected_batch = current_batch.copy() expected_batch[aggregator_value] = defaultdict(list) expected_batch[aggregator_value]["targets"].append(target_value) expected_batch[aggregator_value]["observations"].append(dict()) expected_batch[aggregator_value]["raw_docs"].append(dummy_doc) self.assertEqual(result, expected_batch)
def test_notify_on_outlier_correctly_create_email(self): import logging, sys logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) self.test_settings.change_configuration_path( "/app/tests/unit_tests/files/notifications_test.conf") self.test_notifier = TestStubNotifier() doc_generate = DummyDocumentsGenerate() # Create outlier doc = doc_generate.generate_document() outlier = Outlier("dummy type", "dummy reason", "dummy summary", doc) # execute notification es.notifier.notify_on_outlier(outlier) self.assertEqual(len(self.test_notifier.get_list_email()), 1) self.test_notifier.restore_notifier()