def test_add_log_message(self): model = Drain() entries = str.splitlines(""" Dec 10 07:07:38 LabSZ sshd[24206]: input_userauth_request: invalid user test9 [preauth] Dec 10 07:08:28 LabSZ sshd[24208]: input_userauth_request: invalid user webmaster [preauth] Dec 10 09:12:32 LabSZ sshd[24490]: Failed password for invalid user ftpuser from 0.0.0.0 port 62891 ssh2 Dec 10 09:12:35 LabSZ sshd[24492]: Failed password for invalid user pi from 0.0.0.0 port 49289 ssh2 Dec 10 09:12:44 LabSZ sshd[24501]: Failed password for invalid user ftpuser from 0.0.0.0 port 60836 ssh2 Dec 10 07:28:03 LabSZ sshd[24245]: input_userauth_request: invalid user pgadmin [preauth] """) expected = str.splitlines(""" Dec 10 07:07:38 LabSZ sshd[24206]: input_userauth_request: invalid user test9 [preauth] Dec 10 <*> LabSZ <*> input_userauth_request: invalid user <*> [preauth] Dec 10 09:12:32 LabSZ sshd[24490]: Failed password for invalid user ftpuser from 0.0.0.0 port 62891 ssh2 Dec 10 <*> LabSZ <*> Failed password for invalid user <*> from 0.0.0.0 port <*> ssh2 Dec 10 <*> LabSZ <*> Failed password for invalid user <*> from 0.0.0.0 port <*> ssh2 Dec 10 <*> LabSZ <*> input_userauth_request: invalid user <*> [preauth] """) actual = [] for entry in entries: cluster, change_type = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(8, model.get_total_cluster_size())
def test_max_clusters(self): """Verify model respects the max_clusters option. Key difference between this and other tests is that with `max_clusters` set to 1 model is capable of keeping track of a single cluster at a time. Consequently, when log stream switched form the A format to the B and back model doesn't recognize it and returnes a new template with no slots. """ model = Drain(max_clusters=1) entries = str.splitlines(""" A format 1 A format 2 B format 1 B format 2 A format 3 """) expected = str.splitlines(""" A format 1 A format <*> B format 1 B format <*> A format 3 """) actual = [] for entry in entries: cluster, change_type = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(1, model.get_total_cluster_size())
def test_depth_3(self): """When `depth` is set to 3 then there is no prefixing based on any of the tokens""" model = Drain( depth=3, sim_th=0.3, max_children=100, ) entries = str.splitlines(""" X A B X A C Y A B Y A C """) expected = str.splitlines(""" X A B X A <*> <*> A <*> <*> A <*> """) actual = [] for entry in entries: cluster, change_type = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(6, model.get_total_cluster_size())
def test_depth_4(self): """When `depth` is set to 4 then the first token is never wild-carded because it's part of the prefix tree""" model = Drain( depth=4, sim_th=0.3, max_children=100, ) entries = str.splitlines(""" X A B X A C Y A B Y A C """) expected = str.splitlines(""" X A B X A <*> Y A B Y A <*> """) actual = [] for entry in entries: cluster, change_type = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(6, model.get_total_cluster_size())
def test_max_clusters_lru_multiple_leaf_nodes(self): """When all templates end up in different nodes and the max number of clusters is reached, then clusters are removed according to the lru policy. """ model = Drain(max_clusters=2, depth=4, param_str="*") entries = [ "A A A", "A A B", "B A A", "B A B", "C A A", "C A B", "B A A", "A A A", ] expected = [ # lru: [] "A A A", # lru: ["A A A"] "A A *", # lru: ["A A *"] "B A A", # lru: ["B A A", "A A *"] "B A *", # lru: ["B A *", "A A *"] "C A A", # lru: ["C A A", "B A *"] "C A *", # lru: ["C A *", "B A *"] "B A *", # Message "B A A" was normalized because the template "B A *" is # still present in the cache. # lru: ["B A *", "C A *"] "A A A", # Message "A A A" was not normalized because the template "C A A" # pushed out the template "A A *" from the cache. # lru: ["A A A", "C A *"] ] actual = [] for entry in entries: cluster, _ = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(4, model.get_total_cluster_size())
def test_max_clusters_lru(self): """When max number of clusters is reached, then clusters are removed according to the lru policy. """ model = Drain(max_clusters=3, depth=3) entries = [ "A A foramt 1", "A A foramt 2", "A B format 1", "A B format 2", "B format 1", "B format 2", "A A foramt 3", "C foramt 1", "A B format 3", ] expected = [ "A A foramt 1", # LRU = ["A"] "A A foramt <*>", # LRU = ["A"] # Use "A A" prefix to make sure both "A" and "A A" clusters end up # in the same leaf node. This is a setup for an interesting edge # case. "A B format 1", # LRU = ["AA", "A"] "A B format <*>", # LRU = ["AA", "A"] "B format 1", # LRU = ["B", "A A", "A"] "B format <*>", # LRU = ["B", "A A", "A"] "A A foramt <*>", # LRU = ["A", "B", "A A"] "C foramt 1", # LRU = ["C", "A", "B"] # Cluster "A A" should have been removed in the previous step, thus, # it should be recognized as a new cluster with no slots. "A B format 3", # LRU = ["A A', "C", "A"] ] actual = [] for entry in entries: cluster, _ = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(5, model.get_total_cluster_size())
def test_add_log_message_sim_75(self): """When `sim_th` is set to 75% then only certain log entries match. In this test similarity threshold is set to 75% which makes the model less aggressive in grouping entries into clusters. In particular, it only finds clusters for "Failed password" entries. """ model = Drain( depth=4, sim_th=0.75, max_children=100, ) entries = str.splitlines( """ Dec 10 07:07:38 LabSZ sshd[24206]: input_userauth_request: invalid user test9 [preauth] Dec 10 07:08:28 LabSZ sshd[24208]: input_userauth_request: invalid user webmaster [preauth] Dec 10 09:12:32 LabSZ sshd[24490]: Failed password for invalid user ftpuser from 0.0.0.0 port 62891 ssh2 Dec 10 09:12:35 LabSZ sshd[24492]: Failed password for invalid user pi from 0.0.0.0 port 49289 ssh2 Dec 10 09:12:44 LabSZ sshd[24501]: Failed password for invalid user ftpuser from 0.0.0.0 port 60836 ssh2 Dec 10 07:28:03 LabSZ sshd[24245]: input_userauth_request: invalid user pgadmin [preauth] """ ) expected = str.splitlines( """ Dec 10 07:07:38 LabSZ sshd[24206]: input_userauth_request: invalid user test9 [preauth] Dec 10 07:08:28 LabSZ sshd[24208]: input_userauth_request: invalid user webmaster [preauth] Dec 10 09:12:32 LabSZ sshd[24490]: Failed password for invalid user ftpuser from 0.0.0.0 port 62891 ssh2 Dec 10 <*> LabSZ <*> Failed password for invalid user <*> from 0.0.0.0 port <*> ssh2 Dec 10 <*> LabSZ <*> Failed password for invalid user <*> from 0.0.0.0 port <*> ssh2 Dec 10 07:28:03 LabSZ sshd[24245]: input_userauth_request: invalid user pgadmin [preauth] """ ) actual = [] for entry in entries: cluster, change_type = model.add_log_message(entry) actual.append(cluster.get_template()) self.assertListEqual(list(map(str.strip, expected)), actual) self.assertEqual(8, model.get_total_cluster_size())