def test_extract_parameters_direct(self):
config = TemplateMinerConfig()
mi = MaskingInstruction(r"hdfs://[\w.:@-]*((/[\w.~%+-]+)+/?)?",
"hdfs_uri")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"(?P<quote>[\"'`]).*?(?P=quote)",
"quoted_string")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"((?P<p_0>[*_])\2{0,2}).*?\1",
"markdown_emph")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"multiple \*word\* pattern", "*words*")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"some \S+ \S+ pattern", "*words*")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"(\d{1,3}\.){3}\d{1,3}", "ip")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"(?P<number>\d+)\.\d+", "float")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"0[xX][a-fA-F0-9]+", "integer")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"(?P<number>\d+)", "integer")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"HelloWorld", "*")
config.masking_instructions.append(mi)
mi = MaskingInstruction(r"MaskPrefix", "<")
config.masking_instructions.append(mi)
template_miner = TemplateMiner(None, config)
test_vectors = [
("<hdfs_uri>:<integer>+<integer>",
"hdfs://*****:*****@<integer>", "some other cool pattern@0xe1f",
["some other cool pattern", "0xe1f"], ["*words*", "integer"]),
("Another test with <*words*> that includes <integer><integer> and <integer> <*> <integer>",
"Another test with some other 0Xadded pattern that includes 500xc0ffee and 0X4 times 5",
[
"some other 0Xadded pattern", "50", "0xc0ffee", "0X4",
"times", "5"
], ["*words*", "integer", "integer", "integer", "*", "integer"]),
("some <*words*> <*words*>",
"some multiple *word* pattern some confusing *word* pattern",
["multiple *word* pattern",
"some confusing *word* pattern"], ["*words*", "*words*"]),
("<*words*> <*>", "multiple *word* pattern <*words*>",
["multiple *word* pattern", "<*words*>"], ["*words*", "*"]),
("<*> <*>", "HelloWorld Test", ["HelloWorld", "Test"], ["*", "*"]),
("<*> <*>", "HelloWorld <anything>", ["HelloWorld",
"<anything>"], ["*", "*"]),
("<*><integer>", "HelloWorld1", ["HelloWorld",
"1"], ["*", "integer"]),
("<*> works <*>", "This works as-expected",
["This", "as-expected"], ["*", "*"]),
("<memory:<integer>>", "<memory:8>", ["8"], ["integer"]),
("<memory:<integer> <core:<float>>>", "<memory:8 <core:0.5>>",
["8", "0.5"], ["integer", "float"]),
("<*> <memory:<<integer> <core:<float>>>",
"New: <memory:<8 <core:0.5>>", ["New:", "8",
"0.5"], ["*", "integer",
"float"]),
("<<>", "MaskPrefix", ["MaskPrefix"], ["<"]),
("<<<>>", "<MaskPrefix>", ["MaskPrefix"], ["<"]),
("There are no parameters here.", "There are no parameters here.",
[], []),
("<float> <float>", "0.15 10.16 3.19", None, None),
("<float> <float>", "0.15 10.16 test 3.19", None, None),
("<memory:<<integer> <core:<float>>>", "<memory:8 <core:0.5>>",
None, None),
("<<>", "<<>", None, None),
("<*words*> <*words*>", "0.15 0.15", None, None),
]
for template, content, expected_parameters, expected_mask_names in test_vectors:
with self.subTest(template=template,
content=content,
expected_parameters=expected_parameters):
extracted_parameters = template_miner.extract_parameters(
template, content, exact_matching=True)
if expected_parameters is None:
self.assertIsNone(extracted_parameters)
else:
self.assertIsNotNone(extracted_parameters)
self.assertListEqual([
parameter.value for parameter in extracted_parameters
], expected_parameters)
self.assertListEqual([
parameter.mask_name
for parameter in extracted_parameters
], expected_mask_names)
config.load(dirname(__file__) + "/drain3.ini")
config.profiling_enabled = False
template_miner = TemplateMiner(persistence, config)
print(f"Drain3 started with '{persistence_type}' persistence")
print(f"{len(config.masking_instructions)} masking instructions are in use")
print(f"Starting training mode. Reading from std-in ('q' to finish)")
while True:
log_line = input("> ")
if log_line == 'q':
break
result = template_miner.add_log_message(log_line)
result_json = json.dumps(result)
print(result_json)
template = result["template_mined"]
params = template_miner.extract_parameters(template, log_line)
print("Parameters: " + str(params))
print("Training done. Mined clusters:")
for cluster in template_miner.drain.clusters:
print(cluster)
print(
f"Starting inference mode, matching to pre-trained clusters. Input log lines or 'q' to finish"
)
while True:
log_line = input("> ")
if log_line == 'q':
break
cluster = template_miner.match(log_line)
if cluster is None: