def test_remove_all_stars_template(self):
template0 = Template(['*', '*', '2000'])
template1 = Template(['file', 'config', '1000'])
template2 = Template(['*', '*', '*'])
chromosome = Chromosome({3:[template0, template1, template2]})
remove_all_stars_template(chromosome, 3)
self.assertEqual(chromosome.to_string(), "[ * * 2000 ]\n[ file config 1000 ]\n")
def change_template(self, ch: Chromosome, cluster_id: int,
template_index: int):
# get the template at position index
template = ch.templates[cluster_id][template_index]
star_indexes = list()
non_star_indexes = list()
for index in range(0, template.get_length()):
if template.token[index] == '*':
star_indexes.append(index)
elif not (startswith(template.token[index], "[")
or template.token[index] == '#spec#'
or re.match("\B\W\Z", template.token[index])):
non_star_indexes.append(index)
if random.random() <= 0.50:
if len(star_indexes) > 0:
index = random.choice(star_indexes)
message_index = random.choice(template.matched_lines)
message = self.chGenerator.messages[cluster_id][message_index]
template.token[index] = message.words[index]
else:
if len(non_star_indexes) > 0:
index = random.choice(non_star_indexes)
clone = Template(template.token[:])
clone.token[index] = '*'
compute_matched_lines(self.chGenerator.messages, clone)
if len(clone.matched_lines) > len(template.matched_lines):
template.token[index] = '*'
template.set_changed(True)
compute_matched_lines(self.chGenerator.messages, template)
def test_is_all_star_template(self):
template = Template(['*', '*', '*', '*'])
self.assertTrue(is_all_star_template(template))
template1 = Template(['*', 'message', '*', '*'])
chromosome = Chromosome({4: [template1]})
self.assertFalse(is_all_star_template(chromosome.templates[4][0]))
def test_fix_all_star_template(self):
message1 = Message(['message', 'sent', 'A1'])
message2 = Message(['message', 'sent', 'A2'])
message3 = Message(['message', 'sent', 'A2', 'from', ':', 'B1'])
message4 = Message(['message', 'sent', 'A2', 'from', ':', 'B2'])
messages = {3: [message1, message2], 6: [message3, message4]}
template = Template(['*', '*', '*', '*', '*', '*'])
template.matched_lines = [0, 1]
ch = Chromosome({6: [template]})
fix_all_star_template(ch, 6, 0, messages)
self.assertFalse(is_all_star_template(ch.templates[6][0]))
def test_add_template_to_reach_100cov(self):
logfile = ROOT_DIR + '/test/resources/File.log'
chrom_gen = ChromosomeGenerator(logfile, 0, '\n', ["'[\w\d\$\-:,\./_ ><\|]*'"])
chrom_mutator_100 = ChromosomeMutator100cov(chrom_gen)
template = Template(['Message', 'sent', 'by', 'EEE', ',', 'at', 'port', '1'])
template.matched_lines = [0]
chromosome = Chromosome({8: [template]})
chrom_mutator_100.add_template_to_reach_100cov(chromosome, 8)
chromosome_matched_lines = []
for t in chromosome.templates[8]:
for i in t.matched_lines:
chromosome_matched_lines.append(i)
self.assertEqual(len(set(chromosome_matched_lines)), len(chrom_gen.messages[8]))
def test_apply_mutation(self):
logfile = ROOT_DIR + '/test/resources/File.log'
chrom_gen = ChromosomeGenerator(logfile, 0, '\n', ["'[\w\d\$\-:,\./_ ><\|]*'"])
chrom_mutator_100 = ChromosomeMutator100cov(chrom_gen)
template = Template(['Driver', ':', '*'])
template.matched_lines = [5]
chromosome = Chromosome({3: [template]})
chrom_mutator_100.apply_mutation(chromosome)
chromosome_matched_lines = []
for t in chromosome.templates[3]:
for i in t.matched_lines:
chromosome_matched_lines.append(i)
self.assertEqual(len(set(chromosome_matched_lines)), len(chrom_gen.messages[3]))
def test_no_variable_parts(self):
# create messages
message1 = Message(['read', 'text', 'file1', 'from', 'ABC1'])
message2 = Message(['read', 'text', 'file2', 'from', 'ABC2'])
message3 = Message(['read', 'text', 'file3', 'from', 'ABC3'])
messages = {5: [message1, message2, message3]}
# create a chromosome
template = Template(['read', 'text', '*', 'from', '*'])
template.matched_lines = [0, 1, 2]
ch = Chromosome({5: [template]})
# code to test
check_variable_parts(ch, messages)
self.assertEqual(ch.templates[5][0].to_string(),
"[ read text * from * ]")
def compute_matched_lines(messages: dict, template: Template):
# if the templates has not be changed
# we don't need to recompute the list of matched lines
if not template.is_changed():
return
# Otherwise, we recompute the list of matched lines
template.matched_lines = []
cluster_id = template.get_length()
for index in range(0, len(messages[cluster_id])):
message = messages[cluster_id][index]
if match(message, template):
template.matched_lines.append(index)
# Now the template has to be set as NOT CHANGED
template.set_changed(False)
def template_match(template1: Template, template2: Template):
""" Compare two lists of strings
:param template1: first template
:param template2: second template
:return: True if template1 is a "sub-set" of template2
False if two lists are different
"""
if template1.get_length() != template2.get_length():
return False
for index in range(0, template1.get_length()):
if template2.token[index] == "*":
continue
if template1.token[index] != template2.token[index]:
return False
return True
def generate_template_from_line(self, cluster_id: int, rand_value: int):
""" Generates a Template from a cluster
"""
template = self.messages[cluster_id][rand_value].words[:]
if len(template) > 1:
contains_star = False
for word in template:
if word == "#spec#":
contains_star = True
break
if not contains_star:
modifiable_indexes = list()
for index in range(0, len(template)):
if not (startswith(template[index], "[")
or re.match("\B\W\Z", template[index])):
modifiable_indexes.append(index)
if len(modifiable_indexes) > 0:
index = random.choice(modifiable_indexes)
template[index] = '*'
t = Template(template)
compute_matched_lines(self.messages, t)
return t
def test_compute_matched_lines(self):
messages_dict = {1:[Message(['configuration']), Message(['Data']), Message(['Object'])],
2:[Message(['configuration', 'file']), Message(['context', 'folder']), Message(['Close', 'file'])],
4:[Message(['AAA', 'Found', 'context', 'configuration'])]}
template = Template(['context', '*'])
compute_matched_lines(messages_dict, template)
self.assertTrue(template.matched_lines, [1])
def match(message: Message, template: Template):
""" Compare two lists of strings
:param message: a message from the log file
:param template: a template from the chromosome
:return: True if two list are equal
False if two lists are different
"""
for index in range(0, template.get_length()):
if template.token[index] == "*" or template.token[index] == "#spec#":
continue
if template.token[index] != message.words[index]:
return False
return True
def add_template(self, template: Template):
""" Adds a template to the chromosome
:param template: an object of type Template
:return: the template will be added to the corresponding cluster
with key = len(template)
"""
key = template.get_length()
if key in self.templates.keys():
self.templates[key].append(template)
else:
self.templates[key] = []
self.templates[key].append(template)
def test_remove_clones(self):
# 1) construct the pop of chromosomes
t1 = Template(['generate', 'code', '*'])
t2 = Template(['state', ':', 'close'])
t3 = Template(['Message', 'sent', 'by', 'EEE', 'at', 'port', '*'])
t4 = Template(['*', ':', 'run'])
ch1 = Chromosome({3: [t1, t2], 7: [t3]})
ch2 = Chromosome({3: [t1, t2]})
ch3 = Chromosome({3: [t1, t4], 7: [t3]})
ch4 = Chromosome({3: [t1, t2]})
ch5 = Chromosome({3: [t1, t2]})
pop = [ch1, ch2, ch3, ch4, ch5]
unique = remove_clones(pop)
self.assertEqual(len(unique), 3)
templates1 = "[ generate code * ]\n[ state : close ]\n" \
"[ Message sent by EEE at port * ]\n"
templates2 = "[ generate code * ]\n[ * : run ]\n" \
"[ Message sent by EEE at port * ]\n"
self.assertEqual(unique[0].to_string(), templates1)
self.assertEqual(unique[2].to_string(), templates2)
def delete_template(self, template: Template):
""" Deletes the template from the given cluster id
:param
index: cluster id
:return
the cluster without the template
:raises
IndexError: An error occurs if a negative cluster id is provided
or if the cluster id doesn't exist
"""
key = template.get_length()
if key not in self.templates:
raise IndexError("No cluster with the giving id!")
self.templates[key].remove(template)
def test_check_variable_parts_2templates(self):
# create messages
message1 = Message(['read', 'text', 'file1', 'from', 'ABC1'])
message2 = Message(['read', 'text', 'file2', 'from', 'ABC3'])
message3 = Message(['read', 'text', 'file2', 'from', 'ABC4'])
message4 = Message(
['read', 'text', 'file', 'ABC', 'from', 'DB', '98765'])
message5 = Message(
['read', 'text', 'file', 'DSE', 'from', 'DB', '7654'])
messages = {5: [message1, message2, message3], 7: [message4, message5]}
# create a chromosome
template1 = Template(['read', '*', '*', 'from', '*'])
template1.matched_lines = [0, 1, 2]
template2 = Template(['read', '*', 'file', '*', 'from', 'DB', '*'])
template2.matched_lines = [0, 1]
ch = Chromosome({5: [template1], 7: [template2]})
# code to test
check_variable_parts(ch, messages)
self.assertEqual(ch.templates[5][0].to_string(),
"[ read text * from * ]")
self.assertEqual(ch.templates[7][0].to_string(),
"[ read text file * from DB * ]")
def test_no_match1(self):
message = Message(['AAA', 'BBB', 'CCC', 'DDD:', 'xyz.txt'])
template = Template(['read', 'config', 'File'])
is_matching = match(message, template)
self.assertFalse(is_matching)
def test_remove_super_templates(self):
t1 = Template(['read', 'text', 'file', 'ABC', 'from', 'DB', '*'])
t1.matched_lines = [3]
t2 = Template(['read', 'text', 'file', '*', 'from', 'DB', '*'])
t2.matched_lines = [0, 1]
t3 = Template(['read', 'text', 'file', '*', 'from', 'DB', '2323232'])
t3.matched_lines = [0]
t4 = Template(['read', 'text', 'file', '*', 'from', 'DB', '*'])
t4.matched_lines = [0, 1]
t5 = Template(['file', 'configuration', '*'])
t5.matched_lines = [2, 3, 4]
t6 = Template(['file', 'configuration', 'A'])
t6.matched_lines = [2]
ch = Chromosome({7: [t1, t2, t3, t4, t3, t4], 3: [t5, t6]})
remove_super_templates(ch, 7)
self.assertEqual(ch.cluster_size(7), 2)
remove_super_templates(ch, 3)
self.assertEqual(ch.cluster_size(3), 1)
self.assertEqual(ch.templates[3][0].token, ['file', 'configuration', 'A'])
def test_derive_template(self):
t1 = Template(['configuration', 'file', '*'])
t1.matched_lines = [0, 1, 2, 3, 4, 5]
t2 = Template(['configuration', 'file', 'f1'])
t2.matched_lines = [1]
t3 = Template(['configuration', 'file', '*'])
t3.matched_lines= [0, 1, 2, 3, 4, 5]
t4 = Template(['configuration', 'file', 'f3'])
t4.matched_lines = [4]
t5 = Template(['configuration', 'file', 'f5'])
t5.matched_lines = [5]
t6 = Template(['configuration', 'file', 'f6'])
t6.matched_lines = [3]
partition = [t1, t2, t3, t4, t5, t6]
to_remove_sub = set()
derive_sub_template(partition, to_remove_sub)
self.assertEqual(len(to_remove_sub),5)
to_remove_super = set()
derive_super_template(partition, to_remove_super)
self.assertEqual(len(to_remove_super), 2)
def test_template_no_match(self):
t1 = Template(['read', 'text', 'file', 'EFG', 'from', 'DB', '*'])
t2 = Template(['read', 'text', 'file', 'ABC', 'from', 'DB', 'ZZZ'])
is_subset = template_match(t1, t2)
self.assertFalse(is_subset)
def test_no_match2(self):
message = Message(['AAA', 'BBB', 'EEE', 'DDD:', 'xyz.txt'])
template = Template(['AAA', 'BBB', 'CCC', 'DDD:', '*'])
is_matching = match(message, template)
self.assertFalse(is_matching)