def test_getValidationResult_coverage_error_call(self):
expected_results = [
(None, None),
(None, None),
("false(unreach-call)", result.CATEGORY_CORRECT),
(None, None),
(None, None),
]
for expected, file in zip(expected_results, files):
run = copy.deepcopy(mock_get_verification_result(file))
run.set("properties", "coverage-error-call")
status_from_verification = run.find('column[@title="status"]').get(
"value")
category_from_verification = run.find(
'column[@title="category"]').get("value")
actual = mergeBenchmarkSets.get_validation_result(
run,
mock_witness_sets(),
status_from_verification,
category_from_verification,
)
self.assertEqual(expected, actual[:2])
self.assertEqual(status_from_verification, actual[2])
if file == "../sv-benchmarks/c/array-patterns/array28_pattern.yml":
self.assertEqual(result.CATEGORY_CORRECT, actual[3])
self.assertNotEqual(None, run.find('column[@title="score"]'))
else:
self.assertEqual(category_from_verification, actual[3])
def test_getValidationResult_multiple_witnesses(self):
new_witness_results = [
("ERROR (invalid witness syntax)", result.CATEGORY_ERROR),
("ERROR (invalid witness file)", result.CATEGORY_ERROR),
("false (unreach-call)", result.CATEGORY_WRONG),
("true", result.CATEGORY_WRONG),
("false (unreach-call)", result.CATEGORY_CORRECT),
]
expected_results = [
("witness invalid (true)", result.CATEGORY_ERROR),
("result invalid (TIMEOUT)", result.CATEGORY_ERROR),
("result invalid (false(unreach-call))", result.CATEGORY_ERROR),
("false(unreach-call)", result.CATEGORY_CORRECT),
("witness invalid (false(unreach-call))", result.CATEGORY_ERROR),
]
witness_set_1 = mock_witness_sets()
witness_set_2 = copy.deepcopy(witness_set_1)
for expected, file, new_witness_result in zip(
expected_results, files, new_witness_results
):
verification_run = mock_get_verification_result(file)
witness_run = witness_set_2[0].get(file)
witness_run.find('column[@title="status"]').set(
"value", new_witness_result[0]
)
witness_run.find('column[@title="category"]').set(
"value", new_witness_result[1]
)
status_from_verification = verification_run.find(
'column[@title="status"]'
).get("value")
category_from_verification = verification_run.find(
'column[@title="category"]'
).get("value")
actual = mergeBenchmarkSets.get_validation_result(
verification_run,
witness_set_1 + [{file: witness_run}],
status_from_verification,
category_from_verification,
)
self.assertEqual(expected, actual[:2])
self.assertEqual(
(status_from_verification, category_from_verification), actual[2:]
)
def test_getValidationResult_malformed_coverage(self):
modified_run = copy.deepcopy(
results_xml.find(
'run[@name="../sv-benchmarks/c/array-examples/sanfoundry_24-1.yml"]'
))
modified_run.set("properties", "coverage-branches")
modified_witness_run = copy.deepcopy(
witness_xml_1.find(
'run[@name="../sv-benchmarks/c/array-examples/sanfoundry_24-1.yml"]'
))
coverage_column = ET.Element(
"column",
title="branches_covered",
value="fifty percent", # this cannot be parsed into a number
)
modified_witness_run.append(coverage_column)
actual = mergeBenchmarkSets.get_validation_result(
modified_run,
[{
modified_witness_run.get("name"): modified_witness_run
}],
result.RESULT_TRUE_PROP,
result.CATEGORY_CORRECT,
)
# we should still be able to assign the correct results:
self.assertTupleEqual(
(
result.RESULT_TRUE_PROP,
result.CATEGORY_CORRECT,
result.RESULT_TRUE_PROP,
result.CATEGORY_CORRECT,
),
actual,
)
# score should be None since we were not able to parse "fifty percent" above:
self.assertTrue(
modified_witness_run.find('column[@title="score"]') is None)
def test_getValidationResult_coverage_branches(self):
for file in files:
run = copy.deepcopy(mock_get_verification_result(file))
run.set("properties", "coverage-branches")
status_from_verification = run.find('column[@title="status"]').get(
"value")
category_from_verification = run.find(
'column[@title="category"]').get("value")
actual = mergeBenchmarkSets.get_validation_result(
run,
mock_witness_sets(),
status_from_verification,
category_from_verification,
)
self.assertTupleEqual(
(
status_from_verification,
result.CATEGORY_CORRECT,
status_from_verification,
result.CATEGORY_CORRECT,
),
actual,
)
self.assertNotEqual(None, run.find('column[@title="score"]'))
def test_getValidationResult_single_witness(self):
expected_results = [
("true", result.CATEGORY_CORRECT_UNCONFIRMED),
("result invalid (TIMEOUT)", result.CATEGORY_ERROR),
("result invalid (false(unreach-call))", result.CATEGORY_ERROR),
("false(unreach-call)", result.CATEGORY_CORRECT),
("witness invalid (false(unreach-call))", result.CATEGORY_ERROR),
]
for expected, file in zip(expected_results, files):
run = mock_get_verification_result(file)
status_from_verification = run.find('column[@title="status"]').get(
"value")
category_from_verification = run.find(
'column[@title="category"]').get("value")
actual = mergeBenchmarkSets.get_validation_result(
run,
mock_witness_sets(),
status_from_verification,
category_from_verification,
)
self.assertEqual(expected, actual[:2])
self.assertEqual(
(status_from_verification, category_from_verification),
actual[2:])