def testValidateSimpleAnyArch(self):
s = {
"architectures": "any",
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "foo",
"sources": ["a_is_a_good_name.c", "b-IS-also-A-good-name.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
schema.validateBenchmarkSpecification(s)
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateUnknownCorrectness(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": None}},
}
self.appendSchemaVersion(s)
schema.validateBenchmarkSpecification(s)
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateSimpleWithVariants(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"name": "mybenchmark",
"memory_model": "precise",
"sources": ["a.c", "b.c"],
"variants": {"config1": ["FOO" "BAR=BAZ", "NUM=0"], "config2": ["NUM=1"]},
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
schema.validateBenchmarkSpecification(s)
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testMissingCategories(self):
s = {
"architectures": ["x86_64"],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"'categories' is a required property"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateMissingVerificationTask(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
}
self.appendSchemaVersion(s)
msgRegex = r"'verification_tasks' is a required property"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testCreateSimpleWithImplicitVerficationTasksWithCex(self):
s = {
'architectures': ['x86_64'],
'categories': ['xxx'],
'language': 'c99',
'name': 'foo',
'sources': ['a_is_a_good_name.c', 'b-IS-also-A-good-name.c'],
'verification_tasks': {
'no_assert_fail': {
'correct':
False,
'counter_examples': [
{
"locations": [{
"line": 1,
"file": "a_is_a_good_name.c",
}]
},
]
}
},
}
self.appendSchemaVersion(s)
# Validate benchmark specification
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
benchmarkObjs = svcb.benchmark.getBenchmarks(
s) # Default addImplicitVerificationTasks=True
self.assertTrue(len(benchmarkObjs) == 1)
# Test properties of the benchmark object
b = benchmarkObjs[0]
self.assertEqual(b.name, "foo")
self.assertEqual(b.sources,
['a_is_a_good_name.c', 'b-IS-also-A-good-name.c'])
self.assertEqual(b.architectures, ['x86_64'])
self.assertEqual(b.defines, {}) # Implicitly empty
self.assertEqual(b.language, 'c99')
self.assertEqual(b.description, '') # Implicity empty
self.assertEqual(b.categories, {'xxx'})
self.assertTrue(b.isLanguageC())
self.assertFalse(b.isLanguageCXX())
expectedTasks = copy.deepcopy(
svcb.benchmark.DefaultVerificationTaskStatuses)
expectedTasks['no_assert_fail'] = copy.deepcopy(
s['verification_tasks']['no_assert_fail'])
expectedTasks['no_assert_fail'][
'exhaustive_counter_examples'] = True # Expected implicitly added field
self.assertEqual(b.verificationTasks, expectedTasks)
def testValidateEmptyVerificationTask(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {},
}
self.appendSchemaVersion(s)
msgRegex = r"Failed validating 'minProperties' in schema\['properties'\]\['verification_tasks'\]"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateEmptySources(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": [],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"Failed validating 'minItems' in schema\['properties'\]\['sources'\]"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectRelativeSourcePath(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["../a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"not allowed for '\.\./a.c'"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectSchemaVersion(self):
s = {
"architecture": "foo",
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
"schema_version": 123456,
}
msgRegex = r"Schema version used by benchmark \(\d+\) does not match the currently support schema \(\d+\)"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidCorrectnessType(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": "XXX"}},
}
self.appendSchemaVersion(s)
msgRegex = r"'XXX' is not valid under any of the given schemas"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidMixAnyAndOtherArch(self):
s = {
"architectures": ["any", "x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "foo",
"sources": ["a_is_a_good_name.c", "b-IS-also-A-good-name.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"\['any', 'x86_64'\] is not valid under any of the given schemas"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidatePthreadsDepWithVersion(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "foo",
"sources": ["a_is_a_good_name.c", "b-IS-also-A-good-name.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
"dependencies": {
"pthreads": {"version": "4"}
}, # Should we enforce what the version string should look like?
}
self.appendSchemaVersion(s)
schema.validateBenchmarkSpecification(s)
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testDuplicateCategories(self):
s = {
"architectures": ["x86_64"],
"categories": ["foo", "foo"],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"\['foo',\s*'foo'\]\s*has\s+non-unique\s+elements"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidMemoryModel(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "trusty",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"'trusty' is not one of"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidVerificationTask(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_super_function": {"correct": False}},
}
self.appendSchemaVersion(s)
msgRegex = r"Additional properties are not allowed \('no_reach_super_function' was unexpected\)"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectVariantDefine(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"variants": {"config1": ["foo=bad value"]},
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"'foo=bad value' does not match"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateWrongDepType(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"verification_tasks": {"no_reach_error_function": {"correct": True}},
"dependencies": ["xxx", "yyy"],
}
self.appendSchemaVersion(s)
msgRegex = r"\['xxx', 'yyy'\] is not of type 'object'"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectBuildVariantName(self):
s = {
"architectures": ["x86_64"],
"categories": [],
"language": "c99",
"memory_model": "precise",
"name": "mybenchmark",
"sources": ["a.c", "b.c"],
"variants": {"bad build variant name": ["FOO=1"]},
"verification_tasks": {"no_reach_error_function": {"correct": True}},
}
self.appendSchemaVersion(s)
msgRegex = r"Additional properties are not allowed \('bad build variant name'"
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s)
with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex):
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testCreateSimple(self):
s = {
'architectures': ['x86_64'],
'categories': ['xxx'],
'language': 'c99',
'name': 'foo',
'sources': ['a_is_a_good_name.c', 'b-IS-also-A-good-name.c'],
'verification_tasks': {
'no_assert_fail': {
'correct': True
}
},
}
self.appendSchemaVersion(s)
# Validate benchmark specification
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
benchmarkObjs = svcb.benchmark.getBenchmarks(
s, addImplicitVerificationTasks=False)
self.assertTrue(len(benchmarkObjs) == 1)
# Test properties of the benchmark object
b = benchmarkObjs[0]
self.assertEqual(b.name, "foo")
self.assertEqual(b.sources,
['a_is_a_good_name.c', 'b-IS-also-A-good-name.c'])
self.assertEqual(b.architectures, ['x86_64'])
self.assertEqual(b.defines, {}) # Implicitly empty
self.assertEqual(b.language, 'c99')
self.assertEqual(b.description, '') # Implicity empty
self.assertEqual(b.categories, {'xxx'})
self.assertEqual(b.verificationTasks,
{'no_assert_fail': {
'correct': True
}})
self.assertTrue(b.isLanguageC())
self.assertFalse(b.isLanguageCXX())
self.assertEqual(b.misc, {}) # Implicitly empty
def testCreateTwoVariants(self):
s = {
'architectures': ['x86_64'],
'categories': ['xxx', 'cheese'],
'description': 'global comment',
'defines': {
'DUMMY': '1'
},
'dependencies': {
'klee_runtime': {}
},
'language': 'c99',
'name': 'basename',
'runtime_environment': {
'command_line_arguments': ['--foo', '--bar'],
'environment_variables': {
'FOO': 'BAR',
'BAZ': 'TWO',
},
},
'sources': ['a.c', 'b.c'],
'misc': {
'dummy': 1
},
'variants': {
'foo': {
'verification_tasks': {
'no_assert_fail': {
'correct': True
}
},
'description': 'This is foo',
'defines': {
'FAIL': '0'
},
'dependencies': {
'pthreads': {}
},
'categories': ['foo_category', 'cheese'],
'runtime_environment': {
'command_line_arguments': ['--baz'],
'environment_variables': {
'TEST': 'THREE'
},
},
},
'bar': {
'verification_tasks': {
'no_assert_fail': {
'correct': False
}
},
'defines': {
'FAIL': '1'
},
'dependencies': {
'openmp': {}
},
'categories': ['bar_category'],
}
}
}
self.appendSchemaVersion(s)
# Validate benchmark specification
schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
benchmarkObjs = svcb.benchmark.getBenchmarks(
s, addImplicitVerificationTasks=False)
self.assertTrue(len(benchmarkObjs) == 2)
# Extract the two benchmarks. The order is not defined so get based on name
fooBenchmark = list(
filter(lambda b: b.name == 'basename_foo', benchmarkObjs))[0]
barBenchmark = list(
filter(lambda b: b.name == 'basename_bar', benchmarkObjs))[0]
# Check foo
self.assertEqual(fooBenchmark.architectures, ['x86_64'])
self.assertEqual(fooBenchmark.categories,
{'cheese', 'foo_category', 'xxx'})
self.assertEqual(fooBenchmark.description,
'global comment\nThis is foo')
self.assertEqual(fooBenchmark.defines, {'DUMMY': '1', 'FAIL': '0'})
self.assertEqual(fooBenchmark.dependencies, {
'klee_runtime': {},
'pthreads': {}
})
self.assertEqual(fooBenchmark.language, 'c99')
self.assertEqual(fooBenchmark.name, 'basename_foo')
self.assertEqual(fooBenchmark.sources, ['a.c', 'b.c'])
self.assertEqual(fooBenchmark.verificationTasks,
{'no_assert_fail': {
'correct': True
}})
self.assertTrue(fooBenchmark.isLanguageC())
self.assertFalse(fooBenchmark.isLanguageCXX())
self.assertEqual(fooBenchmark.misc, {'dummy': 1})
self.assertEqual(
fooBenchmark.runtimeEnvironment['command_line_arguments'],
['--foo', '--bar', '--baz'])
self.assertEqual(
fooBenchmark.runtimeEnvironment['environment_variables'], {
'BAZ': 'TWO',
'FOO': 'BAR',
'TEST': 'THREE'
})
# Check bar
self.assertEqual(barBenchmark.architectures, ['x86_64'])
self.assertEqual(barBenchmark.categories,
{'bar_category', 'cheese', 'xxx'})
self.assertEqual(barBenchmark.description, 'global comment')
self.assertEqual(barBenchmark.defines, {'DUMMY': '1', 'FAIL': '1'})
self.assertEqual(barBenchmark.dependencies, {
'klee_runtime': {},
'openmp': {}
})
self.assertEqual(barBenchmark.language, 'c99')
self.assertEqual(barBenchmark.name, 'basename_bar')
self.assertEqual(barBenchmark.sources, ['a.c', 'b.c'])
self.assertEqual(
barBenchmark.verificationTasks, {
'no_assert_fail': {
'correct': False,
'exhaustive_counter_examples': False
}
})
self.assertTrue(barBenchmark.isLanguageC())
self.assertFalse(barBenchmark.isLanguageCXX())
self.assertEqual(barBenchmark.misc, {'dummy': 1})
self.assertEqual(
barBenchmark.runtimeEnvironment['command_line_arguments'],
['--foo', '--bar'])
self.assertEqual(
barBenchmark.runtimeEnvironment['environment_variables'], {
'BAZ': 'TWO',
'FOO': 'BAR'
})
