def testValidateSimpleAnyArch(self): s = { "architectures": "any", "categories": [], "language": "c99", "memory_model": "precise", "name": "foo", "sources": ["a_is_a_good_name.c", "b-IS-also-A-good-name.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) schema.validateBenchmarkSpecification(s) schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateUnknownCorrectness(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": None}}, } self.appendSchemaVersion(s) schema.validateBenchmarkSpecification(s) schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateSimpleWithVariants(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "name": "mybenchmark", "memory_model": "precise", "sources": ["a.c", "b.c"], "variants": {"config1": ["FOO" "BAR=BAZ", "NUM=0"], "config2": ["NUM=1"]}, "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) schema.validateBenchmarkSpecification(s) schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testMissingCategories(self): s = { "architectures": ["x86_64"], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"'categories' is a required property" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateMissingVerificationTask(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], } self.appendSchemaVersion(s) msgRegex = r"'verification_tasks' is a required property" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testCreateSimpleWithImplicitVerficationTasksWithCex(self): s = { 'architectures': ['x86_64'], 'categories': ['xxx'], 'language': 'c99', 'name': 'foo', 'sources': ['a_is_a_good_name.c', 'b-IS-also-A-good-name.c'], 'verification_tasks': { 'no_assert_fail': { 'correct': False, 'counter_examples': [ { "locations": [{ "line": 1, "file": "a_is_a_good_name.c", }] }, ] } }, } self.appendSchemaVersion(s) # Validate benchmark specification schema.validateBenchmarkSpecification(s, schema=self.persistentSchema) benchmarkObjs = svcb.benchmark.getBenchmarks( s) # Default addImplicitVerificationTasks=True self.assertTrue(len(benchmarkObjs) == 1) # Test properties of the benchmark object b = benchmarkObjs[0] self.assertEqual(b.name, "foo") self.assertEqual(b.sources, ['a_is_a_good_name.c', 'b-IS-also-A-good-name.c']) self.assertEqual(b.architectures, ['x86_64']) self.assertEqual(b.defines, {}) # Implicitly empty self.assertEqual(b.language, 'c99') self.assertEqual(b.description, '') # Implicity empty self.assertEqual(b.categories, {'xxx'}) self.assertTrue(b.isLanguageC()) self.assertFalse(b.isLanguageCXX()) expectedTasks = copy.deepcopy( svcb.benchmark.DefaultVerificationTaskStatuses) expectedTasks['no_assert_fail'] = copy.deepcopy( s['verification_tasks']['no_assert_fail']) expectedTasks['no_assert_fail'][ 'exhaustive_counter_examples'] = True # Expected implicitly added field self.assertEqual(b.verificationTasks, expectedTasks)
def testValidateEmptyVerificationTask(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {}, } self.appendSchemaVersion(s) msgRegex = r"Failed validating 'minProperties' in schema\['properties'\]\['verification_tasks'\]" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateEmptySources(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": [], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"Failed validating 'minItems' in schema\['properties'\]\['sources'\]" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectRelativeSourcePath(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["../a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"not allowed for '\.\./a.c'" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectSchemaVersion(self): s = { "architecture": "foo", "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, "schema_version": 123456, } msgRegex = r"Schema version used by benchmark \(\d+\) does not match the currently support schema \(\d+\)" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidCorrectnessType(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": "XXX"}}, } self.appendSchemaVersion(s) msgRegex = r"'XXX' is not valid under any of the given schemas" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidMixAnyAndOtherArch(self): s = { "architectures": ["any", "x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "foo", "sources": ["a_is_a_good_name.c", "b-IS-also-A-good-name.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"\['any', 'x86_64'\] is not valid under any of the given schemas" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidatePthreadsDepWithVersion(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "foo", "sources": ["a_is_a_good_name.c", "b-IS-also-A-good-name.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, "dependencies": { "pthreads": {"version": "4"} }, # Should we enforce what the version string should look like? } self.appendSchemaVersion(s) schema.validateBenchmarkSpecification(s) schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testDuplicateCategories(self): s = { "architectures": ["x86_64"], "categories": ["foo", "foo"], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"\['foo',\s*'foo'\]\s*has\s+non-unique\s+elements" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidMemoryModel(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "trusty", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"'trusty' is not one of" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateInvalidVerificationTask(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_super_function": {"correct": False}}, } self.appendSchemaVersion(s) msgRegex = r"Additional properties are not allowed \('no_reach_super_function' was unexpected\)" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectVariantDefine(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "variants": {"config1": ["foo=bad value"]}, "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"'foo=bad value' does not match" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateWrongDepType(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "verification_tasks": {"no_reach_error_function": {"correct": True}}, "dependencies": ["xxx", "yyy"], } self.appendSchemaVersion(s) msgRegex = r"\['xxx', 'yyy'\] is not of type 'object'" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testValidateIncorrectBuildVariantName(self): s = { "architectures": ["x86_64"], "categories": [], "language": "c99", "memory_model": "precise", "name": "mybenchmark", "sources": ["a.c", "b.c"], "variants": {"bad build variant name": ["FOO=1"]}, "verification_tasks": {"no_reach_error_function": {"correct": True}}, } self.appendSchemaVersion(s) msgRegex = r"Additional properties are not allowed \('bad build variant name'" with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s) with self.assertRaisesRegex(schema.BenchmarkSpecificationValidationError, msgRegex): schema.validateBenchmarkSpecification(s, schema=self.persistentSchema)
def testCreateSimple(self): s = { 'architectures': ['x86_64'], 'categories': ['xxx'], 'language': 'c99', 'name': 'foo', 'sources': ['a_is_a_good_name.c', 'b-IS-also-A-good-name.c'], 'verification_tasks': { 'no_assert_fail': { 'correct': True } }, } self.appendSchemaVersion(s) # Validate benchmark specification schema.validateBenchmarkSpecification(s, schema=self.persistentSchema) benchmarkObjs = svcb.benchmark.getBenchmarks( s, addImplicitVerificationTasks=False) self.assertTrue(len(benchmarkObjs) == 1) # Test properties of the benchmark object b = benchmarkObjs[0] self.assertEqual(b.name, "foo") self.assertEqual(b.sources, ['a_is_a_good_name.c', 'b-IS-also-A-good-name.c']) self.assertEqual(b.architectures, ['x86_64']) self.assertEqual(b.defines, {}) # Implicitly empty self.assertEqual(b.language, 'c99') self.assertEqual(b.description, '') # Implicity empty self.assertEqual(b.categories, {'xxx'}) self.assertEqual(b.verificationTasks, {'no_assert_fail': { 'correct': True }}) self.assertTrue(b.isLanguageC()) self.assertFalse(b.isLanguageCXX()) self.assertEqual(b.misc, {}) # Implicitly empty
def testCreateTwoVariants(self): s = { 'architectures': ['x86_64'], 'categories': ['xxx', 'cheese'], 'description': 'global comment', 'defines': { 'DUMMY': '1' }, 'dependencies': { 'klee_runtime': {} }, 'language': 'c99', 'name': 'basename', 'runtime_environment': { 'command_line_arguments': ['--foo', '--bar'], 'environment_variables': { 'FOO': 'BAR', 'BAZ': 'TWO', }, }, 'sources': ['a.c', 'b.c'], 'misc': { 'dummy': 1 }, 'variants': { 'foo': { 'verification_tasks': { 'no_assert_fail': { 'correct': True } }, 'description': 'This is foo', 'defines': { 'FAIL': '0' }, 'dependencies': { 'pthreads': {} }, 'categories': ['foo_category', 'cheese'], 'runtime_environment': { 'command_line_arguments': ['--baz'], 'environment_variables': { 'TEST': 'THREE' }, }, }, 'bar': { 'verification_tasks': { 'no_assert_fail': { 'correct': False } }, 'defines': { 'FAIL': '1' }, 'dependencies': { 'openmp': {} }, 'categories': ['bar_category'], } } } self.appendSchemaVersion(s) # Validate benchmark specification schema.validateBenchmarkSpecification(s, schema=self.persistentSchema) benchmarkObjs = svcb.benchmark.getBenchmarks( s, addImplicitVerificationTasks=False) self.assertTrue(len(benchmarkObjs) == 2) # Extract the two benchmarks. The order is not defined so get based on name fooBenchmark = list( filter(lambda b: b.name == 'basename_foo', benchmarkObjs))[0] barBenchmark = list( filter(lambda b: b.name == 'basename_bar', benchmarkObjs))[0] # Check foo self.assertEqual(fooBenchmark.architectures, ['x86_64']) self.assertEqual(fooBenchmark.categories, {'cheese', 'foo_category', 'xxx'}) self.assertEqual(fooBenchmark.description, 'global comment\nThis is foo') self.assertEqual(fooBenchmark.defines, {'DUMMY': '1', 'FAIL': '0'}) self.assertEqual(fooBenchmark.dependencies, { 'klee_runtime': {}, 'pthreads': {} }) self.assertEqual(fooBenchmark.language, 'c99') self.assertEqual(fooBenchmark.name, 'basename_foo') self.assertEqual(fooBenchmark.sources, ['a.c', 'b.c']) self.assertEqual(fooBenchmark.verificationTasks, {'no_assert_fail': { 'correct': True }}) self.assertTrue(fooBenchmark.isLanguageC()) self.assertFalse(fooBenchmark.isLanguageCXX()) self.assertEqual(fooBenchmark.misc, {'dummy': 1}) self.assertEqual( fooBenchmark.runtimeEnvironment['command_line_arguments'], ['--foo', '--bar', '--baz']) self.assertEqual( fooBenchmark.runtimeEnvironment['environment_variables'], { 'BAZ': 'TWO', 'FOO': 'BAR', 'TEST': 'THREE' }) # Check bar self.assertEqual(barBenchmark.architectures, ['x86_64']) self.assertEqual(barBenchmark.categories, {'bar_category', 'cheese', 'xxx'}) self.assertEqual(barBenchmark.description, 'global comment') self.assertEqual(barBenchmark.defines, {'DUMMY': '1', 'FAIL': '1'}) self.assertEqual(barBenchmark.dependencies, { 'klee_runtime': {}, 'openmp': {} }) self.assertEqual(barBenchmark.language, 'c99') self.assertEqual(barBenchmark.name, 'basename_bar') self.assertEqual(barBenchmark.sources, ['a.c', 'b.c']) self.assertEqual( barBenchmark.verificationTasks, { 'no_assert_fail': { 'correct': False, 'exhaustive_counter_examples': False } }) self.assertTrue(barBenchmark.isLanguageC()) self.assertFalse(barBenchmark.isLanguageCXX()) self.assertEqual(barBenchmark.misc, {'dummy': 1}) self.assertEqual( barBenchmark.runtimeEnvironment['command_line_arguments'], ['--foo', '--bar']) self.assertEqual( barBenchmark.runtimeEnvironment['environment_variables'], { 'BAZ': 'TWO', 'FOO': 'BAR' })