def test_ResultIsInteresting_unknown(): """An unknown outcome is not interesting.""" gs_harness = MockHarness() filters = MockFilters() result = opencl_fuzz.ResultIsInteresting( deepsmith_pb2.Result(outcome=deepsmith_pb2.Result.UNKNOWN), difftests.UnaryTester(), difftests.GoldStandardDiffTester( difftests.NamedOutputIsEqual('stdout')), gs_harness, filters) assert not result # Only the unary tester was called, no differential test was required. assert not gs_harness.RunTestcases_call_requests assert len(filters.PreDifftest_call_args) == 0
def test_ResultIsInteresting_build_timeout(): """A build timeout is interesting.""" gs_harness = MockHarness() filters = MockFilters() result = opencl_fuzz.ResultIsInteresting( deepsmith_pb2.Result(outcome=deepsmith_pb2.Result.BUILD_TIMEOUT), difftests.UnaryTester(), difftests.GoldStandardDiffTester( difftests.NamedOutputIsEqual('stdout')), gs_harness, filters) assert result assert result.outputs['difftest_outcome'] == 'ANOMALOUS_BUILD_FAILURE' # Only the unary tester was called, no differential test was required. assert not gs_harness.RunTestcases_call_requests assert len(filters.PreDifftest_call_args) == 0
def test_ResultIsInteresting_build_crash(): """A build crash is interesting.""" gs_harness = MockHarness() filters = MockFilters() result = opencl_fuzz.ResultIsInteresting( deepsmith_pb2.Result(outcome=deepsmith_pb2.Result.BUILD_CRASH), difftests.UnaryTester(), difftests.GoldStandardDiffTester(difftests.NamedOutputIsEqual("stdout")), gs_harness, filters, ) assert result assert result.outputs["difftest_outcome"] == "ANOMALOUS_BUILD_FAILURE" # Only the unary tester was called, no differential test was required. assert not gs_harness.RunTestcases_call_requests assert len(filters.PreDifftest_call_args) == 0
def RunBatch( generator: base_generator.GeneratorServiceBase, dut_harness: base_harness.HarnessBase, gs_harness: base_harness.HarnessBase, filters: difftests.FiltersBase, batch_size: int, ) -> typing.List[deepsmith_pb2.Result]: """Run one batch of testing. A batch of testing involves generating a set of testcases, executing them on the device under test, then determining for each whether the result is interesting. The interesting-ness test may involve executing testcases on the gold-standard device and comparing the outputs. Args: generator: The generator for testcases. dut_harness: The device under test. gs_harness: The gold-standard device, used to compare outputs against the device under test. filters: A testcase filters instance. batch_size: The number of testcases to generate and evaluate. Returns: A list of results which are determined to be interesting. """ # Our differential testers and result filters. unary_difftester = difftests.UnaryTester() gs_difftester = difftests.GoldStandardDiffTester( difftests.NamedOutputIsEqual("stdout") ) interesting_results = [] # Generate testcases. app.Log(1, "Generating %d testcases ...", batch_size) req = generator_pb2.GenerateTestcasesRequest() req.num_testcases = batch_size res = generator.GenerateTestcases(req, None) testcases = [ testcase for testcase in res.testcases if filters.PreExec(testcase) ] if len(res.testcases) - len(testcases): app.Log( 1, "Discarded %d testcases prior to execution.", len(res.testcases) - len(testcases), ) # Evaluate testcases. app.Log( 1, "Evaluating %d testcases on %s ...", len(testcases), dut_harness.testbeds[0].opts["platform"][:12], ) unfiltered_results = RunTestcases(dut_harness, testcases) results = [ result for result in unfiltered_results if filters.PostExec(result) ] if len(unfiltered_results) - len(results): app.Log(1, "Discarded %d results.", len(unfiltered_results) - len(results)) for i, result in enumerate(results): interesting_result = ResultIsInteresting( result, unary_difftester, gs_difftester, gs_harness, filters ) if interesting_result: interesting_results.append(interesting_result) return interesting_results