def test_ResultIsInteresting_unknown():
"""An unknown outcome is not interesting."""
gs_harness = MockHarness()
filters = MockFilters()
result = opencl_fuzz.ResultIsInteresting(
deepsmith_pb2.Result(outcome=deepsmith_pb2.Result.UNKNOWN),
difftests.UnaryTester(),
difftests.GoldStandardDiffTester(
difftests.NamedOutputIsEqual('stdout')), gs_harness, filters)
assert not result
# Only the unary tester was called, no differential test was required.
assert not gs_harness.RunTestcases_call_requests
assert len(filters.PreDifftest_call_args) == 0
def test_ResultIsInteresting_build_timeout():
"""A build timeout is interesting."""
gs_harness = MockHarness()
filters = MockFilters()
result = opencl_fuzz.ResultIsInteresting(
deepsmith_pb2.Result(outcome=deepsmith_pb2.Result.BUILD_TIMEOUT),
difftests.UnaryTester(),
difftests.GoldStandardDiffTester(
difftests.NamedOutputIsEqual('stdout')), gs_harness, filters)
assert result
assert result.outputs['difftest_outcome'] == 'ANOMALOUS_BUILD_FAILURE'
# Only the unary tester was called, no differential test was required.
assert not gs_harness.RunTestcases_call_requests
assert len(filters.PreDifftest_call_args) == 0
def test_ResultIsInteresting_build_crash():
"""A build crash is interesting."""
gs_harness = MockHarness()
filters = MockFilters()
result = opencl_fuzz.ResultIsInteresting(
deepsmith_pb2.Result(outcome=deepsmith_pb2.Result.BUILD_CRASH),
difftests.UnaryTester(),
difftests.GoldStandardDiffTester(difftests.NamedOutputIsEqual("stdout")),
gs_harness,
filters,
)
assert result
assert result.outputs["difftest_outcome"] == "ANOMALOUS_BUILD_FAILURE"
# Only the unary tester was called, no differential test was required.
assert not gs_harness.RunTestcases_call_requests
assert len(filters.PreDifftest_call_args) == 0
def RunBatch(
generator: base_generator.GeneratorServiceBase,
dut_harness: base_harness.HarnessBase,
gs_harness: base_harness.HarnessBase,
filters: difftests.FiltersBase,
batch_size: int,
) -> typing.List[deepsmith_pb2.Result]:
"""Run one batch of testing.
A batch of testing involves generating a set of testcases, executing them on
the device under test, then determining for each whether the result is
interesting. The interesting-ness test may involve executing testcases on the
gold-standard device and comparing the outputs.
Args:
generator: The generator for testcases.
dut_harness: The device under test.
gs_harness: The gold-standard device, used to compare outputs against the
device under test.
filters: A testcase filters instance.
batch_size: The number of testcases to generate and evaluate.
Returns:
A list of results which are determined to be interesting.
"""
# Our differential testers and result filters.
unary_difftester = difftests.UnaryTester()
gs_difftester = difftests.GoldStandardDiffTester(
difftests.NamedOutputIsEqual("stdout")
)
interesting_results = []
# Generate testcases.
app.Log(1, "Generating %d testcases ...", batch_size)
req = generator_pb2.GenerateTestcasesRequest()
req.num_testcases = batch_size
res = generator.GenerateTestcases(req, None)
testcases = [
testcase for testcase in res.testcases if filters.PreExec(testcase)
]
if len(res.testcases) - len(testcases):
app.Log(
1,
"Discarded %d testcases prior to execution.",
len(res.testcases) - len(testcases),
)
# Evaluate testcases.
app.Log(
1,
"Evaluating %d testcases on %s ...",
len(testcases),
dut_harness.testbeds[0].opts["platform"][:12],
)
unfiltered_results = RunTestcases(dut_harness, testcases)
results = [
result for result in unfiltered_results if filters.PostExec(result)
]
if len(unfiltered_results) - len(results):
app.Log(1, "Discarded %d results.", len(unfiltered_results) - len(results))
for i, result in enumerate(results):
interesting_result = ResultIsInteresting(
result, unary_difftester, gs_difftester, gs_harness, filters
)
if interesting_result:
interesting_results.append(interesting_result)
return interesting_results
