def ResultIsInteresting(
result: deepsmith_pb2.Result,
unary_difftester: difftests.UnaryTester,
gs_difftester: difftests.GoldStandardDiffTester,
gs_harness: base_harness.HarnessBase,
filters: difftests.FiltersBase,
) -> typing.Optional[deepsmith_pb2.Result]:
"""Determine if a result is interesting, and return it if it is.
Args:
result: The result to test.
unary_difftester: A unary difftester.
gs_difftester: A golden standard difftester.
gs_harness: A golden standard test harness.
filters: A set of difftest filters.
Returns:
The result if it is interesting, else None.
"""
# First perform a unary difftest to see if the result is interesting without
# needing to difftest, such as a compiler crash.
unary_dt_outcome = unary_difftester([result])[0]
if (
unary_dt_outcome != deepsmith_pb2.DifferentialTest.PASS
and unary_dt_outcome != deepsmith_pb2.DifferentialTest.UNKNOWN
):
result.outputs[
"difftest_outcome"
] = deepsmith_pb2.DifferentialTest.Outcome.Name(unary_dt_outcome)
return result
if not (
unary_dt_outcome == deepsmith_pb2.DifferentialTest.PASS
and result.outcome == deepsmith_pb2.Result.PASS
):
return NotInteresting(result)
# Determine whether we can difftest the testcase.
dt = filters.PreDifftest(deepsmith_pb2.DifferentialTest(result=[result]))
if not dt:
return NotInteresting(result)
result = dt.result[0]
# Run testcases against gold standard devices and difftest.
gs_result = RunTestcases(gs_harness, [result.testcase])[0]
dt_outcomes = gs_difftester([gs_result, result])
dt_outcome = dt_outcomes[1]
app.Log(
1,
"Differential test outcome: %s.",
deepsmith_pb2.DifferentialTest.Outcome.Name(dt_outcome),
)
# Determine whether we can use the difftest result.
dt = filters.PostDifftest(
deepsmith_pb2.DifferentialTest(
result=[gs_result, result], outcome=dt_outcomes
)
)
if not dt:
app.Log(1, "Cannot use gold standard difftest result.")
return NotInteresting(result)
result = dt.result[1]
if dt_outcome != deepsmith_pb2.DifferentialTest.PASS:
# Add the differential test outcome to the result.
result.outputs[
"difftest_outcome"
] = deepsmith_pb2.DifferentialTest.Outcome.Name(dt_outcome)
result.outputs["gs_stdout"] = dt.result[0].outputs["stdout"]
result.outputs["gs_stderr"] = dt.result[0].outputs["stderr"]
return result
return NotInteresting(result)
def RunBatch(
generator: base_generator.GeneratorServiceBase,
dut_harness: base_harness.HarnessBase,
gs_harness: base_harness.HarnessBase,
filters: difftests.FiltersBase,
batch_size: int,
) -> typing.List[deepsmith_pb2.Result]:
"""Run one batch of testing.
A batch of testing involves generating a set of testcases, executing them on
the device under test, then determining for each whether the result is
interesting. The interesting-ness test may involve executing testcases on the
gold-standard device and comparing the outputs.
Args:
generator: The generator for testcases.
dut_harness: The device under test.
gs_harness: The gold-standard device, used to compare outputs against the
device under test.
filters: A testcase filters instance.
batch_size: The number of testcases to generate and evaluate.
Returns:
A list of results which are determined to be interesting.
"""
# Our differential testers and result filters.
unary_difftester = difftests.UnaryTester()
gs_difftester = difftests.GoldStandardDiffTester(
difftests.NamedOutputIsEqual("stdout")
)
interesting_results = []
# Generate testcases.
app.Log(1, "Generating %d testcases ...", batch_size)
req = generator_pb2.GenerateTestcasesRequest()
req.num_testcases = batch_size
res = generator.GenerateTestcases(req, None)
testcases = [
testcase for testcase in res.testcases if filters.PreExec(testcase)
]
if len(res.testcases) - len(testcases):
app.Log(
1,
"Discarded %d testcases prior to execution.",
len(res.testcases) - len(testcases),
)
# Evaluate testcases.
app.Log(
1,
"Evaluating %d testcases on %s ...",
len(testcases),
dut_harness.testbeds[0].opts["platform"][:12],
)
unfiltered_results = RunTestcases(dut_harness, testcases)
results = [
result for result in unfiltered_results if filters.PostExec(result)
]
if len(unfiltered_results) - len(results):
app.Log(1, "Discarded %d results.", len(unfiltered_results) - len(results))
for i, result in enumerate(results):
interesting_result = ResultIsInteresting(
result, unary_difftester, gs_difftester, gs_harness, filters
)
if interesting_result:
interesting_results.append(interesting_result)
return interesting_results