def SetProto(self, proto: deepsmith_pb2.Result) -> deepsmith_pb2.Result:
"""Set a protocol buffer representation.
Args:
proto: A protocol buffer message.
Returns:
A Result message.
"""
self.testcase.SetProto(proto.testcase)
self.testbed.SetProto(proto.testbed)
proto.returncode = self.returncode
for output in self.outputset:
proto.outputs[output.name.string] = output.value.truncated_value
for event in self.profiling_events:
event_proto = proto.profiling_events.add()
event.SetProto(event_proto)
proto.outcome = self.outcome_num
return proto
def GetResultOutcome(
result: deepsmith_pb2.Result) -> deepsmith_pb2.Result.Outcome:
"""Determine the output class of a result.
Args:
result: The result to determine the output class of.
Returns:
The result outcome.
Raises:
ValueError: If the outcome class could not be determined.
"""
def RuntimeCrashOrBuildCrash():
if "Compilation terminated successfully..." in result.outputs[
'stderr']:
return deepsmith_pb2.Result.RUNTIME_CRASH
else:
return deepsmith_pb2.Result.BUILD_CRASH
def RuntimeTimoutOrBuildTimeout():
if "Compilation terminated successfully..." in result.outputs[
'stderr']:
return deepsmith_pb2.Result.RUNTIME_TIMEOUT
else:
return deepsmith_pb2.Result.BUILD_TIMEOUT
runtime_ms = GetResultRuntimeMs(result)
timeout_ms = int(result.testcase.harness.opts.get('timeout_seconds',
60)) * 1000
if result.returncode == 0:
return deepsmith_pb2.Result.PASS
elif result.returncode == 139 or result.returncode == -11:
# 139 is SIGSEV
result.returncode = 139
return RuntimeCrashOrBuildCrash()
elif result.returncode == -5:
# SIGTRAP
return RuntimeCrashOrBuildCrash()
elif result.returncode == -9 and runtime_ms >= timeout_ms:
# SIGKILL
return RuntimeTimoutOrBuildTimeout()
elif result.returncode == -9:
logging.warning('SIGKILL, but only ran for %d ms', runtime_ms)
return RuntimeCrashOrBuildCrash()
elif result.returncode == -4:
# SIGILL
return RuntimeCrashOrBuildCrash()
elif result.returncode == -6:
# SIGABRT
return RuntimeCrashOrBuildCrash()
elif result.returncode == -8:
# SIGFPE
return RuntimeCrashOrBuildCrash()
elif result.returncode == -7:
# SIGBUS
return RuntimeCrashOrBuildCrash()
elif (result.returncode == 1
and 'Error building program:' in result.outputs['stderr']):
return deepsmith_pb2.Result.BUILD_FAILURE
elif result.returncode == 1:
# cl_launcher error
return deepsmith_pb2.Result.UNKNOWN
raise ValueError(f'Failed to output class of result: {result}')
def ResultIsInteresting(
result: deepsmith_pb2.Result,
unary_difftester: difftests.UnaryTester,
gs_difftester: difftests.GoldStandardDiffTester,
gs_harness: base_harness.HarnessBase,
filters: difftests.FiltersBase,
) -> typing.Optional[deepsmith_pb2.Result]:
"""Determine if a result is interesting, and return it if it is.
Args:
result: The result to test.
unary_difftester: A unary difftester.
gs_difftester: A golden standard difftester.
gs_harness: A golden standard test harness.
filters: A set of difftest filters.
Returns:
The result if it is interesting, else None.
"""
# First perform a unary difftest to see if the result is interesting without
# needing to difftest, such as a compiler crash.
unary_dt_outcome = unary_difftester([result])[0]
if (
unary_dt_outcome != deepsmith_pb2.DifferentialTest.PASS
and unary_dt_outcome != deepsmith_pb2.DifferentialTest.UNKNOWN
):
result.outputs[
"difftest_outcome"
] = deepsmith_pb2.DifferentialTest.Outcome.Name(unary_dt_outcome)
return result
if not (
unary_dt_outcome == deepsmith_pb2.DifferentialTest.PASS
and result.outcome == deepsmith_pb2.Result.PASS
):
return NotInteresting(result)
# Determine whether we can difftest the testcase.
dt = filters.PreDifftest(deepsmith_pb2.DifferentialTest(result=[result]))
if not dt:
return NotInteresting(result)
result = dt.result[0]
# Run testcases against gold standard devices and difftest.
gs_result = RunTestcases(gs_harness, [result.testcase])[0]
dt_outcomes = gs_difftester([gs_result, result])
dt_outcome = dt_outcomes[1]
app.Log(
1,
"Differential test outcome: %s.",
deepsmith_pb2.DifferentialTest.Outcome.Name(dt_outcome),
)
# Determine whether we can use the difftest result.
dt = filters.PostDifftest(
deepsmith_pb2.DifferentialTest(
result=[gs_result, result], outcome=dt_outcomes
)
)
if not dt:
app.Log(1, "Cannot use gold standard difftest result.")
return NotInteresting(result)
result = dt.result[1]
if dt_outcome != deepsmith_pb2.DifferentialTest.PASS:
# Add the differential test outcome to the result.
result.outputs[
"difftest_outcome"
] = deepsmith_pb2.DifferentialTest.Outcome.Name(dt_outcome)
result.outputs["gs_stdout"] = dt.result[0].outputs["stdout"]
result.outputs["gs_stderr"] = dt.result[0].outputs["stderr"]
return result
return NotInteresting(result)