def evaluate_result(self, result: Result):
if result.timeout:
result.result = RESULT_TIMEOUT
elif result.exited:
if result.return_code == 0:
if self.query in (TPCH1, TPCH3, SSB11, SSB41):
answer_file = f'databases/{self.db}/answers'
if self.query == TPCH1:
answer_file += '/q1.out'
elif self.query == TPCH3:
answer_file += '/q3.out'
elif self.query == SSB11:
answer_file += '/ssb11.out'
elif self.query == SSB41:
answer_file += '/ssb41.out'
if self.compare_output(answer_file, result.stdout):
result.result = RESULT_OK
result.stdout = None
else:
result.result = RESULT_INCORRECT_OUTPUT
else:
result.result = RESULT_ABNORMAL_BEHAVIOR
elif result.signaled:
result.result = RESULT_CRASH
def evaluate_result(self, result: Result):
if result.timeout:
result.result = RESULT_TIMEOUT
elif result.signaled:
if result.term_sig == 9:
answer_file = None
if self.query == TPCH1:
answer_file = 'databases/monetdb/answers/q1.out'
elif self.query == TPCH3:
answer_file = 'databases/monetdb/answers/q3.out'
if self.compare_output(answer_file, result.stdout):
result.result = RESULT_OK
result.stdout = None
else:
result.result = RESULT_INCORRECT_OUTPUT
else:
result.result = RESULT_CRASH
elif self.query_return_code != 0:
result.result = RESULT_ABNORMAL_BEHAVIOR
else:
result.result = RESULT_UNKNOWN