def evaluate_result(self, result: Result): if result.timeout: result.result = RESULT_TIMEOUT elif result.exited: if result.return_code == 0: if self.query in (TPCH1, TPCH3, SSB11, SSB41): answer_file = f'databases/{self.db}/answers' if self.query == TPCH1: answer_file += '/q1.out' elif self.query == TPCH3: answer_file += '/q3.out' elif self.query == SSB11: answer_file += '/ssb11.out' elif self.query == SSB41: answer_file += '/ssb41.out' if self.compare_output(answer_file, result.stdout): result.result = RESULT_OK result.stdout = None else: result.result = RESULT_INCORRECT_OUTPUT else: result.result = RESULT_ABNORMAL_BEHAVIOR elif result.signaled: result.result = RESULT_CRASH
def evaluate_result(self, result: Result): super().evaluate_result(result) if self.query == TPCH_UPDATES: db_file = os.path.join(self.directory, f'db.{self.iteration}.sqlite') db_hash = subprocess.check_output(['sha1sum', db_file ]).decode('utf-8').split(' ')[0] if db_hash == SHA_UPDATED: result.result = RESULT_OK return elif db_hash != SHA_ORIGINAL and (result.result is None or result.result < 0): result.result = RESULT_INCORRECT_OUTPUT # check for corruption p = subprocess.Popen([ os.path.join(self.database_dir, 'bin/sqlite3'), db_file, 'pragma integrity_check' ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) integrity_check, integrity_stderr = p.communicate() if len(integrity_stderr) > 0: if result.stderr is None or len(result.stderr) == 0: result.stderr = b'[INTEGRITY]:' + integrity_stderr else: result.stderr += b'\n[INTEGRITY]:' + integrity_stderr if p.returncode != 0: result.db_corrupted = True return integrity_check = integrity_check.decode('utf-8').strip() if integrity_check != 'ok': result.db_corrupted = True return
def evaluate_result(self, result: Result): if result.timeout: result.result = RESULT_TIMEOUT elif result.signaled: if result.term_sig == 9: answer_file = None if self.query == TPCH1: answer_file = 'databases/monetdb/answers/q1.out' elif self.query == TPCH3: answer_file = 'databases/monetdb/answers/q3.out' if self.compare_output(answer_file, result.stdout): result.result = RESULT_OK result.stdout = None else: result.result = RESULT_INCORRECT_OUTPUT else: result.result = RESULT_CRASH elif self.query_return_code != 0: result.result = RESULT_ABNORMAL_BEHAVIOR else: result.result = RESULT_UNKNOWN