def compare_test_run(test, idx, iss, output_dir, report): '''Compare results for a single run of a single test Here, test is a dictionary describing the test (read from the testlist YAML file). idx is the iteration index. iss is the chosen instruction set simulator (currently supported: spike and ovpsim). output_dir is the base output directory (which should contain logs from both the ISS and the test run itself). report is the path to the regression report file we're writing. Returns True if the test run passed and False otherwise. ''' # TODO [Haroon]: Support directed asm and directed_c tests post-sim comparison test_name = test['test'] elf = os.path.join(output_dir, 'instr_gen/asm_tests/{}.{}.o'.format(test_name, idx)) logging.info("Comparing %s/DUT sim result : %s" % (iss, elf)) with open(report, 'a') as report_fd: report_fd.write('Test binary: {}\n'.format(elf)) rtl_dir = os.path.join(output_dir, 'rtl_sim', '{}.{}'.format(test_name, idx)) nb_log = os.path.join(rtl_dir, 'trace_core_nb_load.log') rtl_log = os.path.join(rtl_dir, 'trace_core_00000000.log') rtl_log_f = os.path.join(rtl_dir, 'trace_core.log') rtl_csv = os.path.join(rtl_dir, 'trace_core_00000000.csv') uvm_log = os.path.join(rtl_dir, 'sim.log') if not os.path.exists(rtl_log): with open(report, 'a') as report_fd: report_fd.write('{} does not exist\n'.format(rtl_log)) return False try: # Fix RTL log file with non blocking load values nb_post_fix(rtl_log_f, rtl_log, nb_log) except RuntimeError as e: with open(report, 'a') as report_fd: report_fd.write('Post-fixing of Log failed: {}\n'.format(e)) return False try: # Convert the RTL log file to a trace CSV. process_core_sim_log(rtl_log_f, rtl_csv, 1) except RuntimeError as e: with open(report, 'a') as report_fd: report_fd.write('Log processing failed: {}\n'.format(e)) return False if not os.path.exists(rtl_csv): with open(report, 'a') as report_fd: report_fd.write('{} does not exist\n'.format(rtl_csv)) return False # Have a look at the UVM log. We should write out a message on failure or # if we are stopping at this point. no_post_compare = test.get('no_post_compare') # if not check_core_uvm_log(uvm_log, "core", test_name, report, # write=(True if no_post_compare else 'onfail')): # return False if no_post_compare: return True # There were no UVM errors. Process the log file from the ISS. iss_dir = os.path.join(output_dir, 'instr_gen', '{}_sim'.format(iss)) iss_log = os.path.join(iss_dir, '{}.{}.log'.format(test_name, idx)) iss_csv = os.path.join(iss_dir, '{}.{}.csv'.format(test_name, idx)) if not os.path.exists(iss_log): with open(report, 'a') as report_fd: report_fd.write('{} does not exist\n'.format(iss_log)) return False if iss == "spike": process_spike_sim_log(iss_log, iss_csv) else: assert iss == 'ovpsim' # (should be checked by argparse) process_ovpsim_sim_log(iss_log, iss_csv) if not os.path.exists(iss_csv): with open(report, 'a') as report_fd: report_fd.write('{} does not exist\n'.format(iss_csv)) return False compare_result = \ compare_trace_csv(iss_csv, rtl_csv, iss, "core", report, **test.get('compare_opts', {})) # Rather oddly, compare_result is a string. The comparison passed if it # starts with '[PASSED]'. return compare_result.startswith('[PASSED]')
def compare_test_run(test, idx, iss, output_dir, report): '''Compare results for a single run of a single test Here, test is a dictionary describing the test (read from the testlist YAML file). idx is the iteration index. iss is the chosen instruction set simulator (currently supported: spike and ovpsim). output_dir is the base output directory (which should contain logs from both the ISS and the test run itself). report is the path to the regression report file we're writing. Returns True if the test run passed and False otherwise. ''' test_name = test['test'] elf = os.path.join(output_dir, 'instr_gen/asm_tests/{}.{}.o'.format(test_name, idx)) logging.info("Comparing %s/DUT sim result : %s" % (iss, elf)) with open(report, 'a') as report_fd: report_fd.write('Test binary: {}\n'.format(elf)) rtl_dir = os.path.join(output_dir, 'rtl_sim', '{}.{}'.format(test_name, idx)) rtl_log = os.path.join(rtl_dir, 'trace_core_00000000.log') rtl_csv = os.path.join(rtl_dir, 'trace_core_00000000.csv') uvm_log = os.path.join(rtl_dir, 'sim.log') # Convert the RTL log file to a trace CSV. On failure, this will raise an # exception, so we can assume this passed. process_ibex_sim_log(rtl_log, rtl_csv, 1) # Have a look at the UVM log. We should write out a message on failure or # if we are stopping at this point. no_post_compare = test.get('no_post_compare') if not check_ibex_uvm_log(uvm_log, "ibex", test_name, report, write=(True if no_post_compare else 'onfail')): return False if no_post_compare: return True # There were no UVM errors. Process the log file from the ISS. iss_dir = os.path.join(output_dir, 'instr_gen', '{}_sim'.format(iss)) iss_log = os.path.join(iss_dir, '{}.{}.log'.format(test_name, idx)) iss_csv = os.path.join(iss_dir, '{}.{}.csv'.format(test_name, idx)) if iss == "spike": process_spike_sim_log(iss_log, iss_csv) else: assert iss == 'ovpsim' # (should be checked by argparse) process_ovpsim_sim_log(iss_log, iss_csv) compare_result = \ compare_trace_csv(rtl_csv, iss_csv, "ibex", iss, report, **test.get('compare_opts', {})) # Rather oddly, compare_result is a string. The comparison passed if it # starts with '[PASSED]'. return compare_result.startswith('[PASSED]')
def compare_test_run(test: TestEntry, idx: int, seed: int, rtl_log_dir: str, iss: str, iss_log_dir: str, instr_gen_bin_dir: str) -> TestRunResult: '''Compare results for a single run of a single test Here, test is a dictionary describing the test (read from the testlist YAML file). idx is the iteration index and seed is the corresponding seed. iss is the chosen instruction set simulator (currently supported: spike and ovpsim). rtl_log_dir is the directory containing log output from the RTL simulation. iss_log_dir is the directory that contains logs for ISS runs. Returns a _CompareResult with a pass/fail flag, together with some information about the run (to be written to the log file). ''' test_name = test['test'] assert isinstance(test_name, str) uvm_log = os.path.join(rtl_log_dir, 'sim.log') elf = os.path.join(instr_gen_bin_dir, '{}_{}.o'.format(test_name, idx)) rtl_trace = os.path.join(rtl_log_dir, 'trace_core_00000000.log') kv_data = { 'name': test_name, 'idx': idx, 'seed': seed, 'binary': elf, 'uvm_log': uvm_log, 'rtl_trace': rtl_trace, 'rtl_trace_csv': None, 'iss_trace': None, 'iss_trace_csv': None, 'comparison_log': None, 'passed': False, 'failure_message': None } # Have a look at the UVM log. Report a failure if an issue is seen in the # log. try: uvm_pass, uvm_log_lines = check_ibex_uvm_log(uvm_log) except IOError as e: kv_data['failure_message'] = str(e) kv_data['failure_message'] += \ '\n[FAILED] Could not open simulation log' return TestRunResult(**kv_data) if not uvm_pass: kv_data['failure_message'] = '\n'.join(uvm_log_lines) kv_data['failure_message'] += '\n[FAILED]: sim error seen' return TestRunResult(**kv_data) rtl_trace_csv = os.path.join(rtl_log_dir, 'trace_core_00000000.csv') kv_data['rtl_trace_csv'] = rtl_trace_csv try: # Convert the RTL log file to a trace CSV. process_ibex_sim_log(rtl_trace, rtl_trace_csv, 1) except (OSError, RuntimeError) as e: kv_data['failure_message'] = \ '[FAILED]: Log processing failed: {}'.format(e) return TestRunResult(**kv_data) no_post_compare = test.get('no_post_compare', False) assert isinstance(no_post_compare, bool) # no_post_compare skips the final ISS v RTL log check, so if we've reached # here we're done when no_post_compare is set. if no_post_compare: kv_data['passed'] = True return TestRunResult(**kv_data) # There were no UVM errors. Process the log file from the ISS. iss_log = os.path.join(iss_log_dir, '{}.{}.log'.format(test_name, idx)) iss_csv = os.path.join(iss_log_dir, '{}.{}.csv'.format(test_name, idx)) kv_data['iss_trace'] = iss_log kv_data['iss_trace_csv'] = iss_csv try: if iss == "spike": process_spike_sim_log(iss_log, iss_csv) else: assert iss == 'ovpsim' # (should be checked by argparse) process_ovpsim_sim_log(iss_log, iss_csv) except (OSError, RuntimeError) as e: kv_data['failure_message'] = \ '[FAILED]: Log processing failed: {}'.format(e) return TestRunResult(**kv_data) compare_log = os.path.join(rtl_log_dir, 'compare.log') kv_data['comparison_log'] = compare_log # Delete any existing file at compare_log (the compare_trace_csv function # would append to it, which is rather confusing). try: os.remove(compare_log) except FileNotFoundError: pass compare_result = \ compare_trace_csv(rtl_trace_csv, iss_csv, "ibex", iss, compare_log, **test.get('compare_opts', {})) try: compare_log_file = open(compare_log) compare_log_contents = compare_log_file.read() compare_log_file.close() except IOError as e: kv_data['failure_message'] = \ '[FAILED]: Could not read compare log: {}'.format(e) return TestRunResult(**kv_data) # Rather oddly, compare_result is a string. The comparison passed if it # starts with '[PASSED]' and failed otherwise. compare_passed = compare_result.startswith('[PASSED]: ') if not compare_passed: assert compare_result.startswith('[FAILED]: ') kv_data['failure_message'] = ('RTL / ISS trace comparison failed\n' + compare_log_contents) return TestRunResult(**kv_data) kv_data['passed'] = True return TestRunResult(**kv_data)
def compare_test_run(test, idx, iss, output_dir, report): '''Compare results for a single run of a single test Here, test is a dictionary describing the test (read from the testlist YAML file). idx is the iteration index. iss is the chosen instruction set simulator (currently supported: spike and ovpsim). output_dir is the base output directory (which should contain logs from both the ISS and the test run itself). report is the path to the regression report file we're writing. Returns True if the test run passed and False otherwise. ''' with open(report, 'a') as report_fd: test_name = test['test'] elf = os.path.join(output_dir, 'instr_gen/asm_test/{}.{}.o'.format(test_name, idx)) logging.info("Comparing %s/DUT sim result : %s" % (iss, elf)) test_name_idx = '{}.{}'.format(test_name, idx) test_underline = '-' * len(test_name_idx) report_fd.write('\n{}\n{}\n'.format(test_name_idx, test_underline)) report_fd.write('Test binary: {}\n'.format(elf)) rtl_dir = os.path.join(output_dir, 'rtl_sim', '{}.{}'.format(test_name, idx)) uvm_log = os.path.join(rtl_dir, 'sim.log') # Have a look at the UVM log. Report a failure if an issue is seen in the # log. uvm_pass, uvm_log_lines = check_ibex_uvm_log(uvm_log) report_fd.write('sim log: {}\n'.format(uvm_log)) if not uvm_pass: for line in uvm_log_lines: report_fd.write(line) report_fd.write('[FAILED]: sim error seen\n') return False rtl_log = os.path.join(rtl_dir, 'trace_core_00000000.log') rtl_csv = os.path.join(rtl_dir, 'trace_core_00000000.csv') try: # Convert the RTL log file to a trace CSV. process_ibex_sim_log(rtl_log, rtl_csv, 1) except (OSError, RuntimeError) as e: report_fd.write('[FAILED]: Log processing failed: {}\n'.format(e)) return False no_post_compare = test.get('no_post_compare', False) assert isinstance(no_post_compare, bool) # no_post_compare skips the final ISS v RTL log check, so if we've reached # here we're done when no_post_compare is set. if no_post_compare: report_fd.write('[PASSED]\n') return True # There were no UVM errors. Process the log file from the ISS. iss_dir = os.path.join(output_dir, 'instr_gen', '{}_sim'.format(iss)) iss_log = os.path.join(iss_dir, '{}.{}.log'.format(test_name, idx)) iss_csv = os.path.join(iss_dir, '{}.{}.csv'.format(test_name, idx)) try: if iss == "spike": process_spike_sim_log(iss_log, iss_csv) else: assert iss == 'ovpsim' # (should be checked by argparse) process_ovpsim_sim_log(iss_log, iss_csv) except (OSError, RuntimeError) as e: report_fd.write('[FAILED]: Log processing failed: {}\n'.format(e)) return False compare_result = \ compare_trace_csv(rtl_csv, iss_csv, "ibex", iss, report, **test.get('compare_opts', {})) # Rather oddly, compare_result is a string. The comparison passed if it # starts with '[PASSED]'. return compare_result.startswith('[PASSED]')
def aux(): csv_output_fd = csv_output.open("w") csv_writer = RiscvInstructionTraceCsv(csv_fd=csv_output_fd) csv_writer.start_new_trace() total_num_processed = -1 prev_checkpoint = 0 do_compare = lambda : total_num_processed - prev_checkpoint >= compare_every while True: instr : int = yield cpu.instr pc : int = yield cpu.pc if (yield cpu.fetch): total_num_processed += 1 if do_compare(): csv_output_fd.flush() logging.info(f"total: {total_num_processed}, previous checkpoint: {prev_checkpoint}. calling 'compare_trace_csv'...") prev_checkpoint = total_num_processed # order of passing mtkcpu and spike matters - we allow only first passed log to be shorter compare_trace_csv(csv_output.absolute(), iss_csv.absolute(), "mtkcpu", "spike", log=0) instr_disas = disassemble(instr) logging.info(f"{hex(pc)}: {hex(instr)} : {instr_disas}") instr_str = str(instr_disas) if instr_disas else "unknown" entry = RiscvInstructionTraceEntry() entry.pc = hex(pc)[2:] entry.instr_str = instr_str instr_str_splitted = " ".join(instr_str.split()).split(" ") reg_name = instr_str_splitted[1][:-1] if len(instr_str_splitted) > 1 else "NO REG NAME" entry.binary = hex(instr)[2:] entry.mode = "3" # '3' stands for machine mode # entry.gpr value is unknown, need to calculate it if instr == 0x73: # 'ecall' - Spike simulation finishes with ecall. By default we compare with Spike. logging.critical(f"found {hex(instr)} : {instr_str} intruction: finishing simulation") return timeout_cyc = 50 found = False for _ in range(timeout_cyc): if (yield cpu.writeback): if not (yield cpu.should_write_rd): logging.info(f"detected instruction that doesn't write to rd. - {instr_str}") found = True csv_output_fd.write(f"# {hex(instr)}: {instr_str}\n") break en = yield cpu.reg_write_port.en assert en rd = yield cpu.reg_write_port.addr data = yield cpu.reg_write_port.data if instr_disas and getattr(instr_disas, "rd", None): if rd != instr_disas.rd: raise ValueError(f"{hex(instr)} reg {rd} != expected {instr_disas.rd}") else: found = True entry.gpr = [ f"{reg_name}:{hex(data)[2:]}" ] # depending on '-f' param passed to 'scripts/spike_log_to_trace_csv.py' inside riscv-dv repo, # we need either to print register-state-changing instructions only (like here), # or all instructions, then we would put the line below after the loop. csv_writer.write_trace_entry(entry) break yield if not found: raise ValueError("No register write observed!") yield
def compare_test_run(test: TestEntry, idx: int, seed: int, rtl_log_dir: str, iss: str, iss_log_dir: str) -> _CompareResult: '''Compare results for a single run of a single test Here, test is a dictionary describing the test (read from the testlist YAML file). idx is the iteration index and seed is the corresponding seed. iss is the chosen instruction set simulator (currently supported: spike and ovpsim). rtl_log_dir is the directory containing log output from the RTL simulation. iss_log_dir is the directory that contains logs for ISS runs. Returns a _CompareResult with a pass/fail flag, together with some information about the run (to be written to the log file). ''' test_name = test['test'] assert isinstance(test_name, str) uvm_log = os.path.join(rtl_log_dir, 'sim.log') kv_data = { 'test name': test_name, 'iteration': str(idx), 'seed': str(seed), 'UVM log': uvm_log } # Have a look at the UVM log. Report a failure if an issue is seen in the # log. uvm_pass, uvm_log_lines = check_ibex_uvm_log(uvm_log) if not uvm_pass: return (False, 'simulation error', kv_data) rtl_log = os.path.join(rtl_log_dir, 'trace_core_00000000.log') rtl_csv = os.path.join(rtl_log_dir, 'trace_core_00000000.csv') kv_data['rtl log'] = rtl_log kv_data['rtl csv'] = rtl_csv try: # Convert the RTL log file to a trace CSV. process_ibex_sim_log(rtl_log, rtl_csv, 1) except (OSError, RuntimeError) as e: return (False, f'RTL log processing failed ({e})', kv_data) no_post_compare = test.get('no_post_compare', False) assert isinstance(no_post_compare, bool) # no_post_compare skips the final ISS v RTL log check, so if we've reached # here we're done when no_post_compare is set. if no_post_compare: return (True, None, kv_data) # There were no UVM errors. Process the log file from the ISS. iss_log = os.path.join(iss_log_dir, '{}.{}.log'.format(test_name, idx)) iss_csv = os.path.join(iss_log_dir, '{}.{}.csv'.format(test_name, idx)) kv_data['ISS log'] = iss_log kv_data['ISS csv'] = iss_csv try: if iss == "spike": process_spike_sim_log(iss_log, iss_csv) else: assert iss == 'ovpsim' # (should be checked by argparse) process_ovpsim_sim_log(iss_log, iss_csv) except (OSError, RuntimeError) as e: return (False, f'ISS log processing failed ({e})', kv_data) compare_log = os.path.join(rtl_log_dir, 'compare.log') kv_data['comparison log'] = compare_log # Delete any existing file at compare_log (the compare_trace_csv function # would append to it, which is rather confusing). try: os.remove(compare_log) except FileNotFoundError: pass compare_result = \ compare_trace_csv(rtl_csv, iss_csv, "ibex", iss, compare_log, **test.get('compare_opts', {})) # Rather oddly, compare_result is a string. The comparison passed if it # starts with '[PASSED]' and failed otherwise. compare_passed = compare_result.startswith('[PASSED]: ') if not compare_passed: assert compare_result.startswith('[FAILED]: ') # compare_result[10:] will look like "123 matched, 321 mismatch", # meaning that 123 instructions matched and 321 instructions didn't. kv_data['compared instructions'] = compare_result[10:] return (False, 'mismatch between ISS and RTL', kv_data) # compare_result[10:] will look like "123 matched", meaning that 123 # instructions matched. kv_data['compared instructions'] = compare_result[10:] return (True, None, kv_data)