def _main(): isas = list( itertools.product(TOPLEVELS, OPCODE_TYPES, INSTR_TYPES, TERMINATE_TYPES)) print(LINE_SEP) print(green("Launching %d experiments ..." % (len(isas) * len(RUNS)))) print(LINE_SEP) # create a temp dir to store config files try: tmp_dir = os.path.join(os.getcwd(), "tmp%d") i = 0 while os.path.isdir(tmp_dir % i): i += 1 tmp_dir = tmp_dir % i os.mkdir(tmp_dir) # create config files on the fly and launch experiments for toplevel, opcode_type, instr_type, terminate_type in isas: for run in RUNS: # craft config dictionary cdict = copy.deepcopy(CONFIG_DICT) # Set experiment name experiment_name = EXPERIMENT_BASE_NAME % ( toplevel, opcode_type, instr_type, terminate_type, run) experiment_name = experiment_name.replace("_", "-") # print(experiment_name) # continue cdict["experiment_name"] = experiment_name cdict["toplevel"] = toplevel # Set configurations cdict["fuzzer_params"]["duration_mins"] = DURATION_MINS cdict["model_params"]["opcode_type"] = opcode_type cdict["model_params"]["instr_type"] = instr_type if terminate_type == "invalidop": cdict["model_params"]["terminate_on_invalid_opcode"] = 1 else: cdict["model_params"]["terminate_on_invalid_opcode"] = 0 # write to HJSON file hjson_filename = experiment_name + ".hjson" hjson_file_path = os.path.join(tmp_dir, hjson_filename) with open(hjson_file_path, "w") as fp: hjson.dump(cdict, fp) # launch fuzz the DUT fuzz(["--fail-silently", hjson_file_path]) # cleanup config file os.remove(hjson_file_path) finally: for directory in glob.glob("tmp*"): shutil.rmtree(directory) print(LINE_SEP) print(green("DONE!")) print(LINE_SEP)
def build_docker_image(config): """Creates docker image containing DUT to fuzz.""" if not config.args.silent: print(LINE_SEP) print("Building Docker image to fuzz %s ..." % config.toplevel) print(LINE_SEP) # Set Dockerfile path if config.soc == "other": dockerfile_path = "%s/hw/other/%s" % (config.root_path, config.toplevel) else: dockerfile_path = "%s/hw/%s" % (config.root_path, config.soc) # Build command cmd = [ "docker", "build", "--build-arg", "TOPLEVEL=%s" % config.toplevel, "--build-arg", "TB_TYPE=%s" % config.tb_type, "--build-arg", "FUZZER=%s" % config.fuzzer, "--build-arg", "VERSION=%s" % config.version, "-t", config.docker_image, dockerfile_path ] error_str = "ERROR: image build FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("IMAGE BUILD SUCCESSFUL -- Done!"))
def load_bb_data(data_root): print(yellow("Loading data ...")) exp2data = {} # TODO: change this to automatically extract names from a single exp. number # extract each data file into a Pandas dataframe exp_combos = list(itertools.product(STATES, WIDTHS, EXP_BASE_NAMES)) for num_states, width, exp_base_name in exp_combos: for trial in TRIALS: # Build complete path to data files exp_name_wo_trialnum = exp_base_name % (num_states, width) exp_name = "%s-%d" % (exp_name_wo_trialnum, trial) data_path = os.path.join(data_root, exp_name) # Extract experiment info. exp_name_list = exp_name.split("-") instr_type = exp_name_list[6] if len(exp_name_list) > 9: fs_opt = True else: fs_opt = False # Load fuzzing data into an object exp2data[exp_name_wo_trialnum] = FuzzingData( num_states, width, instr_type, fs_opt, trial, data_path) print(green("Done.")) print(LINE_SEP) return exp2data
def check_num_active_vm_instances(config): """Checks number of active VM instances on GCE as a $$$ safety measure.""" if not config.args.silent: print(LINE_SEP) print("Checking number of active VMs on GCE ...") print(LINE_SEP) cmd = [ "gcloud", "compute", "instances", "list", "--zones=%s" % config.gcp_params["zone"] ] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) num_active_vm_instances = -1 # first line is header while True: line = proc.stdout.readline() if not line: break num_active_vm_instances += 1 if num_active_vm_instances < config.args.max_vm_instances: if not config.args.silent: print(green("%d active VM(s)" % num_active_vm_instances)) else: if not config.args.silent: print(red("%d active VM(s)" % num_active_vm_instances)) print( yellow("waiting %d seconds and trying again ..." % config.args.vm_launch_wait_time_s)) return num_active_vm_instances
def compute_instr_type_mann_whitney(instr_rts): print( yellow( "Computing Mann-Whitney U-test on instrumentation complexity data ..." )) for num_states in STATES: sub_rt_df = instr_rts[instr_rts[NUM_STATES_LABEL] == num_states] full_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] == INSTR_TYPE_MAPPINGS["full"]][RUN_TIME_LABEL] duttb_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] == INSTR_TYPE_MAPPINGS["duttb"]][RUN_TIME_LABEL] dut_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] == INSTR_TYPE_MAPPINGS["dut"]][RUN_TIME_LABEL] # mw_full_duttb = stats.mannwhitneyu(full_instr_data, duttb_instr_data) mw_full_dut = stats.mannwhitneyu(full_instr_data, dut_instr_data) # mw_duttb_dut = stats.mannwhitneyu(duttb_instr_data, dut_instr_data) print("%d States - Mann-Whitney:" % num_states) # print( # "\t%s vs. %s:" % # (INSTR_TYPE_MAPPINGS["full"], INSTR_TYPE_MAPPINGS["duttb"]), # mw_full_duttb) print( "\t%s vs. %s:" % (INSTR_TYPE_MAPPINGS["full"], INSTR_TYPE_MAPPINGS["dut"]), mw_full_dut) # print( # "\t%s vs. %s:" % # (INSTR_TYPE_MAPPINGS["duttb"], INSTR_TYPE_MAPPINGS["dut"]), # mw_duttb_dut) print(green("Done.")) print(LINE_SEP)
def build_bbs_df(exp2data): print(yellow("Building basic block stats dataframe ...")) # Create empty dictionary that will be used to create Pandas # a DataFrame that look like the following: # +---------------------------------------------------+ # | # states | instrumentation level | # basic blocks | # +---------------------------------------------------+ # | ... | ... | ... | bbs_dict = { NUM_STATES_LABEL: [], INSTR_TYPE_LABEL: [], NUM_BB_LABEL: [], } for exp_name, fd in exp2data.items(): bbs_dict[NUM_STATES_LABEL].extend([fd.num_states] * 2) # bbs_dict[INSTR_TYPE_LABEL].extend(["Simulation Engine", "TB", "DUT"]) bbs_dict[INSTR_TYPE_LABEL].extend(["Simulation Engine + TB", "DUT"]) # bbs_dict[NUM_BB_LABEL].append(fd.vltrt_bbs) # bbs_dict[NUM_BB_LABEL].append(fd.tb_bbs) bbs_dict[NUM_BB_LABEL].append(fd.vltrt_bbs + fd.tb_bbs) bbs_dict[NUM_BB_LABEL].append(fd.dut_bbs) # bbs_dict[NUM_STATES_LABEL].append(fd.num_states) # bbs_dict[INSTR_TYPE_LABEL].append(INSTR_TYPE_MAPPINGS[fd.instr_type]) # if fd.instr_type == "full": # bbs_dict[NUM_BB_LABEL].append(fd.full_bbs) # elif fd.instr_type == "duttb": # bbs_dict[NUM_BB_LABEL].append(fd.duttb_bbs) # elif fd.instr_type == "dut": # bbs_dict[NUM_BB_LABEL].append(fd.dut_bbs) # else: # print(red("ERROR: unknown instrumentation type.")) # sys.exit(1) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(bbs_dict)
def _check_simulation_results(encrypt_results, decrypt_results, test_pair, verbose=False): error = False for i in range(len(test_pair.encrypt.data_in_line_starts)): # Extract plaintexts/ciphertexts with open(encrypt_results, "r") as fp: log_lines = fp.readlines() encrypt_in_data = _get_crypt_io_data_blocks( log_lines, test_pair.encrypt.data_in_line_starts[i], test_pair.encrypt.data_block_size) encrypt_out_data = _get_crypt_io_data_blocks( log_lines, test_pair.encrypt.data_out_line_starts[i], test_pair.encrypt.data_block_size) with open(decrypt_results, "r") as fp: log_lines = fp.readlines() decrypt_in_data = _get_crypt_io_data_blocks( log_lines, test_pair.decrypt.data_in_line_starts[i], test_pair.decrypt.data_block_size) decrypt_out_data = _get_crypt_io_data_blocks( log_lines, test_pair.decrypt.data_out_line_starts[i], test_pair.decrypt.data_block_size) # Check all data blocks are the same length if (len(encrypt_in_data) != len(encrypt_out_data) or len(encrypt_in_data) != len(decrypt_in_data) or len(encrypt_in_data) != len(decrypt_out_data)): error = True error_msg = "data LENGTH MISMATCH for encrypt/decrypt data blocks." if not error: for i in range(len(encrypt_out_data)): if verbose: print( "\n Encrypt Out = 0x{:0>8X}; Decrypt In = 0x{:0>8X}" .format(encrypt_out_data[i], decrypt_in_data[i])) if encrypt_out_data[i] != decrypt_in_data[i]: error_msg = "encryption OUTPUT does not match decryption INPUT." error = True break if verbose: print(" --------------------------------------------") if not error: for i in range(len(encrypt_in_data)): if verbose: print( " Encrypt In = 0x{:0>8X}; Decrypt Out = 0x{:0>8X}". format(encrypt_in_data[i], decrypt_out_data[i])) if encrypt_in_data[i] != decrypt_out_data[i]: error_msg = "encryption INPUT does not match decryption OUTPUT." break if verbose: print(" --------------------------------------------") # If an error occured during any test, terminate if error: break # Print a color-coded message with results if not error: print(green("PASS")) else: print("".join([red("ERROR"), ": ", error_msg]))
def build_fs_opt_rts_df(exp2data): print(yellow("Building fork server optimization dataframe ...")) FS_OPT_BASELINE = True # Create empty dictionary that will be used to create Pandas # a DataFrame that look like the following: # +----------------------------------------------------+ # | # states | fork server optimization? | runtime (s) | # +----------------------------------------------------+ # | ... | ... | ... | runtimes_dict = { NUM_STATES_LABEL: [], OPT_TYPE_LABEL: [], RUN_TIME_LABEL: [], } # Aggregate data into a dictionary exp2rts = _aggregrate_fs_opt_rts(exp2data) # Compute scale factors for each set of num_states experiments states2scales = {} for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): if instr_type == "full" and fs_opt is FS_OPT_BASELINE: scale_factor = np.median(runtimes) states2scales[num_states] = scale_factor # Build the dataframe for plotting for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): runtimes = list(map(lambda x: x / states2scales[num_states], runtimes)) runtimes_dict[NUM_STATES_LABEL].extend([num_states] * len(runtimes)) runtimes_dict[OPT_TYPE_LABEL].extend([OPT_TYPE_MAPPINGS[fs_opt]] * len(runtimes)) runtimes_dict[RUN_TIME_LABEL].extend(runtimes) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(runtimes_dict)
def build_instr_complex_rts_df(exp2data): print(yellow("Building instruction complexity dataframe ...")) INSTR_TYPE_BASELINE = "dut" # Create empty dictionary that will be used to create Pandas # a DataFrame that look like the following: # +------------------------------------------------+ # | # states | instrumentation level | runtime (s) | # +------------------------------------------------+ # | ... | ... | ... | runtimes_dict = { NUM_STATES_LABEL: [], INSTR_TYPE_LABEL: [], RUN_TIME_LABEL: [], } # Aggregate data into a dictionary exp2rts = _aggregrate_instr_complex_rts(exp2data) # Compute scale factors for each set of num_states experiments states2scales = {} for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): if instr_type == INSTR_TYPE_BASELINE and fs_opt is False: scale_factor = np.median(runtimes) states2scales[num_states] = scale_factor # Build the dataframe for plotting for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): runtimes = list(map(lambda x: x / states2scales[num_states], runtimes)) runtimes_dict[NUM_STATES_LABEL].extend([num_states] * len(runtimes)) runtimes_dict[INSTR_TYPE_LABEL].extend([INSTR_TYPE_MAPPINGS[instr_type]] * len(runtimes)) runtimes_dict[RUN_TIME_LABEL].extend(runtimes) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(runtimes_dict)
def plot_avg_coverage_vs_time_broken(hwf_cov_df, rfuzz_cov_df, time_units="m"): print(yellow("Generating plot ...")) # Set plot style and extract only HDL line coverage # sns.set_theme(context="notebook", style="darkgrid") hdl_cov_df = pd.concat([hwf_cov_df, rfuzz_cov_df]) # create subplots fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(6, 4)) # create figure and plot the data # fig, ax = plt.subplots(1, 1, figsize=(6, 4)) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, style=FUZZER_LABEL, ax=ax1) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, style=FUZZER_LABEL, ax=ax2) # set axis ranges ax1.set_ylim(0.3, 1.0) ax2.set_ylim(0, 0.05) # hide the spines between the two axes ax1.spines['bottom'].set_visible(False) ax2.spines['top'].set_visible(False) ax1.xaxis.tick_top() ax1.tick_params(labeltop=False) ax2.xaxis.tick_bottom() # format the plot if time_units == "m": time_units_label = "min." elif time_units == "h": time_units_label = "hours" else: time_units_label = "s" # ax1.set_xlabel(TIME_LABEL + " (%s)" % time_units_label, # fontsize=LABEL_FONT_SIZE) # ax1.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE) # ax1.tick_params("x", labelsize=TICK_FONT_SIZE) # ax1.tick_params("y", labelsize=TICK_FONT_SIZE) # plt.legend(fontsize=LEGEND_FONT_SIZE, # title_fontsize=LEGEND_TITLE_FONT_SIZE, # bbox_to_anchor=(1.01, 0.75), # loc='upper left') plt.tight_layout() # save the plot plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def _main(argv): # Part command line arguments args = _parse_args(argv) # Print initial message with # of total tests isas = list(itertools.product(OPCODE_TYPES, INSTR_TYPES, TERMINATE_TYPES)) num_tests = len(isas) print(LINE_SEP) print(green("Running %d test suites ..." % num_tests)) # Get list of encryption/decryption test pairs yaml_descripts_dir = os.path.join(os.getenv("HW_FUZZING"), "hw", "opentitan", "aes", "seed_descriptions") test_pairs = _extract_test_pairs(yaml_descripts_dir, args.testcase) test_suite_num = 1 for opcode_type, instr_type, terminate_type in isas: print(LINE_SEP) print("Test Suite #: ", test_suite_num) print("Opcode Type: ", opcode_type) print("Instruction Type:", instr_type) print("Termination Type:", terminate_type) print(LINE_SEP) # Set ISA parameters config_dict = _set_isa_params(HJSON_CONFIG_TEMPLATE, opcode_type, instr_type, terminate_type) # Perform E2E encryption/decryption simulation tests on OpenTitan AES block for tp in test_pairs: try: print("(%s)\t- " % tp.test_id, end="") sys.stdout.flush() # Run simulations in separate thread to show animated waiting spinner # TEST_IS_EXECUTING = True # t = threading.Thread(target=_waiting_animation) # t.daemon = True # t.start() _run_simulation(config_dict, ENCRYPT_LOG, tp.encrypt) _run_simulation(config_dict, DECRYPT_LOG, tp.decrypt) # TEST_IS_EXECUTING = False # Check simulation results _check_simulation_results(ENCRYPT_LOG, DECRYPT_LOG, tp, args.verbose) finally: try: os.remove(TMP_HJSON_CONFIG) os.remove(ENCRYPT_LOG) os.remove(DECRYPT_LOG) except OSError as e: if e.errno != errno.ENOENT: raise test_suite_num += 1
def _main(): print(LINE_SEP) print(green("Launching %d experiments ..." % (len(TOPLEVELS) * len(RUNS)))) print(LINE_SEP) # create a temp dir to store config files with tempfile.TemporaryDirectory() as tmp_dir: # create config files on the fly and launch experiments for toplevel in TOPLEVELS: for run in RUNS: # craft config dictionary cdict = copy.deepcopy(CONFIG_DICT) # Set experiment name experiment_name = EXPERIMENT_BASE_NAME % (toplevel, DURATION_MINS, run) experiment_name = experiment_name.replace("_", "-").lower() cdict["experiment_name"] = experiment_name cdict["toplevel"] = toplevel # Set configurations cdict["fuzzer_params"]["duration_mins"] = DURATION_MINS # write to HJSON file hjson_filename = experiment_name + ".hjson" hjson_file_path = os.path.join(tmp_dir, hjson_filename) with open(hjson_file_path, "w") as fp: hjson.dump(cdict, fp) # launch fuzz the DUT # fuzz(["--fail-silently", hjson_file_path]) fuzz([ "-y", "--gcp-config-filename", "gcp_config.east1b.hjson", hjson_file_path ]) # cleanup config file os.remove(hjson_file_path) print(LINE_SEP) print(green("DONE!")) print(LINE_SEP)
def compute_fs_opt_mann_whitney(instr_rts): print(yellow("Computing Mann-Whitney U-test on fork server opt. data ...")) for num_states in STATES: sub_rt_df = instr_rts[instr_rts[NUM_STATES_LABEL] == num_states] no_opt_data = sub_rt_df[sub_rt_df[OPT_TYPE_LABEL] == OPT_TYPE_MAPPINGS[False]][RUN_TIME_LABEL] opt_data = sub_rt_df[sub_rt_df[OPT_TYPE_LABEL] == OPT_TYPE_MAPPINGS[True]][RUN_TIME_LABEL] mw = stats.mannwhitneyu(no_opt_data, opt_data) print("%d States - Mann-Whitney:" % num_states) print("\t%s vs. %s:" % (OPT_TYPE_MAPPINGS[False], OPT_TYPE_MAPPINGS[True]), mw.pvalue) print(green("Done.")) print(LINE_SEP)
def gen_seed(input_yaml_file_name, output_file_name, verbose): """Parse YAML HW fuzzing opcodes and translates them in binary to file.""" print(f"Creating fuzzer seed from YAML: {input_yaml_file_name} ...") with open(input_yaml_file_name, "r") as fp: fuzz_opcodes = yaml.load(fp, Loader=yaml.Loader) with open(output_file_name, "wb") as fp: for instr in fuzz_opcodes: hwf_instr = TLULFuzzInstr(instr) if verbose: print(hwf_instr) for _ in range(hwf_instr.repeat): fp.write(hwf_instr.to_bytes()) print(green("Seed file generated!")) if verbose: dump_seed_file_to_stdin(output_file_name)
def push_docker_image_to_gcr(config): """Pushes docker image to GCR if it does not exist there yet.""" if not config.args.silent: print(LINE_SEP) print("Pushing Docker image to GCR ...") print(LINE_SEP) if config.args.update or not check_if_docker_image_exists_in_gcr(config): cmd = ["docker", "push", config.docker_image] error_str = "ERROR: pushing image to GCR FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("IMAGE PUSH SUCCESSFUL -- Done!")) else: if not config.args.silent: print(yellow("IMAGE ALREADY EXISTS IN GCR -- Done!"))
def build_docker_image(config): """Creates docker image containing DUT to fuzz.""" if not config.args.silent: print(LINE_SEP) print("Building Docker image to fuzz %s ..." % config.toplevel) print(LINE_SEP) cmd = [ "docker", "build", "--build-arg", "FUZZER=%s" % config.fuzzer, "--build-arg", "VERSION=%s" % config.version, "-t", config.docker_image, "%s/hw/%s" % (config.root_path, config.toplevel) ] error_str = "ERROR: image build FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("IMAGE BUILD SUCCESSFUL -- Done!"))
def plot_coverage_vs_time(coverage_dfs): print(yellow("Generating plots ...")) cov_metrics = [ SW_LINE_COVERAGE_LABEL, SW_REGION_COVERAGE_LABEL, HW_LINE_COVERAGE_LABEL ] num_cores = len(TOPLEVELS) num_cov_metrics = len(cov_metrics) sns.set_theme(context="notebook", style="darkgrid") fig, axes = plt.subplots(num_cov_metrics, num_cores, sharex=True, sharey=True) for trial in range(len(coverage_dfs)): # Select experiment trial number cov_df = coverage_dfs[trial] for row in range(len(axes)): # select portion of data corresponding to current COVERAGE METRIC sub_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == cov_metrics[row]] for col in range(len(axes[row])): # select portion of data corresponding to current core plt_df = sub_cov_df[sub_cov_df[TOPLEVEL_LABEL] == TOPLEVELS[col]] # sns.set_context("paper") curr_ax = sns.lineplot(data=plt_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=GRAMMAR_LABEL, ax=axes[row][col], legend=False) if row == 0 and col == 0 and trial == 0: lines = curr_ax.get_lines() axes[row][col].set_title("Coverage = %s | Core = %s" % (cov_metrics[row], TOPLEVELS[col])) fig.legend( lines, [ "Const. Opcode & Variable Frame", "Const. Opcode & Fixed Frame", "Mapped Opcode & Variable Frame", "Mapped Opcode & Fixed Frame", ], loc="lower center", ncol=4, ) print(green("Done.")) print(LINE_SEP) plt.show()
def build_coverage_df(exp2data, trial): print(yellow("Building coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | isa (grammar) | coverage type | time (s) | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], GRAMMAR_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } # Add rows to the dataframe for exp_name, fd_list in exp2data.items(): fd = fd_list[trial] for time, row in fd.afl_data.iterrows(): cov_df_idx = row["paths_total"] - 1 for _ in range(3): coverage_dict[TOPLEVEL_LABEL].append(fd.toplevel) coverage_dict[GRAMMAR_LABEL].append(fd.grammar) coverage_dict[TIME_LABEL].append(time) # Add kcov coverage kcov = fd.kcov_data.loc[cov_df_idx, "Line-Coverage-(%)"] coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(kcov * 100.0) # Add LLVM coverage llvm_cov = fd.llvm_cov_data.loc[cov_df_idx, "Region-Coverage-(%)"] coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(llvm_cov * 100.0) # Add Verilator coverage vlt_cov = (float(fd.vlt_cov_data.loc[cov_df_idx, "Lines-Covered"]) / float(fd.vlt_cov_data.loc[cov_df_idx, "Total-Lines"])) coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(vlt_cov * 100.0) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict)
def push_vm_management_scripts_to_gcs(config): """Pushes VM management (startup/shutdown scripts to GCS.""" if not config.args.silent: print(LINE_SEP) print("Copying VM management script to GCS ...") print(LINE_SEP) cmd = [ "gsutil", "cp", "%s/infra/hwfp/%s" % (config.root_path, config.gcp_params["startup_script"]), "gs://%s-%s/%s" % (config.gcp_params["project_id"], config.gcp_params["vm_management_bucket"], config.gcp_params["startup_script"]) ] error_str = "ERROR: pushing scripts to GCS FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("COPY SUCCESSFUL -- Done!"))
def compute_stats(hwf_cov_dict, rfuzz_cov_dict): print(yellow("Computing stats ...")) # Compute HDL coverage % differences cov_diffs_sum = 0 for toplevel, hwf_cov in hwf_cov_dict.items(): min_hwf_cov = min(hwf_cov) max_rfuzz_cov = max(rfuzz_cov_dict[toplevel]) cov_diff = min_hwf_cov - max_rfuzz_cov cov_diffs_sum += cov_diff print("HWF vs. RFUZZ coverage (%15s): %.3f%%" % (toplevel, cov_diff)) cov_diffs_avg = float(cov_diffs_sum) / float(len(hwf_cov_dict.keys())) print("Avg. coverage difference: %.3f%%" % (cov_diffs_avg)) print('-' * len(LINE_SEP)) for toplevel, hwf_cov in hwf_cov_dict.items(): rfuzz_cov = rfuzz_cov_dict[toplevel] myu = stats.mannwhitneyu(hwf_cov, rfuzz_cov) print("HWF vs. RFUZZ Mann-Whitney (%15s): %s" % (toplevel, myu)) print(green("Done.")) print(LINE_SEP)
def create_local_experiment_data_dir(config): """Creates local directories to store fuzzing experiment data.""" if not config.args.silent: print(LINE_SEP) print("Creating local directories for fuzzing data ...") print(LINE_SEP) exp_data_path = "%s/data/%s" % \ (config.root_path, config.experiment_name) # Create directories os.makedirs(exp_data_path) os.chmod(exp_data_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) os.mkdir(os.path.join(exp_data_path, "out")) os.chmod(os.path.join(exp_data_path, "out"), stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) os.mkdir(os.path.join(exp_data_path, "logs")) os.chmod(os.path.join(exp_data_path, "logs"), stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) # Copy over seeds that were used in this experiment seeds_dir = "%s/hw/%s/%s/seeds" % (config.root_path, config.soc, config.toplevel) seed_descripts_dir = "%s/hw/%s/%s/seed_descriptions" % ( config.root_path, config.soc, config.toplevel) if os.path.isdir(seeds_dir): shutil.copytree(seeds_dir, os.path.join(exp_data_path, "seeds")) os.chmod(os.path.join(exp_data_path, "seeds"), stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) elif os.path.isdir(seed_descripts_dir): shutil.copytree(seed_descripts_dir, os.path.join(exp_data_path, "seed_descriptions")) os.chmod(os.path.join(exp_data_path, "seed_descriptions"), stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH) else: print(red("ERROR: no seeds found. Terminating experiment!"), file=sys.stderr) # Copy over HJSON config file that was used shutil.copy2(config.config_filename, exp_data_path) if not config.args.silent: print(green("DIRECTORY CREATION SUCCESSFUL -- Done!")) return exp_data_path
def plot_bbs(instr_bbs, orientation="h"): print(yellow("Generating plots ...")) LABEL_FONT_SIZE = 14 sns.set() if orientation == "h": ax = sns.barplot(y=NUM_STATES_LABEL, x=NUM_BB_LABEL, hue=INSTR_TYPE_LABEL, data=instr_bbs, orient="h", ci=None) ax.set_ylabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE) ax.set_xlabel(NUM_BB_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("y", labelsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=LABEL_FONT_SIZE) ax.set_xlim(10, 10000) plt.xscale("log") else: ax = sns.barplot(x=NUM_STATES_LABEL, y=NUM_BB_LABEL, hue=INSTR_TYPE_LABEL, data=instr_bbs, ci=None) ax.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE) ax.set_ylabel(NUM_BB_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=LABEL_FONT_SIZE) ax.tick_params("y", labelsize=LABEL_FONT_SIZE) ax.set_ylim(10, 10000) plt.yscale("log") plt.legend(title=INSTR_TYPE_LABEL, fontsize=LABEL_FONT_SIZE, title_fontsize=LABEL_FONT_SIZE, ncol=3, loc="upper center", bbox_to_anchor=(0.5, 1.25)) plt.tight_layout() plt.savefig("hwf_components_bbs.png", format="PNG") print(green("Done.")) print(LINE_SEP)
def plot_avg_coverage_vs_time(hwf_cov_df, rfuzz_cov_df, time_units="m"): print(yellow("Generating plot ...")) # Set plot style and extract only HDL line coverage sns.set_theme(context="notebook", style="darkgrid") hdl_cov_df = pd.concat([hwf_cov_df, rfuzz_cov_df]) # create figure and plot the data fig, ax = plt.subplots(1, 1, figsize=(6, 4)) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, style=FUZZER_LABEL, ax=ax) # format the plot if time_units == "m": time_units_label = "min." elif time_units == "h": time_units_label = "hours" else: time_units_label = "s" ax.set_xlabel(TIME_LABEL + " (%s)" % time_units_label, fontsize=LABEL_FONT_SIZE) ax.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=TICK_FONT_SIZE) ax.tick_params("y", labelsize=TICK_FONT_SIZE) plt.legend(fontsize=LEGEND_FONT_SIZE, title_fontsize=LEGEND_TITLE_FONT_SIZE, bbox_to_anchor=(1.01, 0.75), loc='upper left') plt.tight_layout() # save the plot plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def plot_avg_coverage_vs_time(cov_df, time_units="m"): print(yellow("Generating plot ...")) # Set plot style and extract only HDL line coverage sns.set_theme(context="notebook", style="darkgrid") hdl_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == HW_LINE_COVERAGE_LABEL] # create figure and plot the data fig, ax = plt.subplots(1, 1, figsize=(4, 2)) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, ax=ax, markers="x") # format the plot if time_units == "m": time_units_label = "min." elif time_units == "h": time_units_label = "hours" else: time_units_label = "s" ax.set_xlabel(TIME_LABEL + " (%s)" % time_units_label, fontsize=LABEL_FONT_SIZE) ax.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=TICK_FONT_SIZE) ax.tick_params("y", labelsize=TICK_FONT_SIZE) plt.legend(title="Core", fontsize=LEGEND_FONT_SIZE, title_fontsize=LEGEND_TITLE_FONT_SIZE, ncol=2) plt.tight_layout() # save the plot plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def build_min_hwf_coverage_df(exp2data, time_units="m", normalize_to_start=False, consolidation="max"): print(yellow("Building HWF coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | fuzzer | coverage type | time | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], FUZZER_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } cov_dict = collections.defaultdict( list) # maps toplevel --> [coverage list] for exp_name, fd_list in exp2data.items(): # get min coverage experiment min_cov = get_max_vlt_cov(fd_list[0].hwf_cov_data) min_cov_fd = fd_list[0] for fd in fd_list: cov = get_max_vlt_cov(fd.hwf_cov_data) cov_dict[fd.toplevel].append(cov) if cov < min_cov: min_cov = cov min_cov_fd = fd # build data frame for plotting for time, row in min_cov_fd.hwf_afl_data.iterrows(): # scale time scaled_time = scale_time(time, time_units) # add circuit, fuzzer, and time values to dataframe row coverage_dict[TOPLEVEL_LABEL].append(min_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(scaled_time) # get the AFL paths_total at the current time paths_total = get_paths_total_at_time(time, min_cov_fd.hwf_afl_data) - 1 # get HWF coverage data hwf_vlt_cov = get_vlt_cov_at_time(paths_total, min_cov_fd.hwf_cov_data) # normalize to start time if requested if time == 0: hwf_vlt_cov_t0 = hwf_vlt_cov if normalize_to_start: hwf_vlt_cov /= hwf_vlt_cov_t0 # add to data frame coverage_dict[FUZZER_LABEL].append("HWFP") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(hwf_vlt_cov) # extend lines to max time value if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME: coverage_dict[TOPLEVEL_LABEL].append(min_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME) coverage_dict[FUZZER_LABEL].append("HWFP") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append( coverage_dict[COVERAGE_LABEL][-1]) print("Min. HW Line coverage (%15s): %.3f%%" % (min_cov_fd.toplevel, coverage_dict[COVERAGE_LABEL][-1])) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict), cov_dict
def build_avg_coverage_df(exp2data, time_units="m", normalize_to_start=False, consolidation="max"): print(yellow("Building average coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | isa (grammar) | coverage type | time (s) | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], GRAMMAR_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } for exp_name, fd_list in exp2data.items(): anchor_fd = fd_list[0] for time, row in anchor_fd.afl_data.iterrows(): # scale time if time_units == "h": scaled_time = float(time) / float(3600) elif time_units == "m": scaled_time = float(time) / float(60) else: scaled_time = time # add circuit, grammar, and time values to dataframe row for _ in range(3): coverage_dict[TOPLEVEL_LABEL].append(anchor_fd.toplevel) coverage_dict[GRAMMAR_LABEL].append(anchor_fd.grammar) coverage_dict[TIME_LABEL].append(scaled_time) # compute average coverage at all points in time kcov_avg = 0 llvm_cov_avg = 0 vlt_cov_avg = 0 kcov_max = 0 llvm_cov_max = 0 vlt_cov_max = 0 i = 0 for fd in fd_list: # get the paths_total at the current time paths_total = get_paths_total_at_time(time, fd.afl_data) - 1 # get coverage data # print(exp_name, i) kcov = get_cov_at_time(paths_total, fd.kcov_data, "Line-Coverage-(%)") kcov_avg += kcov kcov_max = max(kcov_max, kcov) llvm_cov = get_cov_at_time(paths_total, fd.llvm_cov_data, "Region-Coverage-(%)") llvm_cov_avg += llvm_cov llvm_cov_max = max(llvm_cov_max, llvm_cov) vlt_cov = get_vlt_cov_at_time(paths_total, fd.vlt_cov_data) vlt_cov_avg += vlt_cov vlt_cov_max = max(vlt_cov_max, vlt_cov) i += 1 kcov_avg /= float(len(fd_list)) llvm_cov_avg /= float(len(fd_list)) vlt_cov_avg /= float(len(fd_list)) # save time 0 coverage to normalize if time == 0: kcov_avg_t0 = kcov_avg llvm_cov_avg_t0 = llvm_cov_avg vlt_cov_avg_t0 = vlt_cov_avg if normalize_to_start: kcov_avg /= kcov_avg_t0 llvm_cov_avg /= llvm_cov_avg_t0 vlt_cov_avg /= vlt_cov_avg_t0 coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) if consolidation == "avg": coverage_dict[COVERAGE_LABEL].append(kcov_avg) coverage_dict[COVERAGE_LABEL].append(llvm_cov_avg) coverage_dict[COVERAGE_LABEL].append(vlt_cov_avg) else: coverage_dict[COVERAGE_LABEL].append(kcov_max) coverage_dict[COVERAGE_LABEL].append(llvm_cov_max) coverage_dict[COVERAGE_LABEL].append(vlt_cov_max) # extend lines to max time value if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME: for _ in range(3): coverage_dict[TOPLEVEL_LABEL].append(anchor_fd.toplevel) coverage_dict[GRAMMAR_LABEL].append(anchor_fd.grammar) coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME) coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].extend( coverage_dict[COVERAGE_LABEL][-3:]) # print("Max SW Line coverage: ", coverage_dict[COVERAGE_LABEL][-3]) # print("Max SW Basic Block coverage:", coverage_dict[COVERAGE_LABEL][-2]) print("Max HW Line coverage: ", coverage_dict[COVERAGE_LABEL][-1]) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict)
def _main(): num_experiments = len(NUM_STATES) * len(COMP_WIDTHS) * len(RUNS) * len( EXPERIMENT_BASE_NAMES) print(LINE_SEP) print(LINE_SEP) print(LINE_SEP) print(green("LAUNCHING %d EXPERIMENTS ..." % num_experiments)) print(LINE_SEP) print(LINE_SEP) print(LINE_SEP) # create a temp dir to store config files try: tmp_dir = os.path.join(os.getcwd(), "tmp%d") i = 0 while os.path.isdir(tmp_dir % i): i += 1 tmp_dir = tmp_dir % i os.mkdir(tmp_dir) # create config files on the fly and launch experiments for experiment_base_name in EXPERIMENT_BASE_NAMES: for states in NUM_STATES: for width in COMP_WIDTHS: for run in RUNS: # craft config dictionary cdict = copy.deepcopy(CONFIG_DICT) # Set experiment name experiment_name = experiment_base_name % (states, width, run) cdict["experiment_name"] = experiment_name # Set test bench if "wopt" in experiment_name: cdict["tb"] = "afl_opt" else: cdict["tb"] = "afl" # Set instrumentation amount if "full-instr" in experiment_name: cdict["instrument_dut"] = 1 cdict["instrument_tb"] = 1 cdict["instrument_vltrt"] = 1 elif "duttb-instr" in experiment_name: cdict["instrument_dut"] = 1 cdict["instrument_tb"] = 1 cdict["instrument_vltrt"] = 0 elif "dut" in experiment_name: cdict["instrument_dut"] = 1 cdict["instrument_tb"] = 0 cdict["instrument_vltrt"] = 0 else: print(red("ERROR: invalid instrumentation config. ABORTING!")) sys.exit(1) # Set lock size cdict["hdl_gen_params"]["num_lock_states"] = states cdict["hdl_gen_params"]["lock_comp_width"] = width # write to HJSON file hjson_filename = experiment_name + ".hjson" hjson_file_path = os.path.join(tmp_dir, hjson_filename) with open(hjson_file_path, "w") as fp: hjson.dump(cdict, fp) # launch fuzz the DUT fuzz(["--fail-silently", hjson_file_path]) # cleanup config file os.remove(hjson_file_path) finally: # remove temp dir for tmp_dir in glob.glob("tmp*"): shutil.rmtree(tmp_dir, ignore_errors=True) print(LINE_SEP) print(LINE_SEP) print(LINE_SEP) print(green("DONE!")) print(LINE_SEP) print(LINE_SEP) print(LINE_SEP)
def build_max_rfuzz_coverage_df(exp2data, time_units="m", normalize_to_start=False, consolidation="max"): print(yellow("Building RFUZZ coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | fuzzer | coverage type | time | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], FUZZER_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } cov_dict = collections.defaultdict( list) # maps toplevel --> [coverage list] for exp_name, fd_list in exp2data.items(): # get max coverage experiment max_cov = get_max_vlt_cov(fd_list[0].rfuzz_cov_data) max_cov_fd = fd_list[0] for fd in fd_list: cov = get_max_vlt_cov(fd.rfuzz_cov_data) cov_dict[fd.toplevel].append(cov) if cov > max_cov: max_cov = cov max_cov_fd = fd for test_id, row in max_cov_fd.rfuzz_data.iterrows(): # scale time scaled_time = scale_time(row["Time (s)"], time_units) # add circuit, fuzzer, and time values to dataframe row coverage_dict[TOPLEVEL_LABEL].append(max_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(scaled_time) # compute average coverage at all points in time rfuzz_vlt_cov = get_vlt_cov_at_time(test_id, max_cov_fd.rfuzz_cov_data) # save time 0 coverage to normalize if requested if test_id == 0: rfuzz_vlt_cov_t0 = rfuzz_vlt_cov if normalize_to_start: rfuzz_vlt_cov /= rfuzz_vlt_cov_t0 # add coverage to dataframe row coverage_dict[FUZZER_LABEL].append("RFUZZ") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(rfuzz_vlt_cov) # extend lines to max time value if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME: coverage_dict[TOPLEVEL_LABEL].append(max_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME) coverage_dict[FUZZER_LABEL].append("RFUZZ") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append( coverage_dict[COVERAGE_LABEL][-1]) print("Max HW Line coverage (%15s): %.3f%%" % (max_cov_fd.toplevel, coverage_dict[COVERAGE_LABEL][-1])) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict), cov_dict
def run_docker_container_on_gce(config): """Runs a Docker container to fuzz the DUT on a Google Compute Engine VM.""" # ***IMPORTANT: check how many VM instances currently up before launching*** launch_vm = False while not launch_vm: # if above under $$$ threshold, create VM instance, else wait if check_num_active_vm_instances(config) < config.args.max_vm_instances: launch_vm = True else: time.sleep(config.args.vm_launch_wait_time_s) # wait before trying again # Launch fuzzing container on VM if not config.args.silent: print(LINE_SEP) print("Launching GCE VM to fuzz %s ..." % config.toplevel) print(LINE_SEP) cmd = [ "gcloud", "compute", "--project=%s" % config.gcp_params["project_id"], "instances", "create-with-container", config.experiment_name, "--container-image", config.docker_image, "--container-stdin", "--container-tty", "--container-privileged", "--container-restart-policy", config.gcp_params["container_restart_policy"], "--zone=%s" % config.gcp_params["zone"], "--machine-type=%s" % config.gcp_params["machine_type"], "--boot-disk-size=%s" % config.gcp_params["boot_disk_size"], "--scopes=%s" % config.gcp_params["scopes"], "--metadata=startup-script-url=gs://%s-%s/%s" % (config.gcp_params["project_id"], config.gcp_params["vm_management_bucket"], config.gcp_params["startup_script"]), ] # Open shell debugging if config.manual: cmd.append("--container-command=/bin/bash") # Set environment variables for general configs cmd.extend(["--container-env", "%s=%s" % ("TOPLEVEL", config.toplevel)]) cmd.extend(["--container-env", "%s=%s" % ("VERSION", config.version)]) cmd.extend(["--container-env", "%s=%s" % ("TB_TYPE", config.tb_type)]) cmd.extend(["--container-env", "%s=%s" % ("TB", config.tb)]) cmd.extend(["--container-env", "%s=%s" % ("FUZZER", config.fuzzer)]) cmd.extend( ["--container-env", "%s=%s" % ("INSTRUMENT_DUT", config.instrument_dut)]) cmd.extend( ["--container-env", "%s=%s" % ("INSTRUMENT_TB", config.instrument_tb)]) cmd.extend([ "--container-env", "%s=%s" % ("INSTRUMENT_VLTRT", config.instrument_vltrt) ]) cmd.extend(["--container-env", "%s=%s" % ("RUN_ON_GCP", config.run_on_gcp)]) # Set environment variables for Verilator/HDL-generator/fuzzer params for params in config.env_var_params: for param, value in params.items(): if value is not None: cmd.extend(["--container-env", "%s=%s" % (param.upper(), value)]) # launch container in VM instance error_str = "ERROR: launching VM on GCE FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("VM LAUNCH SUCCESSFUL -- Done!"))
def run_docker_container_locally(config, exp_data_path): """Runs a Docker container to fuzz the DUT on the local machine.""" if not config.args.silent: print(LINE_SEP) print("Running Docker container to fuzz %s ..." % config.toplevel) print(LINE_SEP) cmd = [ "docker", "run", "-it", "--rm", "--security-opt", "seccomp=unconfined", "--log-driver=%s" % config.args.log_driver, "--name", config.experiment_name, ] # Set environment variables for general configs cmd.extend(["-e", "%s=%s" % ("TOPLEVEL", config.toplevel)]) cmd.extend(["-e", "%s=%s" % ("VERSION", config.version)]) cmd.extend(["-e", "%s=%s" % ("TB_TYPE", config.tb_type)]) cmd.extend(["-e", "%s=%s" % ("TB", config.tb)]) cmd.extend(["-e", "%s=%s" % ("FUZZER", config.fuzzer)]) cmd.extend(["-e", "%s=%s" % ("INSTRUMENT_DUT", config.instrument_dut)]) cmd.extend(["-e", "%s=%s" % ("INSTRUMENT_TB", config.instrument_tb)]) cmd.extend(["-e", "%s=%s" % ("INSTRUMENT_VLTRT", config.instrument_vltrt)]) cmd.extend(["-e", "%s=%s" % ("RUN_ON_GCP", config.run_on_gcp)]) # Set environment variables for Verilator/HDL-generator/fuzzer params for params in config.env_var_params: for param, value in params.items(): if value is not None: cmd.extend(["-e", "%s=%s" % (param.upper(), value)]) # Mount volumes for output data cmd.extend( ["-v", "%s/logs:/src/hw/%s/logs" % (exp_data_path, config.toplevel)]) cmd.extend( ["-v", "%s/out:/src/hw/%s/out" % (exp_data_path, config.toplevel)]) # If manual mode, mount src code for development/debugging if config.manual: cmd.extend( ["-v", "%s/%s:/src/hw/hwfutils" % (config.root_path, HWFUTILS_PATH)]) cmd.extend(["-v", "%s/%s:/src/hw/tb" % (config.root_path, SHARED_TB_PATH)]) cmd.extend([ "-v", "%s/hw/%s:/src/hw/%s" % (config.root_path, config.toplevel, config.toplevel) ]) cmd.extend([ "-v", "%s/infra/base-sim/common.mk:/src/hw/common.mk" % config.root_path ]) cmd.extend( ["-v", "%s/infra/base-sim/exe.mk:/src/hw/exe.mk" % config.root_path]) for script in [ "run", "run-kcov", "run-llvm-cov", "run-vlt-cov", "set_hwf_isa.sh", "cpp-verilator-sim", ]: cmd.extend([ "-v", "%s/infra/base-sim/scripts/%s:/scripts/%s" % (config.root_path, script, script) ]) if config.fuzzer == "afl" or config.fuzzer == "afl-term-on-crash": for script in ["compile", "fuzz"]: cmd.extend([ "-v", "%s/infra/base-afl/%s:/scripts/%s" % (config.root_path, script, script) ]) # Set target Docker image and run cmd.extend(["-t", config.docker_image]) # If manual mode, start shell if config.manual: cmd.append("/bin/bash") error_str = "ERROR: container run FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("CONTAINER RUN SUCCESSFUL -- Done!"))