def load_bb_data(data_root): print(yellow("Loading data ...")) exp2data = {} # TODO: change this to automatically extract names from a single exp. number # extract each data file into a Pandas dataframe exp_combos = list(itertools.product(STATES, WIDTHS, EXP_BASE_NAMES)) for num_states, width, exp_base_name in exp_combos: for trial in TRIALS: # Build complete path to data files exp_name_wo_trialnum = exp_base_name % (num_states, width) exp_name = "%s-%d" % (exp_name_wo_trialnum, trial) data_path = os.path.join(data_root, exp_name) # Extract experiment info. exp_name_list = exp_name.split("-") instr_type = exp_name_list[6] if len(exp_name_list) > 9: fs_opt = True else: fs_opt = False # Load fuzzing data into an object exp2data[exp_name_wo_trialnum] = FuzzingData( num_states, width, instr_type, fs_opt, trial, data_path) print(green("Done.")) print(LINE_SEP) return exp2data
def _print_configs(self): exp_config_table = prettytable.PrettyTable(header=False) exp_config_table.title = "Experiment Parameters" exp_config_table.field_names = ["Parameter", "Value"] # Add main experiment parameters exp_config_table.add_row(["Experiment Name", self.experiment_name]) exp_config_table.add_row(["Toplevel", self.toplevel]) exp_config_table.add_row(["Version", self.version]) exp_config_table.add_row(["Testbench Type", self.tb_type]) exp_config_table.add_row(["Testbench", self.tb]) exp_config_table.add_row(["Fuzzer", self.fuzzer]) exp_config_table.add_row(["Instrument DUT", self.instrument_dut]) exp_config_table.add_row(["Instrument TB", self.instrument_tb]) exp_config_table.add_row(["Instrument VLT-RT", self.instrument_vltrt]) exp_config_table.add_row(["Manual", self.manual]) exp_config_table.add_row(["Run on GCP", self.run_on_gcp]) # Add other parameters for params in [self.gcp_params] + self.env_var_params: for param, value in params.items(): param = param.replace("_", " ").title() exp_config_table.add_row([param, value]) # Print table exp_config_table.align = "l" print(yellow(exp_config_table.get_string()))
def build_bbs_df(exp2data): print(yellow("Building basic block stats dataframe ...")) # Create empty dictionary that will be used to create Pandas # a DataFrame that look like the following: # +---------------------------------------------------+ # | # states | instrumentation level | # basic blocks | # +---------------------------------------------------+ # | ... | ... | ... | bbs_dict = { NUM_STATES_LABEL: [], INSTR_TYPE_LABEL: [], NUM_BB_LABEL: [], } for exp_name, fd in exp2data.items(): bbs_dict[NUM_STATES_LABEL].extend([fd.num_states] * 2) # bbs_dict[INSTR_TYPE_LABEL].extend(["Simulation Engine", "TB", "DUT"]) bbs_dict[INSTR_TYPE_LABEL].extend(["Simulation Engine + TB", "DUT"]) # bbs_dict[NUM_BB_LABEL].append(fd.vltrt_bbs) # bbs_dict[NUM_BB_LABEL].append(fd.tb_bbs) bbs_dict[NUM_BB_LABEL].append(fd.vltrt_bbs + fd.tb_bbs) bbs_dict[NUM_BB_LABEL].append(fd.dut_bbs) # bbs_dict[NUM_STATES_LABEL].append(fd.num_states) # bbs_dict[INSTR_TYPE_LABEL].append(INSTR_TYPE_MAPPINGS[fd.instr_type]) # if fd.instr_type == "full": # bbs_dict[NUM_BB_LABEL].append(fd.full_bbs) # elif fd.instr_type == "duttb": # bbs_dict[NUM_BB_LABEL].append(fd.duttb_bbs) # elif fd.instr_type == "dut": # bbs_dict[NUM_BB_LABEL].append(fd.dut_bbs) # else: # print(red("ERROR: unknown instrumentation type.")) # sys.exit(1) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(bbs_dict)
def compute_instr_type_mann_whitney(instr_rts): print( yellow( "Computing Mann-Whitney U-test on instrumentation complexity data ..." )) for num_states in STATES: sub_rt_df = instr_rts[instr_rts[NUM_STATES_LABEL] == num_states] full_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] == INSTR_TYPE_MAPPINGS["full"]][RUN_TIME_LABEL] duttb_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] == INSTR_TYPE_MAPPINGS["duttb"]][RUN_TIME_LABEL] dut_instr_data = sub_rt_df[sub_rt_df[INSTR_TYPE_LABEL] == INSTR_TYPE_MAPPINGS["dut"]][RUN_TIME_LABEL] # mw_full_duttb = stats.mannwhitneyu(full_instr_data, duttb_instr_data) mw_full_dut = stats.mannwhitneyu(full_instr_data, dut_instr_data) # mw_duttb_dut = stats.mannwhitneyu(duttb_instr_data, dut_instr_data) print("%d States - Mann-Whitney:" % num_states) # print( # "\t%s vs. %s:" % # (INSTR_TYPE_MAPPINGS["full"], INSTR_TYPE_MAPPINGS["duttb"]), # mw_full_duttb) print( "\t%s vs. %s:" % (INSTR_TYPE_MAPPINGS["full"], INSTR_TYPE_MAPPINGS["dut"]), mw_full_dut) # print( # "\t%s vs. %s:" % # (INSTR_TYPE_MAPPINGS["duttb"], INSTR_TYPE_MAPPINGS["dut"]), # mw_duttb_dut) print(green("Done.")) print(LINE_SEP)
def build_fs_opt_rts_df(exp2data): print(yellow("Building fork server optimization dataframe ...")) FS_OPT_BASELINE = True # Create empty dictionary that will be used to create Pandas # a DataFrame that look like the following: # +----------------------------------------------------+ # | # states | fork server optimization? | runtime (s) | # +----------------------------------------------------+ # | ... | ... | ... | runtimes_dict = { NUM_STATES_LABEL: [], OPT_TYPE_LABEL: [], RUN_TIME_LABEL: [], } # Aggregate data into a dictionary exp2rts = _aggregrate_fs_opt_rts(exp2data) # Compute scale factors for each set of num_states experiments states2scales = {} for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): if instr_type == "full" and fs_opt is FS_OPT_BASELINE: scale_factor = np.median(runtimes) states2scales[num_states] = scale_factor # Build the dataframe for plotting for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): runtimes = list(map(lambda x: x / states2scales[num_states], runtimes)) runtimes_dict[NUM_STATES_LABEL].extend([num_states] * len(runtimes)) runtimes_dict[OPT_TYPE_LABEL].extend([OPT_TYPE_MAPPINGS[fs_opt]] * len(runtimes)) runtimes_dict[RUN_TIME_LABEL].extend(runtimes) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(runtimes_dict)
def build_instr_complex_rts_df(exp2data): print(yellow("Building instruction complexity dataframe ...")) INSTR_TYPE_BASELINE = "dut" # Create empty dictionary that will be used to create Pandas # a DataFrame that look like the following: # +------------------------------------------------+ # | # states | instrumentation level | runtime (s) | # +------------------------------------------------+ # | ... | ... | ... | runtimes_dict = { NUM_STATES_LABEL: [], INSTR_TYPE_LABEL: [], RUN_TIME_LABEL: [], } # Aggregate data into a dictionary exp2rts = _aggregrate_instr_complex_rts(exp2data) # Compute scale factors for each set of num_states experiments states2scales = {} for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): if instr_type == INSTR_TYPE_BASELINE and fs_opt is False: scale_factor = np.median(runtimes) states2scales[num_states] = scale_factor # Build the dataframe for plotting for (num_states, instr_type, fs_opt), runtimes in exp2rts.items(): runtimes = list(map(lambda x: x / states2scales[num_states], runtimes)) runtimes_dict[NUM_STATES_LABEL].extend([num_states] * len(runtimes)) runtimes_dict[INSTR_TYPE_LABEL].extend([INSTR_TYPE_MAPPINGS[instr_type]] * len(runtimes)) runtimes_dict[RUN_TIME_LABEL].extend(runtimes) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(runtimes_dict)
def check_num_active_vm_instances(config): """Checks number of active VM instances on GCE as a $$$ safety measure.""" if not config.args.silent: print(LINE_SEP) print("Checking number of active VMs on GCE ...") print(LINE_SEP) cmd = [ "gcloud", "compute", "instances", "list", "--zones=%s" % config.gcp_params["zone"] ] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) num_active_vm_instances = -1 # first line is header while True: line = proc.stdout.readline() if not line: break num_active_vm_instances += 1 if num_active_vm_instances < config.args.max_vm_instances: if not config.args.silent: print(green("%d active VM(s)" % num_active_vm_instances)) else: if not config.args.silent: print(red("%d active VM(s)" % num_active_vm_instances)) print( yellow("waiting %d seconds and trying again ..." % config.args.vm_launch_wait_time_s)) return num_active_vm_instances
def plot_avg_coverage_vs_time_broken(hwf_cov_df, rfuzz_cov_df, time_units="m"): print(yellow("Generating plot ...")) # Set plot style and extract only HDL line coverage # sns.set_theme(context="notebook", style="darkgrid") hdl_cov_df = pd.concat([hwf_cov_df, rfuzz_cov_df]) # create subplots fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(6, 4)) # create figure and plot the data # fig, ax = plt.subplots(1, 1, figsize=(6, 4)) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, style=FUZZER_LABEL, ax=ax1) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, style=FUZZER_LABEL, ax=ax2) # set axis ranges ax1.set_ylim(0.3, 1.0) ax2.set_ylim(0, 0.05) # hide the spines between the two axes ax1.spines['bottom'].set_visible(False) ax2.spines['top'].set_visible(False) ax1.xaxis.tick_top() ax1.tick_params(labeltop=False) ax2.xaxis.tick_bottom() # format the plot if time_units == "m": time_units_label = "min." elif time_units == "h": time_units_label = "hours" else: time_units_label = "s" # ax1.set_xlabel(TIME_LABEL + " (%s)" % time_units_label, # fontsize=LABEL_FONT_SIZE) # ax1.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE) # ax1.tick_params("x", labelsize=TICK_FONT_SIZE) # ax1.tick_params("y", labelsize=TICK_FONT_SIZE) # plt.legend(fontsize=LEGEND_FONT_SIZE, # title_fontsize=LEGEND_TITLE_FONT_SIZE, # bbox_to_anchor=(1.01, 0.75), # loc='upper left') plt.tight_layout() # save the plot plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def run_cmd(cmd, error_str, silent=False, fail_silent=False): """Runs the provided command (list of strings) in a separate process.""" try: if not silent: print("Running command:") print(yellow(subprocess.list2cmdline(cmd))) subprocess.check_call(cmd) else: subprocess.check_call(cmd, stdout=subprocess.DEVNULL) except subprocess.CalledProcessError: if not fail_silent: print(red(error_str), file=sys.stderr) sys.exit(1)
def _verify_action(config, action, action_msg, abort_msg): if config.args.fail_silently: return True elif config.args.no: _abort(abort_msg) elif config.args.yes: action(config) else: ovw = input(yellow(action_msg)) if ovw in {"yes", "y", "Y", "YES", "Yes", ""}: action(config) else: _abort(abort_msg) return False
def compute_fs_opt_mann_whitney(instr_rts): print(yellow("Computing Mann-Whitney U-test on fork server opt. data ...")) for num_states in STATES: sub_rt_df = instr_rts[instr_rts[NUM_STATES_LABEL] == num_states] no_opt_data = sub_rt_df[sub_rt_df[OPT_TYPE_LABEL] == OPT_TYPE_MAPPINGS[False]][RUN_TIME_LABEL] opt_data = sub_rt_df[sub_rt_df[OPT_TYPE_LABEL] == OPT_TYPE_MAPPINGS[True]][RUN_TIME_LABEL] mw = stats.mannwhitneyu(no_opt_data, opt_data) print("%d States - Mann-Whitney:" % num_states) print("\t%s vs. %s:" % (OPT_TYPE_MAPPINGS[False], OPT_TYPE_MAPPINGS[True]), mw.pvalue) print(green("Done.")) print(LINE_SEP)
def push_docker_image_to_gcr(config): """Pushes docker image to GCR if it does not exist there yet.""" if not config.args.silent: print(LINE_SEP) print("Pushing Docker image to GCR ...") print(LINE_SEP) if config.args.update or not check_if_docker_image_exists_in_gcr(config): cmd = ["docker", "push", config.docker_image] error_str = "ERROR: pushing image to GCR FAILED. Terminating experiment!" run_cmd(cmd, error_str, silent=config.args.silent) if not config.args.silent: print(green("IMAGE PUSH SUCCESSFUL -- Done!")) else: if not config.args.silent: print(yellow("IMAGE ALREADY EXISTS IN GCR -- Done!"))
def load_fuzzing_data(hwf_exp_prefix, rfuzz_exp_prefix): print(yellow("Loading data ...")) exp2data = collections.defaultdict(list) for toplevel in TOPLEVELS: for trial in TRIALS: # Build complete path to data files exp_suffix = EXPERIMENT_SUFFIX % (toplevel.lower(), DURATION_MINS) hwf_data_path = "{}-{}-{}".format(hwf_exp_prefix, exp_suffix, trial) rfuzz_data_path = "{}/rfuzz-{}-{}".format(rfuzz_exp_prefix, exp_suffix, trial) # Load fuzzing data into an object exp2data[exp_suffix].append( FuzzingData(toplevel, DURATION_MINS, trial, hwf_data_path, rfuzz_data_path)) return exp2data
def plot_coverage_vs_time(coverage_dfs): print(yellow("Generating plots ...")) cov_metrics = [ SW_LINE_COVERAGE_LABEL, SW_REGION_COVERAGE_LABEL, HW_LINE_COVERAGE_LABEL ] num_cores = len(TOPLEVELS) num_cov_metrics = len(cov_metrics) sns.set_theme(context="notebook", style="darkgrid") fig, axes = plt.subplots(num_cov_metrics, num_cores, sharex=True, sharey=True) for trial in range(len(coverage_dfs)): # Select experiment trial number cov_df = coverage_dfs[trial] for row in range(len(axes)): # select portion of data corresponding to current COVERAGE METRIC sub_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == cov_metrics[row]] for col in range(len(axes[row])): # select portion of data corresponding to current core plt_df = sub_cov_df[sub_cov_df[TOPLEVEL_LABEL] == TOPLEVELS[col]] # sns.set_context("paper") curr_ax = sns.lineplot(data=plt_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=GRAMMAR_LABEL, ax=axes[row][col], legend=False) if row == 0 and col == 0 and trial == 0: lines = curr_ax.get_lines() axes[row][col].set_title("Coverage = %s | Core = %s" % (cov_metrics[row], TOPLEVELS[col])) fig.legend( lines, [ "Const. Opcode & Variable Frame", "Const. Opcode & Fixed Frame", "Mapped Opcode & Variable Frame", "Mapped Opcode & Fixed Frame", ], loc="lower center", ncol=4, ) print(green("Done.")) print(LINE_SEP) plt.show()
def _print_configs(args): # Create table to print configurations to STDIN config_table = prettytable.PrettyTable(header=False) config_table.title = "Seed Generation Parameters" config_table.field_names = ["Parameter", "Value"] # Add parameter values to table config_table.add_row(["Input (YAML) Filename", args.input_filename]) config_table.add_row(["Output Filename", args.output_filename]) config_table.add_row(["Frame Type", args.frame_type]) config_table.add_row(["Opcode Size (# bytes)", args.opcode_size]) config_table.add_row(["Address Size (# bytes)", args.address_size]) config_table.add_row(["Data Size (# bytes)", args.data_size]) # Print table config_table.align = "l" print(yellow(config_table.get_string()))
def build_coverage_df(exp2data, trial): print(yellow("Building coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | isa (grammar) | coverage type | time (s) | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], GRAMMAR_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } # Add rows to the dataframe for exp_name, fd_list in exp2data.items(): fd = fd_list[trial] for time, row in fd.afl_data.iterrows(): cov_df_idx = row["paths_total"] - 1 for _ in range(3): coverage_dict[TOPLEVEL_LABEL].append(fd.toplevel) coverage_dict[GRAMMAR_LABEL].append(fd.grammar) coverage_dict[TIME_LABEL].append(time) # Add kcov coverage kcov = fd.kcov_data.loc[cov_df_idx, "Line-Coverage-(%)"] coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(kcov * 100.0) # Add LLVM coverage llvm_cov = fd.llvm_cov_data.loc[cov_df_idx, "Region-Coverage-(%)"] coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(llvm_cov * 100.0) # Add Verilator coverage vlt_cov = (float(fd.vlt_cov_data.loc[cov_df_idx, "Lines-Covered"]) / float(fd.vlt_cov_data.loc[cov_df_idx, "Total-Lines"])) coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(vlt_cov * 100.0) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict)
def check_for_data_locally(config): """Checks if experiment data already exists locally.""" exp_data_path = "%s/data/%s" % (config.root_path, config.experiment_name) abort_str = "ABORT: re-run with different experiment name." if glob.glob(exp_data_path): if config.args.fail_silently: return True elif config.args.no: _abort(abort_str) elif config.args.yes: shutil.rmtree(exp_data_path) else: ovw = input(yellow("WARNING: experiment data exists. Overwrite? [Yn]")) if ovw in {"yes", "y", "Y", "YES", "Yes", ""}: shutil.rmtree(exp_data_path) else: _abort(abort_str) return False
def compute_stats(hwf_cov_dict, rfuzz_cov_dict): print(yellow("Computing stats ...")) # Compute HDL coverage % differences cov_diffs_sum = 0 for toplevel, hwf_cov in hwf_cov_dict.items(): min_hwf_cov = min(hwf_cov) max_rfuzz_cov = max(rfuzz_cov_dict[toplevel]) cov_diff = min_hwf_cov - max_rfuzz_cov cov_diffs_sum += cov_diff print("HWF vs. RFUZZ coverage (%15s): %.3f%%" % (toplevel, cov_diff)) cov_diffs_avg = float(cov_diffs_sum) / float(len(hwf_cov_dict.keys())) print("Avg. coverage difference: %.3f%%" % (cov_diffs_avg)) print('-' * len(LINE_SEP)) for toplevel, hwf_cov in hwf_cov_dict.items(): rfuzz_cov = rfuzz_cov_dict[toplevel] myu = stats.mannwhitneyu(hwf_cov, rfuzz_cov) print("HWF vs. RFUZZ Mann-Whitney (%15s): %s" % (toplevel, myu)) print(green("Done.")) print(LINE_SEP)
def plot_bbs(instr_bbs, orientation="h"): print(yellow("Generating plots ...")) LABEL_FONT_SIZE = 14 sns.set() if orientation == "h": ax = sns.barplot(y=NUM_STATES_LABEL, x=NUM_BB_LABEL, hue=INSTR_TYPE_LABEL, data=instr_bbs, orient="h", ci=None) ax.set_ylabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE) ax.set_xlabel(NUM_BB_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("y", labelsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=LABEL_FONT_SIZE) ax.set_xlim(10, 10000) plt.xscale("log") else: ax = sns.barplot(x=NUM_STATES_LABEL, y=NUM_BB_LABEL, hue=INSTR_TYPE_LABEL, data=instr_bbs, ci=None) ax.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE) ax.set_ylabel(NUM_BB_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=LABEL_FONT_SIZE) ax.tick_params("y", labelsize=LABEL_FONT_SIZE) ax.set_ylim(10, 10000) plt.yscale("log") plt.legend(title=INSTR_TYPE_LABEL, fontsize=LABEL_FONT_SIZE, title_fontsize=LABEL_FONT_SIZE, ncol=3, loc="upper center", bbox_to_anchor=(0.5, 1.25)) plt.tight_layout() plt.savefig("hwf_components_bbs.png", format="PNG") print(green("Done.")) print(LINE_SEP)
def load_fuzzing_data(afl_data_root, cov_data_root): print(yellow("Loading data ...")) exp2data = collections.defaultdict(list) # TODO: change this to automatically extract names from a single exp. number # extract each data file into a Pandas dataframe isas = list( itertools.product(TOPLEVELS, OPCODE_TYPES, INSTR_TYPES, TERMINATE_TYPES)) for toplevel, opcode_type, instr_type, terminate_type in isas: for trial in TRIALS: # Build complete path to data files exp_name_wo_trialnum = EXPERIMENT_BASE_NAME % ( toplevel, opcode_type, instr_type, terminate_type) exp_name_wo_trialnum = exp_name_wo_trialnum.replace("_", "-") exp_name = "%s-%d" % (exp_name_wo_trialnum, trial) afl_data_path = os.path.join(afl_data_root, exp_name) cov_data_path = os.path.join(cov_data_root, exp_name) # Load fuzzing data into an object exp2data[exp_name_wo_trialnum].append( FuzzingData(toplevel, opcode_type, instr_type, terminate_type, trial, afl_data_path, cov_data_path)) return exp2data
def plot_avg_coverage_vs_time(hwf_cov_df, rfuzz_cov_df, time_units="m"): print(yellow("Generating plot ...")) # Set plot style and extract only HDL line coverage sns.set_theme(context="notebook", style="darkgrid") hdl_cov_df = pd.concat([hwf_cov_df, rfuzz_cov_df]) # create figure and plot the data fig, ax = plt.subplots(1, 1, figsize=(6, 4)) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, style=FUZZER_LABEL, ax=ax) # format the plot if time_units == "m": time_units_label = "min." elif time_units == "h": time_units_label = "hours" else: time_units_label = "s" ax.set_xlabel(TIME_LABEL + " (%s)" % time_units_label, fontsize=LABEL_FONT_SIZE) ax.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=TICK_FONT_SIZE) ax.tick_params("y", labelsize=TICK_FONT_SIZE) plt.legend(fontsize=LEGEND_FONT_SIZE, title_fontsize=LEGEND_TITLE_FONT_SIZE, bbox_to_anchor=(1.01, 0.75), loc='upper left') plt.tight_layout() # save the plot plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def plot_avg_coverage_vs_time(cov_df, time_units="m"): print(yellow("Generating plot ...")) # Set plot style and extract only HDL line coverage sns.set_theme(context="notebook", style="darkgrid") hdl_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == HW_LINE_COVERAGE_LABEL] # create figure and plot the data fig, ax = plt.subplots(1, 1, figsize=(4, 2)) sns.lineplot(data=hdl_cov_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=TOPLEVEL_LABEL, ax=ax, markers="x") # format the plot if time_units == "m": time_units_label = "min." elif time_units == "h": time_units_label = "hours" else: time_units_label = "s" ax.set_xlabel(TIME_LABEL + " (%s)" % time_units_label, fontsize=LABEL_FONT_SIZE) ax.set_ylabel("HDL Line " + COVERAGE_LABEL, fontsize=LABEL_FONT_SIZE) ax.tick_params("x", labelsize=TICK_FONT_SIZE) ax.tick_params("y", labelsize=TICK_FONT_SIZE) plt.legend(title="Core", fontsize=LEGEND_FONT_SIZE, title_fontsize=LEGEND_TITLE_FONT_SIZE, ncol=2) plt.tight_layout() # save the plot plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def build_min_hwf_coverage_df(exp2data, time_units="m", normalize_to_start=False, consolidation="max"): print(yellow("Building HWF coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | fuzzer | coverage type | time | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], FUZZER_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } cov_dict = collections.defaultdict( list) # maps toplevel --> [coverage list] for exp_name, fd_list in exp2data.items(): # get min coverage experiment min_cov = get_max_vlt_cov(fd_list[0].hwf_cov_data) min_cov_fd = fd_list[0] for fd in fd_list: cov = get_max_vlt_cov(fd.hwf_cov_data) cov_dict[fd.toplevel].append(cov) if cov < min_cov: min_cov = cov min_cov_fd = fd # build data frame for plotting for time, row in min_cov_fd.hwf_afl_data.iterrows(): # scale time scaled_time = scale_time(time, time_units) # add circuit, fuzzer, and time values to dataframe row coverage_dict[TOPLEVEL_LABEL].append(min_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(scaled_time) # get the AFL paths_total at the current time paths_total = get_paths_total_at_time(time, min_cov_fd.hwf_afl_data) - 1 # get HWF coverage data hwf_vlt_cov = get_vlt_cov_at_time(paths_total, min_cov_fd.hwf_cov_data) # normalize to start time if requested if time == 0: hwf_vlt_cov_t0 = hwf_vlt_cov if normalize_to_start: hwf_vlt_cov /= hwf_vlt_cov_t0 # add to data frame coverage_dict[FUZZER_LABEL].append("HWFP") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(hwf_vlt_cov) # extend lines to max time value if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME: coverage_dict[TOPLEVEL_LABEL].append(min_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME) coverage_dict[FUZZER_LABEL].append("HWFP") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append( coverage_dict[COVERAGE_LABEL][-1]) print("Min. HW Line coverage (%15s): %.3f%%" % (min_cov_fd.toplevel, coverage_dict[COVERAGE_LABEL][-1])) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict), cov_dict
def plot_avg_coverage_vs_time(cov_df, time_units="m"): print(yellow("Generating plots ...")) cov_metrics = [ SW_LINE_COVERAGE_LABEL, SW_REGION_COVERAGE_LABEL, HW_LINE_COVERAGE_LABEL ] axis_limits = _get_axis_limits() sns.set_theme(context="notebook", style="darkgrid") # create figure and subplot axes fig, axes = plt.subplots(len(cov_metrics), len(TOPLEVELS), sharex="col", figsize=(10, 4)) # plot coverage traces for row in range(len(axes)): # select portion of data corresponding to current COVERAGE METRIC sub_cov_df = cov_df[cov_df[COVERAGE_TYPE_LABEL] == cov_metrics[row]] for col in range(len(axes[row])): # select portion of data corresponding to current core plt_df = sub_cov_df[sub_cov_df[TOPLEVEL_LABEL] == TOPLEVELS[col]] # TODO(ttrippel): there has to be a better way to do the following: # plot legend only for the first subplot so we can extract the labels if row == 0 and col == 0: plot_legend = True else: plot_legend = False ax = sns.lineplot(data=plt_df, x=TIME_LABEL, y=COVERAGE_LABEL, hue=GRAMMAR_LABEL, ax=axes[row][col], legend=plot_legend) ax.axhline(y=1.0, color='r', linestyle='-') # get legend info if we are plotting the first plot if plot_legend: lines = ax.get_lines() labels = _get_legend_labels(ax) # format each figure for row in range(len(axes)): for col in range(len(axes[row])): # get corresponding axis_limits if cov_metrics[row] == SW_LINE_COVERAGE_LABEL: ax_lims = axis_limits[TOPLEVELS[col]].kcov_limits elif cov_metrics[row] == SW_REGION_COVERAGE_LABEL: ax_lims = axis_limits[TOPLEVELS[col]].llvm_cov_limits else: ax_lims = axis_limits[TOPLEVELS[col]].vlt_cov_limits # format plot labels and axes subplot_title = "%s | %s" % (TOPLEVELS[col], cov_metrics[row]) _format_plot(axes[row][col], ax_lims, time_units, subplot_title) plt.tight_layout() # set legend # isa_label_mapping = { # "constant-variable-never": "Const. Opcode | Variable Frame", # "constant-fixed-never": "Const. Opcode | Fixed Frame", # "mapped-variable-never": "Mapped Opcode | Variable Frame", # "mapped-fixed-never": "Mapped Opcode | Fixed Frame", # } isa_label_mapping = { "constant-variable-never": "Constant | Variable", "constant-fixed-never": "Constant | Fixed", "mapped-variable-never": "Mapped | Variable", "mapped-fixed-never": "Mapped | Fixed", } clean_labels = [isa_label_mapping[isa_label] for isa_label in labels] fig.legend(lines, clean_labels, fontsize=LEGEND_FONT_SIZE, title_fontsize=LEGEND_TITLE_FONT_SIZE, loc="upper center", ncol=4, borderaxespad=0.05, bbox_to_anchor=(0.5, 0.12), title=GRAMMAR_LABEL + " (Opcode Format | Frame Format)") plt.subplots_adjust(bottom=0.23, wspace=0.25, hspace=0.25) # adjust figure layout and save to file plt.savefig(PLOT_FILE_NAME, format=PLOT_FORMAT) print(green("Done.")) print(LINE_SEP)
def build_avg_coverage_df(exp2data, time_units="m", normalize_to_start=False, consolidation="max"): print(yellow("Building average coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | isa (grammar) | coverage type | time (s) | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], GRAMMAR_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } for exp_name, fd_list in exp2data.items(): anchor_fd = fd_list[0] for time, row in anchor_fd.afl_data.iterrows(): # scale time if time_units == "h": scaled_time = float(time) / float(3600) elif time_units == "m": scaled_time = float(time) / float(60) else: scaled_time = time # add circuit, grammar, and time values to dataframe row for _ in range(3): coverage_dict[TOPLEVEL_LABEL].append(anchor_fd.toplevel) coverage_dict[GRAMMAR_LABEL].append(anchor_fd.grammar) coverage_dict[TIME_LABEL].append(scaled_time) # compute average coverage at all points in time kcov_avg = 0 llvm_cov_avg = 0 vlt_cov_avg = 0 kcov_max = 0 llvm_cov_max = 0 vlt_cov_max = 0 i = 0 for fd in fd_list: # get the paths_total at the current time paths_total = get_paths_total_at_time(time, fd.afl_data) - 1 # get coverage data # print(exp_name, i) kcov = get_cov_at_time(paths_total, fd.kcov_data, "Line-Coverage-(%)") kcov_avg += kcov kcov_max = max(kcov_max, kcov) llvm_cov = get_cov_at_time(paths_total, fd.llvm_cov_data, "Region-Coverage-(%)") llvm_cov_avg += llvm_cov llvm_cov_max = max(llvm_cov_max, llvm_cov) vlt_cov = get_vlt_cov_at_time(paths_total, fd.vlt_cov_data) vlt_cov_avg += vlt_cov vlt_cov_max = max(vlt_cov_max, vlt_cov) i += 1 kcov_avg /= float(len(fd_list)) llvm_cov_avg /= float(len(fd_list)) vlt_cov_avg /= float(len(fd_list)) # save time 0 coverage to normalize if time == 0: kcov_avg_t0 = kcov_avg llvm_cov_avg_t0 = llvm_cov_avg vlt_cov_avg_t0 = vlt_cov_avg if normalize_to_start: kcov_avg /= kcov_avg_t0 llvm_cov_avg /= llvm_cov_avg_t0 vlt_cov_avg /= vlt_cov_avg_t0 coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) if consolidation == "avg": coverage_dict[COVERAGE_LABEL].append(kcov_avg) coverage_dict[COVERAGE_LABEL].append(llvm_cov_avg) coverage_dict[COVERAGE_LABEL].append(vlt_cov_avg) else: coverage_dict[COVERAGE_LABEL].append(kcov_max) coverage_dict[COVERAGE_LABEL].append(llvm_cov_max) coverage_dict[COVERAGE_LABEL].append(vlt_cov_max) # extend lines to max time value if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME: for _ in range(3): coverage_dict[TOPLEVEL_LABEL].append(anchor_fd.toplevel) coverage_dict[GRAMMAR_LABEL].append(anchor_fd.grammar) coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME) coverage_dict[COVERAGE_TYPE_LABEL].append(SW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(SW_REGION_COVERAGE_LABEL) coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].extend( coverage_dict[COVERAGE_LABEL][-3:]) # print("Max SW Line coverage: ", coverage_dict[COVERAGE_LABEL][-3]) # print("Max SW Basic Block coverage:", coverage_dict[COVERAGE_LABEL][-2]) print("Max HW Line coverage: ", coverage_dict[COVERAGE_LABEL][-1]) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict)
def plot_opt_strategies(instr_rts, fsopt_rts, plot_type="violin"): print(yellow("Generating plots ...")) LABEL_FONT_SIZE = 14 sns.set() # HW fuzzing instrumentation levels if plot_type == "violin": ax1 = sns.violinplot(x=NUM_STATES_LABEL, y=RUN_TIME_LABEL, hue=INSTR_TYPE_LABEL, data=instr_rts) else: ax1 = sns.stripplot(x=NUM_STATES_LABEL, y=RUN_TIME_LABEL, hue=INSTR_TYPE_LABEL, data=instr_rts, dodge=True, jitter=0.3, size=MARKER_SIZE) ax1.axhline(y=1.0, color='r', linestyle='-') ax1.set_ylim(0.5, 3) ax1.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE) ax1.set_ylabel(RUN_TIME_LABEL, fontsize=LABEL_FONT_SIZE) ax1.tick_params("x", labelsize=LABEL_FONT_SIZE) ax1.tick_params("y", labelsize=LABEL_FONT_SIZE) plt.legend(title=INSTR_TYPE_LABEL, fontsize=LABEL_FONT_SIZE, title_fontsize=LABEL_FONT_SIZE) plt.tight_layout() # plt.savefig("hwf_instrumentation_levels.png", format="png") plt.savefig("hwf_instrumentation_levels.pdf", format="pdf") plt.close() # HW fork server optimization if plot_type == "violin": ax2 = sns.violinplot(x=NUM_STATES_LABEL, y=RUN_TIME_LABEL, hue=OPT_TYPE_LABEL, data=fsopt_rts) else: ax2 = sns.stripplot(x=NUM_STATES_LABEL, y=RUN_TIME_LABEL, hue=OPT_TYPE_LABEL, data=fsopt_rts, dodge=True, jitter=0.3, size=MARKER_SIZE) ax2.axhline(y=1.0, color='r', linestyle='-') ax1.set_ylim(0.5, 3) ax2.set_xlabel(NUM_STATES_LABEL, fontsize=LABEL_FONT_SIZE) ax2.set_ylabel(RUN_TIME_LABEL, fontsize=LABEL_FONT_SIZE) ax2.tick_params("x", labelsize=LABEL_FONT_SIZE) ax2.tick_params("y", labelsize=LABEL_FONT_SIZE) plt.legend(title=OPT_TYPE_LABEL, fontsize=LABEL_FONT_SIZE, title_fontsize=LABEL_FONT_SIZE) plt.tight_layout() # plt.savefig("hwf_fs_opt.png", format="png") plt.savefig("hwf_fs_opt.pdf", format="pdf") print(green("Done.")) print(LINE_SEP)
def build_max_rfuzz_coverage_df(exp2data, time_units="m", normalize_to_start=False, consolidation="max"): print(yellow("Building RFUZZ coverage dataframe ...")) # Create empty dictionary that will be used to create a Pandas DataFrame that # looks like the following: # +--------------------------------------------------------------------+ # | toplevel | fuzzer | coverage type | time | coverage (%) | # +--------------------------------------------------------------------+ # | ... | ... | ... | ... | ... | coverage_dict = { TOPLEVEL_LABEL: [], FUZZER_LABEL: [], COVERAGE_TYPE_LABEL: [], TIME_LABEL: [], COVERAGE_LABEL: [], } cov_dict = collections.defaultdict( list) # maps toplevel --> [coverage list] for exp_name, fd_list in exp2data.items(): # get max coverage experiment max_cov = get_max_vlt_cov(fd_list[0].rfuzz_cov_data) max_cov_fd = fd_list[0] for fd in fd_list: cov = get_max_vlt_cov(fd.rfuzz_cov_data) cov_dict[fd.toplevel].append(cov) if cov > max_cov: max_cov = cov max_cov_fd = fd for test_id, row in max_cov_fd.rfuzz_data.iterrows(): # scale time scaled_time = scale_time(row["Time (s)"], time_units) # add circuit, fuzzer, and time values to dataframe row coverage_dict[TOPLEVEL_LABEL].append(max_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(scaled_time) # compute average coverage at all points in time rfuzz_vlt_cov = get_vlt_cov_at_time(test_id, max_cov_fd.rfuzz_cov_data) # save time 0 coverage to normalize if requested if test_id == 0: rfuzz_vlt_cov_t0 = rfuzz_vlt_cov if normalize_to_start: rfuzz_vlt_cov /= rfuzz_vlt_cov_t0 # add coverage to dataframe row coverage_dict[FUZZER_LABEL].append("RFUZZ") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append(rfuzz_vlt_cov) # extend lines to max time value if coverage_dict[TIME_LABEL][-1] != SCALED_MAX_PLOT_TIME: coverage_dict[TOPLEVEL_LABEL].append(max_cov_fd.toplevel) coverage_dict[TIME_LABEL].append(SCALED_MAX_PLOT_TIME) coverage_dict[FUZZER_LABEL].append("RFUZZ") coverage_dict[COVERAGE_TYPE_LABEL].append(HW_LINE_COVERAGE_LABEL) coverage_dict[COVERAGE_LABEL].append( coverage_dict[COVERAGE_LABEL][-1]) print("Max HW Line coverage (%15s): %.3f%%" % (max_cov_fd.toplevel, coverage_dict[COVERAGE_LABEL][-1])) print(green("Done.")) print(LINE_SEP) return pd.DataFrame.from_dict(coverage_dict), cov_dict
def fuzz(argv): """Runs fuzzing experiment with provided configuration filename.""" # Parse cmd args module_description = "Hardware Fuzzing Pipeline" parser = argparse.ArgumentParser(description=module_description) parser.add_argument("--fail-silently", action="store_true", help="Fail silently if data/VM already exists.") parser.add_argument("-s", "--silent", action="store_true", help="Supress stdout messages printed by HWFP.") parser.add_argument("--log-driver", choices=[ "none", "json-file", "syslog", "journald", "gelf", "fluentd", "awslogs", "splunk", ], default="json-file", help="Docker logging driver to use.") parser.add_argument("-n", "--no", action="store_true", help="No to all prompts. (overides -y)") parser.add_argument("-y", "--yes", action="store_true", help="Yes to all prompts.") parser.add_argument("-u", "--update", action="store_true", help="Update Docker image in GCR.") parser.add_argument("--max-vm-instances", default=32, help="Max number of VM instances allowed on GCP zone.") parser.add_argument("--vm-launch-wait-time-s", default=30, help="Max number of VM instances allowed on GCP zone.") parser.add_argument("--gcp-config-filename", default="gcp_config.hjson", help="GCP vonfiguration file in the HJSON format.") parser.add_argument("config_filename", metavar="config.hjson", help="Configuration file in the HJSON format.") args = parser.parse_args(argv) # Load experiment configurations config = Config(args) # Check if experiment data already exists if config.run_on_gcp == 0: if check_for_data_locally(config): if not config.args.silent: print(yellow("WARNING: experiment data exists locally... skipping.")) return else: if check_for_data_in_gcs(config): if not config.args.silent: print(yellow("WARNING: experiment data exists in GCS... skipping.")) return if check_if_gce_vm_up(config): if not config.args.silent: print(yellow("WARNING: experiment VM is already running... skipping.")) return # Build docker image to fuzz target toplevel build_docker_image(config) # Run Docker container to fuzz toplevel if config.run_on_gcp == 0: exp_data_path = create_local_experiment_data_dir(config) run_docker_container_locally(config, exp_data_path) else: push_docker_image_to_gcr(config) push_vm_management_scripts_to_gcs(config) run_docker_container_on_gce(config)